Merge pull request #30 from soxoj/vimeo

Vimeo activation, tags updates

maigret/activation.py

@@ -10,3 +10,12 @@ class ParsingActivator:
         j = r.json()
         guest_token = j[site.activation['src']]
         site.headers['x-guest-token'] = guest_token
+
+    @staticmethod
+    def vimeo(site, logger):
+        headers = dict(site.headers)
+        if 'Authorization' in headers:
+            del headers['Authorization']
+        r = requests.get(site.activation['url'], headers=headers)
+        jwt_token = r.json()['jwt']
+        site.headers['Authorization'] = 'jwt ' + jwt_token

maigret/maigret.py

@@ -182,7 +182,7 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig
         error_text, site_error_text = detect_error_page(html_text, status_code, failure_errors,
                                                         site.ignore_403)
 
-    if site.activation:
+    if site.activation and html_text:
         is_need_activation = any([s for s in site.activation['marks'] if s in html_text])
         if is_need_activation:
             method = site.activation['method']
@@ -196,8 +196,8 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig
     # presense flags
     # True by default
     presense_flags = site.presense_strs
+    is_presense_detected = False
     if html_text:
-        is_presense_detected = False
         if not presense_flags:
             is_presense_detected = True
             site.stats['presense_flag'] = None

maigret/resources/data.json

@@ -309,8 +309,7 @@
         "500px": {
             "tags": [
                 "global",
-                "in",
+                "photo"
-                "photos"
             ],
             "errors": {
                 "INTERNAL_SERVER_ERROR": "Site error",
@@ -1004,14 +1003,17 @@
         },
         "Aptoide": {
             "tags": [
-                "in"
+                "apps"
             ],
             "regexCheck": "^[^\\.]+$",
             "checkType": "status_code",
+            "presenseStrs": [
+                "BreadcrumbList"
+            ],
             "alexaRank": 6653,
             "url": "https://{username}.en.aptoide.com/",
             "urlMain": "https://en.aptoide.com/",
-            "usernameClaimed": "blue",
+            "usernameClaimed": "kodi",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Aqa": {
@@ -1552,10 +1554,8 @@
         "Badoo": {
             "disabled": true,
             "tags": [
-                "br",
+                "global",
-                "de",
+                "dating"
-                "pl",
-                "social"
             ],
             "checkType": "status_code",
             "alexaRank": 2343,
@@ -3224,9 +3224,8 @@
         "DeviantART": {
             "tags": [
                 "global",
-                "images",
+                "photo",
-                "photos",
+                "art"
-                "us"
             ],
             "regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$",
             "checkType": "status_code",
@@ -3984,12 +3983,8 @@
         },
         "EyeEm": {
             "tags": [
-                "de",
                 "global",
-                "in",
+                "photo"
-                "it",
-                "photos",
-                "sd"
             ],
             "checkType": "message",
             "absenceStrs": "Not Found (404) | EyeEm",
@@ -4085,9 +4080,7 @@
         },
         "Facebook": {
             "tags": [
-                "global",
+                "global"
-                "social",
-                "us"
             ],
             "regexCheck": "^[a-zA-Z0-9_\\.]{3,49}(?<!\\.com|\\.org|\\.net)$",
             "checkType": "status_code",
@@ -9857,8 +9850,7 @@
             "tags": [
                 "global",
                 "instagram",
-                "photos",
+                "photo"
-                "us"
             ],
             "checkType": "message",
             "absenceStrs": [
@@ -9919,9 +9911,8 @@
         },
         "Pinterest": {
             "tags": [
-                "images",
+                "art",
-                "photos",
+                "photo"
-                "us"
             ],
             "checkType": "status_code",
             "alexaRank": 152,
@@ -10872,8 +10863,7 @@
         "Reddit": {
             "tags": [
                 "news",
-                "social",
+                "discussions"
-                "us"
             ],
             "checkType": "status_code",
             "presenseStrs": [
@@ -13372,9 +13362,7 @@
         },
         "TryHackMe": {
             "tags": [
-                "be",
+                "hacking"
-                "in",
-                "us"
             ],
             "checkType": "status_code",
             "alexaRank": 35003,
@@ -13386,8 +13374,7 @@
         "Tumblr": {
             "tags": [
                 "blogs",
-                "global",
+                "global"
-                "us"
             ],
             "regexCheck": "^[^\\.]+$",
             "checkType": "status_code",
@@ -13410,7 +13397,7 @@
         },
         "Twitch": {
             "tags": [
-                "us"
+                "streaming"
             ],
             "urlProbe": "https://m.twitch.tv/{username}",
             "checkType": "status_code",
@@ -13422,9 +13409,7 @@
         },
         "Twitter": {
             "tags": [
-                "global",
+                "global"
-                "social",
-                "us"
             ],
             "headers": {
                 "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
@@ -13614,8 +13599,7 @@
         "VK": {
             "tags": [
                 "global",
-                "ru",
+                "ru"
-                "social"
             ],
             "checkType": "response_url",
             "alexaRank": 23,
@@ -13788,9 +13772,20 @@
         },
         "Vimeo": {
             "tags": [
-                "us",
+                "global",
                 "video"
             ],
+            "headers": {
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTA4MDIxODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.x1M4yvCF7fqekRpFfgxjtevGuEZkTP6HK8FXAxlAAQk"
+            },
+            "activation": {
+                "url": "https://vimeo.com/_rv/viewer",
+                "marks": [
+                    "Something strange occurred. Please contact the app owners."
+                ],
+                "method": "vimeo"
+            },
+            "urlProbe": "https://api.vimeo.com/users/{username}?fields=name%2Cgender%2Cbio%2Curi%2Clink%2Cbackground_video%2Clocation_details%2Cpictures%2Cverified%2Cmetadata.public_videos.total%2Cavailable_for_hire%2Ccan_work_remotely%2Cmetadata.connections.videos.total%2Cmetadata.connections.albums.total%2Cmetadata.connections.followers.total%2Cmetadata.connections.following.total%2Cmetadata.public_videos.total%2Ctotal_collection_count%2Ccreated_time%2Cprofile_preferences%2Cmembership%2Cclients%2Cskills%2Cproject_types%2Crates%2Ccategories&fetch_user_profile=1",
             "checkType": "status_code",
             "alexaRank": 158,
             "url": "https://vimeo.com/{username}",
@@ -14063,7 +14058,7 @@
         "Wattpad": {
             "tags": [
                 "in",
-                "social"
+                "reading"
             ],
             "checkType": "message",
             "absenceStrs": "userError-404",
@@ -14109,9 +14104,9 @@
         },
         "We Heart It": {
             "tags": [
-                "in",
+                "photo",
-                "photos",
+                "global",
-                "us"
+                "blogs"
             ],
             "checkType": "message",
             "absenceStrs": "Oops! You've landed on a moving target!",
@@ -15471,8 +15466,8 @@
         },
         "devRant": {
             "tags": [
-                "in",
+                "global",
-                "social"
+                "coding"
             ],
             "checkType": "response_url",
             "alexaRank": 133141,
@@ -16165,8 +16160,7 @@
         },
         "geocaching": {
             "tags": [
-                "de",
+                "hobby"
-                "social"
             ],
             "checkType": "status_code",
             "alexaRank": 17327,
@@ -17718,8 +17712,7 @@
         },
         "social.tchncs.de": {
             "tags": [
-                "in",
+                "de"
-                "vn"
             ],
             "checkType": "status_code",
             "alexaRank": 441382,