diff --git a/maigret/checking.py b/maigret/checking.py
index a32971b..28e7be9 100644
--- a/maigret/checking.py
+++ b/maigret/checking.py
@@ -13,6 +13,7 @@ import tqdm.asyncio
from aiohttp_socks import ProxyConnector
from python_socks import _errors as proxy_errors
from socid_extractor import extract
+from aiohttp.client_exceptions import ServerDisconnectedError, ClientConnectorError
from .activation import ParsingActivator, import_aiohttp_cookies
from . import errors
@@ -64,8 +65,10 @@ async def get_response(request_future, logger) -> Tuple[str, int, Optional[Check
except asyncio.TimeoutError as e:
error = CheckError("Request timeout", str(e))
- except aiohttp.client_exceptions.ClientConnectorError as e:
+ except ClientConnectorError as e:
error = CheckError("Connecting failure", str(e))
+ except ServerDisconnectedError as e:
+ error = CheckError("Server disconnected", str(e))
except aiohttp.http_exceptions.BadHttpMessage as e:
error = CheckError("HTTP", str(e))
except proxy_errors.ProxyError as e:
@@ -155,7 +158,7 @@ def process_site_result(
# additional check for errors
if status_code and not check_error:
check_error = detect_error_page(
- html_text, status_code, site.errors, site.ignore403
+ html_text, status_code, site.errors_dict, site.ignore403
)
# parsing activation
diff --git a/maigret/maigret.py b/maigret/maigret.py
index 57cd96c..2ad6d41 100755
--- a/maigret/maigret.py
+++ b/maigret/maigret.py
@@ -48,7 +48,7 @@ def notify_about_errors(search_results: QueryResultWrapper, query_notify):
text = f'Too many errors of type "{e["err"]}" ({e["perc"]}%)'
solution = errors.solution_of(e['err'])
if solution:
- text = '. '.join([text, solution])
+ text = '. '.join([text, solution.capitalize()])
query_notify.warning(text, '!')
was_errs_displayed = True
diff --git a/maigret/resources/data.json b/maigret/resources/data.json
index 5757dc6..380feb7 100644
--- a/maigret/resources/data.json
+++ b/maigret/resources/data.json
@@ -8676,8 +8676,7 @@
},
"Mobypicture": {
"tags": [
- "in",
- "nl"
+ "photo"
],
"checkType": "message",
"absenceStrs": [
@@ -9477,16 +9476,17 @@
"usernameUnclaimed": "noonewouldeverusethis7"
},
"Newgrounds": {
- "tags": [
- "us"
+ "absenceStrs": [
+ "icon-steam"
+ ],
+ "presenseStrs": [
+ "user-header-name"
],
- "regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$",
- "checkType": "status_code",
- "alexaRank": 5748,
- "urlMain": "https://newgrounds.com",
"url": "https://{username}.newgrounds.com",
- "usernameClaimed": "blue",
- "usernameUnclaimed": "noonewouldeverusethis7"
+ "urlMain": "https://newgrounds.com",
+ "usernameClaimed": "john",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message"
},
"Newreporter": {
"tags": [
@@ -10308,8 +10308,7 @@
},
"Picsart": {
"tags": [
- "in",
- "us"
+ "photo"
],
"checkType": "status_code",
"alexaRank": 9103,
@@ -12755,7 +12754,7 @@
"us"
],
"headers": {
- "authorization": "Bearer BQC9jhqCGwU7SIBfusXOot9I_1wQ2F_4pS7ySsOwO20H9jWZE2ICJ-6rQ6gITvtIrXu6WbDLBTBUB4Ms1wI"
+ "authorization": "Bearer BQB2AjQkezkQ8gs4IPTf-DaiUD5xf0_agBYgfI8wGAI-tdC7GANMDZMwiYBls8kfD_bIhmtQoYwNfUgB3rg"
},
"errors": {
"Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
@@ -14140,7 +14139,7 @@
"sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
"authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
- "x-guest-token": "1390788770346147842"
+ "x-guest-token": "1391077876263444481"
},
"errors": {
"Bad guest token": "x-guest-token update required"
@@ -14540,7 +14539,7 @@
"video"
],
"headers": {
- "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjA0MjU3NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.hn10YMXkg1jk0qy4XBCxdfeN5kbWzjzJcYD8J84mjWk"
+ "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjA0OTU2NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.p2UxuvEPXrKk6wE_qMze9n9KeZIGo_uZ6nAE-UFKGJs"
},
"activation": {
"url": "https://vimeo.com/_rv/viewer",
@@ -17068,7 +17067,8 @@
},
"gfycat": {
"tags": [
- "sharing"
+ "sharing",
+ "photo"
],
"checkType": "status_code",
"alexaRank": 2217,
@@ -24424,9 +24424,7 @@
},
"1x": {
"tags": [
- "in",
- "photo",
- "us"
+ "photo"
],
"checkType": "message",
"presenseStrs": [
@@ -26245,6 +26243,202 @@
"Cookie": "SUB=_2AkMXyuc_f8NxqwJRmP8SyWPrbo13zAvEieKhlhbkJRMxHRl-123"
},
"checkType": "message"
+ },
+ "Hatena": {
+ "absenceStrs": [
+ "404 Not Found"
+ ],
+ "presenseStrs": [
+ "profile",
+ "myprofile",
+ "profile-dt",
+ "profile-dd",
+ "hatena-profile"
+ ],
+ "url": "http://profile.hatena.com/{username}/",
+ "urlMain": "http://profile.hatena.com",
+ "usernameClaimed": "john",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message"
+ },
+ "angel.co": {
+ "absenceStrs": [
+ "render_not_found"
+ ],
+ "presenseStrs": [
+ "Profile",
+ "profiles",
+ "User profile",
+ "name",
+ "layouts/profile"
+ ],
+ "url": "https://angel.co/u/{username}",
+ "urlMain": "https://angel.co",
+ "usernameClaimed": "john",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message"
+ },
+ "nelubit.ru": {
+ "urlMain": "https://nelubit.ru",
+ "engine": "phpBB/Search",
+ "usernameClaimed": "alex",
+ "usernameUnclaimed": "noonewouldeverusethis7"
+ },
+ "actual-porn.org": {
+ "urlMain": "http://actual-porn.org",
+ "engine": "vBulletin",
+ "usernameClaimed": "alex",
+ "usernameUnclaimed": "noonewouldeverusethis7"
+ },
+ "Aminus3": {
+ "absenceStrs": [
+ "Expires",
+ " no-cache"
+ ],
+ "presenseStrs": [
+ "image/ico",
+ " title="
+ ],
+ "url": "https://{username}.aminus3.com/",
+ "urlMain": "https://aminus3.com",
+ "usernameClaimed": "beautifulworld",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message",
+ "tags": [
+ "photo"
+ ]
+ },
+ "lomography": {
+ "absenceStrs": [
+ "404 \u00b7 Lomography"
+ ],
+ "presenseStrs": [
+ "Lomography",
+ " @lomography"
+ ],
+ "url": "https://www.lomography.com/homes/{username}",
+ "urlMain": "https://www.lomography.com",
+ "usernameClaimed": "steved7755",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message",
+ "tags": [
+ "photo"
+ ]
+ },
+ "jAlbum.net": {
+ "absenceStrs": [
+ "section",
+ " error_head"
+ ],
+ "regexCheck": "^[^\\.]+$",
+ "presenseStrs": [
+ "alternate",
+ " og:image"
+ ],
+ "url": "https://{username}.jalbum.net/",
+ "urlMain": "https://jalbum.net",
+ "usernameClaimed": "laza",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message",
+ "tags": [
+ "photo"
+ ]
+ },
+ "23hq": {
+ "absenceStrs": [
+ "my-modal",
+ " enable",
+ " modal"
+ ],
+ "presenseStrs": [
+ "frame",
+ "first active",
+ "user",
+ "last",
+ "country-name"
+ ],
+ "url": "http://www.23hq.com/{username}",
+ "urlMain": "http://www.23hq.com",
+ "usernameClaimed": "nellyb",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message",
+ "tags": [
+ "photo"
+ ]
+ },
+ "bliphoto": {
+ "absenceStrs": [
+ "Your photo journal | Blipfoto"
+ ],
+ "presenseStrs": [
+ "biography",
+ "biography-full",
+ "profile-sidebar",
+ "profile-content",
+ "state"
+ ],
+ "url": "https://www.blipfoto.com/{username}",
+ "urlMain": "https://www.blipfoto.com",
+ "usernameClaimed": "Wildstar",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message",
+ "tags": [
+ "photo"
+ ]
+ },
+ "Fotki": {
+ "absenceStrs": [
+ "'404 - Member Not Found'"
+ ],
+ "presenseStrs": [
+ "profile-cities",
+ "profile-friends",
+ "profile-aboutme",
+ "profile-country",
+ "user_profile_info"
+ ],
+ "url": "https://members.fotki.com/{username}/about/",
+ "urlMain": "https://fotki.com",
+ "usernameClaimed": "normargab",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message",
+ "tags": [
+ "photo"
+ ]
+ },
+ "viewbug": {
+ "absenceStrs": [
+ "missing-photos"
+ ],
+ "presenseStrs": [
+ "profile",
+ " profile_content"
+ ],
+ "url": "https://www.viewbug.com/member/{username}",
+ "urlMain": "https://www.viewbug.com",
+ "usernameClaimed": "melaniejwood",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message",
+ "tags": [
+ "photo"
+ ]
+ },
+ "Piccsy": {
+ "absenceStrs": [
+ "my-modal"
+ ],
+ "presenseStrs": [
+ "Username"
+ ],
+ "regexCheck": "^[^\\.]+$",
+ "url": "http://{username}.piccsy.com/",
+ "urlMain": "http://piccsy.com",
+ "usernameClaimed": "orientcement",
+ "usernameUnclaimed": "noonewouldeverusethis7",
+ "checkType": "message",
+ "tags": [
+ "Photo"
+ ]
}
},
"engines": {
diff --git a/maigret/sites.py b/maigret/sites.py
index d6b56b7..5cb126b 100644
--- a/maigret/sites.py
+++ b/maigret/sites.py
@@ -182,6 +182,14 @@ class MaigretSite:
return result
+ @property
+ def errors_dict(self) -> dict:
+ errors: Dict[str, str] = {}
+ if self.engine_obj:
+ errors.update(self.engine_obj.site.get('errors', {}))
+ errors.update(self.errors)
+ return errors
+
def get_url_type(self) -> str:
url = URLMatcher.extract_main_part(self.url)
if url.startswith("{username}"):
diff --git a/maigret/submit.py b/maigret/submit.py
index b3812e4..599b50a 100644
--- a/maigret/submit.py
+++ b/maigret/submit.py
@@ -200,6 +200,7 @@ async def check_features_manually(
# cookies
cookie_dict = None
if cookie_file:
+ logger.info(f'Use {cookie_file} for cookies')
cookie_jar = await import_aiohttp_cookies(cookie_file)
cookie_dict = {c.key: c.value for c in cookie_jar}
@@ -328,17 +329,22 @@ async def submit_dialog(db, url_exists, cookie_file, logger):
print(
"Try to run this mode again and increase features count or choose others."
)
+ return False
else:
if (
input(
f"Site {chosen_site.name} successfully checked. Do you want to save it in the Maigret DB? [Yn] "
- ).lower()
- in "y"
+ )
+ .lower()
+ .strip("y")
):
- logger.debug(chosen_site.json)
- site_data = chosen_site.strip_engine_data()
- logger.debug(site_data.json)
- db.update_site(site_data)
- return True
+ return False
- return False
+ chosen_site.name = input("Change site name if you want: ") or chosen_site.name
+ chosen_site.tags = input("Site tags: ").split(',')
+
+ logger.debug(chosen_site.json)
+ site_data = chosen_site.strip_engine_data()
+ logger.debug(site_data.json)
+ db.update_site(site_data)
+ return True
diff --git a/tests/test_sites.py b/tests/test_sites.py
index 988132e..75700c8 100644
--- a/tests/test_sites.py
+++ b/tests/test_sites.py
@@ -103,6 +103,7 @@ def test_saving_site_error():
amperka = db.sites[0]
assert len(amperka.errors) == 2
+ assert len(amperka.errors_dict) == 2
assert amperka.strip_engine_data().errors == {'error1': 'text1'}
assert amperka.strip_engine_data().json['errors'] == {'error1': 'text1'}