diff --git a/maigret.py b/maigret.py new file mode 100755 index 0000000..0bb4fde --- /dev/null +++ b/maigret.py @@ -0,0 +1,18 @@ +#! /usr/bin/env python3 +import asyncio +import sys + +from maigret.maigret import main + + +def run(): + try: + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) + except KeyboardInterrupt: + print('Maigret is interrupted.') + sys.exit(1) + + +if __name__ == "__main__": + run() \ No newline at end of file diff --git a/maigret/maigret.py b/maigret/maigret.py index eb2e340..c3c2c96 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -1,5 +1,3 @@ -#! /usr/bin/env python3 - """ Maigret main module """ @@ -202,7 +200,13 @@ async def maigret(username, site_data, query_notify, logger, headers.update(net_info["headers"]) # URL of user on site (if it exists) - url = net_info.get('url').format(username) + url = net_info.get('url').format( + urlMain=net_info['urlMain'], + urlSubpath=net_info.get('urlSubpath', ''), + username=username + ) + # workaround to prevent slash errors + url = url.replace('///', '/') # Don't make request if username is invalid for the site regex_check = net_info.get("regexCheck") @@ -226,7 +230,12 @@ async def maigret(username, site_data, query_notify, logger, else: # There is a special URL for probing existence separate # from where the user profile normally can be found. - url_probe = url_probe.format(username) + url_probe = url_probe.format( + urlMain=net_info['urlMain'], + urlSubpath=net_info.get('urlSubpath', ''), + username=username, + ) + if net_info["errorType"] == 'status_code' and net_info.get("request_head_only", True): # In most cases when we are detecting by status code, @@ -520,6 +529,7 @@ async def site_self_check(site_name, site_data, logger): async def self_check(json_file, logger): + data = json.load(open(json_file)) sites = SitesInformation(json_file) all_sites = {} @@ -552,7 +562,8 @@ async def self_check(json_file, logger): print(f'{message} {total_disabled} checked sites. Run with `--info` flag to get more information') with open(json_file, 'w') as f: - json.dump(all_sites, f, indent=4) + data['sites'] = all_sites + json.dump(data, f, indent=4) async def main(): @@ -862,7 +873,8 @@ async def main(): def run(): try: - asyncio.run(main()) + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) except KeyboardInterrupt: print('Maigret is interrupted.') sys.exit(1) diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 7538c2f..5908950 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -5,7 +5,8 @@ "errorMsg": [ "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d. \u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430, \u0432\u0432\u0435\u0434\u0438\u0442\u0435 \u0434\u0440\u0443\u0433\u043e\u0435 \u0438\u043c\u044f." ], - "errorType": "message" + "errorType": "message", + "url": "{urlMain}/members/?username={username}" } }, "phpBB": { @@ -18,13 +19,17 @@ }, "vBulletin": { "site": { + "errors": { + "The administrator has banned your IP address": "IP ban" + }, "errorMsg": [ "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043d \u0438 \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u043e\u0444\u0438\u043b\u044f \u0434\u043b\u044f \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u0430.", "Bu \u00dcye kay\u0131tl\u0131 \u00dcyemiz de\u011fildir. Bu sebebten dolay\u0131 \u00dcyeye ait Profil g\u00f6sterilemiyor.", "This user has not registered and therefore does not have a profile to view.", "\u041a\u043e\u0440\u0438\u0441\u0442\u0443\u0432\u0430\u0447 \u043d\u0435 \u0437\u0430\u0440\u0435\u0454\u0441\u0442\u0440\u043e\u0432\u0430\u043d\u0438\u0439 \u0456 \u043d\u0435 \u043c\u0430\u0454 \u043f\u0440\u043e\u0444\u0456\u043b\u044e, \u044f\u043a\u0438\u0439 \u043c\u043e\u0436\u043d\u0430 \u043f\u0435\u0440\u0435\u0433\u043b\u044f\u043d\u0443\u0442\u0438." ], - "errorType": "message" + "errorType": "message", + "url": "{urlMain}/{urlSubpath}/member.php?username={username}" } } }, @@ -33,11 +38,11 @@ "disabled": false, "errorMsg": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", "errorType": "message", - "rank": 4816157, + "rank": 4817765, "tags": [ "ru" ], - "url": "http://0-3.ru/members/?username={}", + "url": "http://0-3.ru/members/?username={username}", "urlMain": "http://0-3.ru", "username_claimed": "donna", "username_unclaimed": "noonewouldeverusethis7" @@ -51,7 +56,7 @@ "tags": [ "ru" ], - "url": "http://0k.clan.su/index/8-0-{}", + "url": "http://0k.clan.su/index/8-0-{username}", "urlMain": "http://0k.clan.su", "username_claimed": "eruzz", "username_unclaimed": "noonewouldeverusethis7" @@ -59,25 +64,24 @@ "1001mem.ru": { "errorMsg": "\u042d\u0442\u043e\u0442 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0441\u0443\u0449\u0435\u0441\u0442\u0432\u0443\u0435\u0442, \u0438\u043b\u0438 \u0437\u0430\u0431\u043b\u043e\u043a\u0438\u0440\u043e\u0432\u0430\u043d.", "errorType": "message", - "rank": 1642180, + "rank": 1644185, "tags": [ "ru" ], - "url": "http://1001mem.ru/{}", + "url": "http://1001mem.ru/{username}", "urlMain": "http://1001mem.ru", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "101xp.com": { - "disabled": false, "errorMsg": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", "errorType": "message", - "rank": 160443, + "rank": 161008, "tags": [ "gaming", "ru" ], - "url": "https://forum-ru.101xp.com/members/?username={}", + "url": "https://forum-ru.101xp.com/members/?username={username}", "urlMain": "https://forum-ru.101xp.com", "username_claimed": "aida", "username_unclaimed": "noonewouldeverusethis7" @@ -85,26 +89,25 @@ "11x2": { "disabled": false, "errorType": "status_code", - "rank": 2325531, + "rank": 2326605, "tags": [ "global" ], - "url": "https://11x2.com/user/home/{}", + "url": "https://11x2.com/user/home/{username}", "urlMain": "https://11x2.com", "username_claimed": "hazelamy", "username_unclaimed": "noonewouldeverusethis7" }, "123rf": { - "disabled": false, "errorType": "response_url", - "rank": 928, + "rank": 926, "tags": [ "images", "in", "ru", "us" ], - "url": "https://ru.123rf.com/profile_{}", + "url": "https://ru.123rf.com/profile_{username}", "urlMain": "https://ru.123rf.com", "username_claimed": "rawpixel", "username_unclaimed": "noonewouldeverusethis7" @@ -118,22 +121,21 @@ "in", "torrent" ], - "url": "https://1337x.to/user/{}/", + "url": "https://1337x.to/user/{username}/", "urlMain": "https://1337x.to", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "1x": { - "disabled": false, "errorMsg": "This user does not exist or is not approved yet. Come back later.", "errorType": "message", - "rank": 178750, + "rank": 176187, "tags": [ "ba", "in", "se" ], - "url": "https://1x.com/member/{}", + "url": "https://1x.com/member/{username}", "urlMain": "https://1x.com", "username_claimed": "blue", "username_unclaimed": "noonewouldeverusethis7" @@ -141,39 +143,37 @@ "1xforum": { "disabled": false, "engine": "vBulletin", - "rank": 2529907, + "rank": 2634700, "tags": [ "ru" ], - "url": "https://1xforum.com/member.php?username={}", "urlMain": "https://1xforum.com", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "247sports": { - "disabled": false, "errorType": "status_code", - "rank": 2590, + "rank": 2583, "request_head_only": false, "tags": [ "sport", "us" ], - "url": "https://247sports.com/user/{}/", + "url": "https://247sports.com/user/{username}/", "urlMain": "https://247sports.com", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "24open": { - "disabled": false, "errorType": "status_code", - "rank": 44693, + "rank": 44851, "request_head_only": false, "tags": [ "dating", - "ru" + "ru", + "us" ], - "url": "https://24open.ru/user/{}/", + "url": "https://24open.ru/user/{username}/", "urlMain": "https://24open.ru", "username_claimed": "niko3193", "username_unclaimed": "noonewouldeverusethis7" @@ -182,32 +182,30 @@ "disabled": false, "errorType": "status_code", "rank": 0, - "url": "https://2Dimensions.com/a/{}", + "url": "https://2Dimensions.com/a/{username}", "urlMain": "https://2Dimensions.com/", "username_claimed": "blue", "username_unclaimed": "noonewouldeverusethis7" }, "2berega.spb.ru": { - "disabled": false, "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d", "errorType": "message", - "rank": 670358, + "rank": 671040, "tags": [ "ru" ], - "url": "https://2berega.spb.ru/user/{}", + "url": "https://2berega.spb.ru/user/{username}", "urlMain": "https://2berega.spb.ru", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "2d-3d": { - "disabled": false, "errorType": "status_code", - "rank": 316974, + "rank": 318397, "tags": [ "ru" ], - "url": "https://www.2d-3d.ru/user/{}/", + "url": "https://www.2d-3d.ru/user/{username}/", "urlMain": "https://www.2d-3d.ru", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" @@ -216,11 +214,11 @@ "disabled": false, "errorMsg": "Deze gebruiker is niet geregistreerd, zodat je zijn of haar profiel niet kunt bekijken.", "errorType": "message", - "rank": 0, + "rank": 6296836, "tags": [ "nl" ], - "url": "https://www.2fast4u.be/members/?username={}", + "url": "https://www.2fast4u.be/members/?username={username}", "urlMain": "https://www.2fast4u.be", "username_claimed": "Schussboelie", "username_unclaimed": "noonewouldeverusethis7" @@ -232,12 +230,12 @@ "presenseStrs": [ "\u041f\u0440\u043e\u0444\u0438\u043b\u044c \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f" ], - "rank": 1413513, + "rank": 1414686, "regexCheck": "^[a-zA-Z0-9-]{3,}$", "tags": [ "ru" ], - "url": "http://{}.33bru.com/", + "url": "http://{username}.33bru.com/", "urlMain": "http://33bru.com/", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" @@ -246,165 +244,153 @@ "disabled": false, "errorMsg": "The specified member cannot be found", "errorType": "message", - "rank": 1349906, + "rank": 1351385, "tags": [ "global" ], - "url": "https://www.3dcadforums.com/members/?username={}", + "url": "https://www.3dcadforums.com/members/?username={username}", "urlMain": "https://www.3dcadforums.com/", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "3ddd": { - "disabled": false, "errorType": "status_code", - "rank": 12588, + "rank": 12616, "tags": [ "ru" ], - "url": "https://3ddd.ru/users/{}", + "url": "https://3ddd.ru/users/{username}", "urlMain": "https://3ddd.ru", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "3dnews": { - "disabled": false, "engine": "vBulletin", - "rank": 8696, + "rank": 8670, "tags": [ "ru" ], - "url": "http://forum.3dnews.ru/member.php?username={}", + "url": "http://forum.3dnews.ru/member.php?username={username}", "urlMain": "http://forum.3dnews.ru/", "username_claimed": "red", "username_unclaimed": "noonewouldeverusethis7" }, "3dtoday": { - "disabled": false, "errorType": "response_url", - "rank": 78672, + "rank": 78517, "tags": [ "ru" ], - "url": "https://3dtoday.ru/blogs/{}", + "url": "https://3dtoday.ru/blogs/{username}", "urlMain": "https://3dtoday.ru/", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "4cheat": { - "disabled": false, "engine": "vBulletin", - "rank": 246166, + "rank": 244909, "tags": [ "ru" ], - "url": "https://4cheat.ru/member.php?username={}", + "url": "https://4cheat.ru/member.php?username={username}", "urlMain": "https://4cheat.ru", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "4gameforum": { - "disabled": false, "errorMsg": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d. \u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430, \u0432\u0432\u0435\u0434\u0438\u0442\u0435 \u0434\u0440\u0443\u0433\u043e\u0435 \u0438\u043c\u044f.", "errorType": "message", - "rank": 89637, + "rank": 89429, "tags": [ "ru" ], - "url": "https://4gameforum.com/members/?username={}", + "url": "https://4gameforum.com/members/?username={username}", "urlMain": "https://4gameforum.com", "username_claimed": "persty", "username_unclaimed": "noonewouldeverusethis7" }, "4pda": { - "disabled": false, "errorMsg": "\u041a \u0441\u043e\u0436\u0430\u043b\u0435\u043d\u0438\u044e, \u0412\u0430\u0448 \u043f\u043e\u0438\u0441\u043a \u043d\u0435 \u0434\u0430\u043b \u043d\u0438\u043a\u0430\u043a\u0438\u0445 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u043e\u0432.", "errorType": "message", - "rank": 2907, + "rank": 2913, "tags": [ "ru" ], - "url": "https://4pda.ru/forum/index.php?act=search&source=pst&noform=1&username={}", + "url": "https://4pda.ru/forum/index.php?act=search&source=pst&noform=1&username={username}", "urlMain": "https://4pda.ru/", "username_claimed": "green", "username_unclaimed": "noonewouldeverusethis7" }, "4stor": { - "disabled": false, "errorType": "status_code", - "rank": 240122, + "rank": 245674, "tags": [ "ru" ], - "url": "https://4stor.ru/user/{}", + "url": "https://4stor.ru/user/{username}", "urlMain": "https://4stor.ru", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "500px": { - "disabled": false, "errorMsg": "No message available", "errorType": "message", "errors": { "INTERNAL_SERVER_ERROR": "Site error", "Something just went wrong": "Site error" }, - "rank": 2855, + "rank": 2854, "tags": [ "images", "in" ], - "url": "https://500px.com/p/{}", + "url": "https://500px.com/p/{username}", "urlMain": "https://500px.com/", - "urlProbe": "https://api.500px.com/graphql?operationName=ProfileRendererQuery&variables=%7B%22username%22%3A%22{}%22%7D&extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%225a17a9af1830b58b94a912995b7947b24f27f1301c6ea8ab71a9eb1a6a86585b%22%7D%7D", + "urlProbe": "https://api.500px.com/graphql?operationName=ProfileRendererQuery&variables=%7B%22username%22%3A%22{username}%22%7D&extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%225a17a9af1830b58b94a912995b7947b24f27f1301c6ea8ab71a9eb1a6a86585b%22%7D%7D", "username_claimed": "blue", "username_unclaimed": "noonewouldeverusethis7" }, "7Cups": { - "disabled": false, "errorType": "status_code", - "rank": 32195, + "rank": 32084, "tags": [ "in", "pl" ], - "url": "https://www.7cups.com/@{}", + "url": "https://www.7cups.com/@{username}", "urlMain": "https://www.7cups.com/", "username_claimed": "blue", "username_unclaimed": "noonewouldeverusethis7" }, "7dach": { - "disabled": false, "errorType": "status_code", - "rank": 11593, + "rank": 11689, "tags": [ "ru" ], - "url": "https://7dach.ru/profile/{}", + "url": "https://7dach.ru/profile/{username}", "urlMain": "https://7dach.ru/", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "7ya": { - "disabled": false, "errorType": "status_code", - "rank": 43460, + "rank": 43672, "tags": [ "ru" ], - "url": "https://blog.7ya.ru/{}/", + "url": "https://blog.7ya.ru/{username}/", "urlMain": "https://blog.7ya.ru", "username_claimed": "trotter", "username_unclaimed": "noonewouldeverusethis7" }, "9GAG": { - "disabled": false, "errorType": "status_code", - "rank": 405, + "rank": 404, "tags": [ "de" ], - "url": "https://www.9gag.com/u/{}", + "url": "https://www.9gag.com/u/{username}", "urlMain": "https://www.9gag.com/", "username_claimed": "blue", "username_unclaimed": "noonewouldeverusethis7" @@ -417,20 +403,19 @@ "tags": [ "ua" ], - "url": "https://aback.com.ua/user/{}", + "url": "https://aback.com.ua/user/{username}", "urlMain": "https://aback.com.ua", "username_claimed": "adam", "username_unclaimed": "noonewouldeverusethis7" }, "About.me": { - "disabled": false, "errorType": "status_code", - "rank": 12037, + "rank": 12085, "tags": [ "in", "social" ], - "url": "https://about.me/{}", + "url": "https://about.me/{username}", "urlMain": "https://about.me/", "username_claimed": "blue", "username_unclaimed": "noonewouldeverusethis7" @@ -439,94 +424,88 @@ "disabled": false, "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043d \u0438 \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u043e\u0444\u0438\u043b\u044f \u0434\u043b\u044f \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u0430.", "errorType": "message", - "rank": 4962162, + "rank": 5723107, "tags": [ "ru" ], - "url": "http://aboutcar.ru/members/{}.html", + "url": "http://aboutcar.ru/members/{username}.html", "urlMain": "http://aboutcar.ru", "username_claimed": "krolenya", "username_unclaimed": "noonewouldeverusethis7" }, "Academia.edu": { - "disabled": false, "errorType": "status_code", "rank": 254, "regexCheck": "^[^\\.]+$", "tags": [ "id" ], - "url": "https://independent.academia.edu/{}", + "url": "https://independent.academia.edu/{username}", "urlMain": "https://www.academia.edu/", "username_claimed": "blue", "username_unclaimed": "noonewouldeverusethis7" }, "Acomics": { - "disabled": false, "errorType": "status_code", - "rank": 127396, + "rank": 127801, "tags": [ "ru" ], - "url": "https://acomics.ru/-{}", + "url": "https://acomics.ru/-{username}", "urlMain": "https://acomics.ru", "username_claimed": "Garage", "username_unclaimed": "noonewouldeverusethis7" }, "AdultFriendFinder": { - "disabled": false, "errorMsg": "