From 02cf330e37719deac33bfe5682730affdae6990a Mon Sep 17 00:00:00 2001 From: Soxoj Date: Wed, 2 Jun 2021 23:45:11 +0300 Subject: [PATCH] Added I2P sites support --- maigret/checking.py | 22 +++++++++++++++++++--- maigret/maigret.py | 9 +++++++++ maigret/resources/data.json | 14 ++++++++++++-- maigret/sites.py | 8 +++++++- tests/test_cli.py | 1 + 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/maigret/checking.py b/maigret/checking.py index bfabf6c..a612a9d 100644 --- a/maigret/checking.py +++ b/maigret/checking.py @@ -127,7 +127,7 @@ class SimpleAiohttpChecker(CheckerBase): return str(html_text), status_code, error -class TorAiohttpChecker(SimpleAiohttpChecker): +class ProxiedAiohttpChecker(SimpleAiohttpChecker): def __init__(self, *args, **kwargs): proxy = kwargs.get('proxy') cookie_jar = kwargs.get('cookie_jar') @@ -556,6 +556,7 @@ async def maigret( query_notify=None, proxy=None, tor_proxy=None, + i2p_proxy=None, timeout=3, is_parsing_enabled=False, id_type="username", @@ -621,10 +622,17 @@ async def maigret( # TODO tor_checker = CheckerMock() if tor_proxy: - tor_checker = TorAiohttpChecker( # type: ignore + tor_checker = ProxiedAiohttpChecker( # type: ignore proxy=tor_proxy, cookie_jar=cookie_jar, logger=logger ) + # TODO + i2p_checker = CheckerMock() + if i2p_proxy: + i2p_checker = ProxiedAiohttpChecker( # type: ignore + proxy=i2p_proxy, cookie_jar=cookie_jar, logger=logger + ) + # TODO dns_checker = CheckerMock() if check_domains: @@ -649,6 +657,7 @@ async def maigret( '': clearweb_checker, 'tor': tor_checker, 'dns': dns_checker, + 'i2p': i2p_checker, } options["parsing"] = is_parsing_enabled options["timeout"] = timeout @@ -705,6 +714,8 @@ async def maigret( await clearweb_checker.close() if tor_proxy: await tor_checker.close() + if i2p_proxy: + await i2p_checker.close() # notify caller that all queries are finished query_notify.finish() @@ -744,6 +755,7 @@ async def site_self_check( db: MaigretDatabase, silent=False, tor_proxy=None, + i2p_proxy=None, ): changes = { "disabled": False, @@ -768,6 +780,7 @@ async def site_self_check( no_progressbar=True, retries=1, tor_proxy=tor_proxy, + i2p_proxy=i2p_proxy, ) # don't disable entries with other ids types @@ -823,6 +836,7 @@ async def self_check( silent=False, max_connections=10, tor_proxy=None, + i2p_proxy=None, ) -> bool: sem = asyncio.Semaphore(max_connections) tasks = [] @@ -834,7 +848,9 @@ async def self_check( disabled_old_count = disabled_count(all_sites.values()) for _, site in all_sites.items(): - check_coro = site_self_check(site, logger, sem, db, silent, tor_proxy) + check_coro = site_self_check( + site, logger, sem, db, silent, tor_proxy, i2p_proxy + ) future = asyncio.ensure_future(check_coro) tasks.append(future) diff --git a/maigret/maigret.py b/maigret/maigret.py index 9334b90..6e7e1ae 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -246,6 +246,13 @@ def setup_arguments_parser(): default='socks5://127.0.0.1:9050', help="Specify URL of your Tor gateway. Default is socks5://127.0.0.1:9050", ) + parser.add_argument( + "--i2p-proxy", + metavar='I2P_PROXY_URL', + action="store", + default='http://127.0.0.1:4444', + help="Specify URL of your I2P gateway. Default is http://127.0.0.1:4444", + ) parser.add_argument( "--with-domains", action="store_true", @@ -534,6 +541,7 @@ async def main(): logger, max_connections=args.connections, tor_proxy=args.tor_proxy, + i2p_proxy=args.i2p_proxy, ) if is_need_update: if input('Do you want to save changes permanently? [Yn]\n').lower() in ( @@ -610,6 +618,7 @@ async def main(): query_notify=query_notify, proxy=args.proxy, tor_proxy=args.tor_proxy, + i2p_proxy=args.i2p_proxy, timeout=args.timeout, is_parsing_enabled=parsing_enabled, id_type=id_type, diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 550da92..35e0d82 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -13024,7 +13024,7 @@ "us" ], "headers": { - "authorization": "Bearer BQDEpoSTjg2Ko86QUHZjJmZvp5AuI1ru6rJySe8_cD0bRqMZk6PfmdsmJBu3QeiNHgUPGQPDz2VeSvRr16w" + "authorization": "Bearer BQCypIuUtz7zDFov8xN86mj1BelLf7Apf9WBaC5yYfNkmGe4r7Hz4Awp6dqPuCAP9K9F5yYtjbyZX_vlr4I" }, "errors": { "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn" @@ -14857,7 +14857,7 @@ "video" ], "headers": { - "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjI2NjM1MjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.bKcisdrE5nJZMvrbagUC8lZQOs9spg3IKMlK15IclM4" + "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjI2NjcxMjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.V4VVbLzNwPU21rNP5moSxrPcPw--C7_Qz9VHgcJc1CA" }, "activation": { "url": "https://vimeo.com/_rv/viewer", @@ -28202,6 +28202,16 @@ "tor" ] }, + "i2pforum": { + "protocol": "i2p", + "urlMain": "http://i2pforum.i2p", + "usernameClaimed": "zzz", + "usernameUnclaimed": "noonewouldeverusethis7", + "engine": "phpBB/Search", + "tags": [ + "i2p" + ] + }, "forum.freeton.org": { "urlMain": "https://forum.freeton.org", "engine": "Discourse", diff --git a/maigret/sites.py b/maigret/sites.py index 008842f..b612531 100644 --- a/maigret/sites.py +++ b/maigret/sites.py @@ -66,6 +66,7 @@ SUPPORTED_TAGS = [ "bookmarks", "design", "tor", + "i2p", ] @@ -310,7 +311,12 @@ class MaigretDatabase: ) is_id_type_ok = lambda x: x.type == id_type - filter_tags_engines_fun = lambda x: not tags or is_engine_ok(x) or is_tags_ok(x) or is_protocol_in_tags(x) + filter_tags_engines_fun = ( + lambda x: not tags + or is_engine_ok(x) + or is_tags_ok(x) + or is_protocol_in_tags(x) + ) filter_names_fun = lambda x: not names or is_name_ok(x) or is_source_ok(x) filter_fun = ( diff --git a/tests/test_cli.py b/tests/test_cli.py index 302c3ae..eee3741 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -33,6 +33,7 @@ DEFAULT_ARGS: Dict[str, Any] = { 'tags': '', 'timeout': 30, 'tor_proxy': 'socks5://127.0.0.1:9050', + 'i2p_proxy': 'http://127.0.0.1:4444', 'top_sites': 500, 'txt': False, 'use_disabled_sites': False,