Merge pull request #165 from soxoj/i2p-support

Added I2P sites support

maigret/checking.py

@@ -127,7 +127,7 @@ class SimpleAiohttpChecker(CheckerBase):
         return str(html_text), status_code, error
 
 
-class TorAiohttpChecker(SimpleAiohttpChecker):
+class ProxiedAiohttpChecker(SimpleAiohttpChecker):
     def __init__(self, *args, **kwargs):
         proxy = kwargs.get('proxy')
         cookie_jar = kwargs.get('cookie_jar')
@@ -556,6 +556,7 @@ async def maigret(
     query_notify=None,
     proxy=None,
     tor_proxy=None,
+    i2p_proxy=None,
     timeout=3,
     is_parsing_enabled=False,
     id_type="username",
@@ -621,10 +622,17 @@ async def maigret(
     # TODO
     tor_checker = CheckerMock()
     if tor_proxy:
-        tor_checker = TorAiohttpChecker(  # type: ignore
+        tor_checker = ProxiedAiohttpChecker(  # type: ignore
             proxy=tor_proxy, cookie_jar=cookie_jar, logger=logger
         )
 
+    # TODO
+    i2p_checker = CheckerMock()
+    if i2p_proxy:
+        i2p_checker = ProxiedAiohttpChecker(  # type: ignore
+            proxy=i2p_proxy, cookie_jar=cookie_jar, logger=logger
+        )
+
     # TODO
     dns_checker = CheckerMock()
     if check_domains:
@@ -649,6 +657,7 @@ async def maigret(
         '': clearweb_checker,
         'tor': tor_checker,
         'dns': dns_checker,
+        'i2p': i2p_checker,
     }
     options["parsing"] = is_parsing_enabled
     options["timeout"] = timeout
@@ -705,6 +714,8 @@ async def maigret(
     await clearweb_checker.close()
     if tor_proxy:
         await tor_checker.close()
+    if i2p_proxy:
+        await i2p_checker.close()
 
     # notify caller that all queries are finished
     query_notify.finish()
@@ -744,6 +755,7 @@ async def site_self_check(
     db: MaigretDatabase,
     silent=False,
     tor_proxy=None,
+    i2p_proxy=None,
 ):
     changes = {
         "disabled": False,
@@ -768,6 +780,7 @@ async def site_self_check(
                 no_progressbar=True,
                 retries=1,
                 tor_proxy=tor_proxy,
+                i2p_proxy=i2p_proxy,
             )
 
             # don't disable entries with other ids types
@@ -823,6 +836,7 @@ async def self_check(
     silent=False,
     max_connections=10,
     tor_proxy=None,
+    i2p_proxy=None,
 ) -> bool:
     sem = asyncio.Semaphore(max_connections)
     tasks = []
@@ -834,7 +848,9 @@ async def self_check(
     disabled_old_count = disabled_count(all_sites.values())
 
     for _, site in all_sites.items():
-        check_coro = site_self_check(site, logger, sem, db, silent, tor_proxy)
+        check_coro = site_self_check(
+            site, logger, sem, db, silent, tor_proxy, i2p_proxy
+        )
         future = asyncio.ensure_future(check_coro)
         tasks.append(future)
 

maigret/maigret.py

@@ -246,6 +246,13 @@ def setup_arguments_parser():
         default='socks5://127.0.0.1:9050',
         help="Specify URL of your Tor gateway. Default is socks5://127.0.0.1:9050",
     )
+    parser.add_argument(
+        "--i2p-proxy",
+        metavar='I2P_PROXY_URL',
+        action="store",
+        default='http://127.0.0.1:4444',
+        help="Specify URL of your I2P gateway. Default is http://127.0.0.1:4444",
+    )
     parser.add_argument(
         "--with-domains",
         action="store_true",
@@ -534,6 +541,7 @@ async def main():
             logger,
             max_connections=args.connections,
             tor_proxy=args.tor_proxy,
+            i2p_proxy=args.i2p_proxy,
         )
         if is_need_update:
             if input('Do you want to save changes permanently? [Yn]\n').lower() in (
@@ -610,6 +618,7 @@ async def main():
             query_notify=query_notify,
             proxy=args.proxy,
             tor_proxy=args.tor_proxy,
+            i2p_proxy=args.i2p_proxy,
             timeout=args.timeout,
             is_parsing_enabled=parsing_enabled,
             id_type=id_type,

maigret/resources/data.json

@@ -13024,7 +13024,7 @@
                 "us"
             ],
             "headers": {
-                "authorization": "Bearer BQDEpoSTjg2Ko86QUHZjJmZvp5AuI1ru6rJySe8_cD0bRqMZk6PfmdsmJBu3QeiNHgUPGQPDz2VeSvRr16w"
+                "authorization": "Bearer BQCypIuUtz7zDFov8xN86mj1BelLf7Apf9WBaC5yYfNkmGe4r7Hz4Awp6dqPuCAP9K9F5yYtjbyZX_vlr4I"
             },
             "errors": {
                 "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
@@ -14857,7 +14857,7 @@
                 "video"
             ],
             "headers": {
-                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjI2NjM1MjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.bKcisdrE5nJZMvrbagUC8lZQOs9spg3IKMlK15IclM4"
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MjI2NjcxMjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.V4VVbLzNwPU21rNP5moSxrPcPw--C7_Qz9VHgcJc1CA"
             },
             "activation": {
                 "url": "https://vimeo.com/_rv/viewer",
@@ -28202,6 +28202,16 @@
                 "tor"
             ]
         },
+        "i2pforum": {
+            "protocol": "i2p",
+            "urlMain": "http://i2pforum.i2p",
+            "usernameClaimed": "zzz",
+            "usernameUnclaimed": "noonewouldeverusethis7",
+            "engine": "phpBB/Search",
+            "tags": [
+                "i2p"
+            ]
+        },
         "forum.freeton.org": {
             "urlMain": "https://forum.freeton.org",
             "engine": "Discourse",

maigret/sites.py

@@ -66,6 +66,7 @@ SUPPORTED_TAGS = [
     "bookmarks",
     "design",
     "tor",
+    "i2p",
 ]
 
 
@@ -310,7 +311,12 @@ class MaigretDatabase:
         )
         is_id_type_ok = lambda x: x.type == id_type
 
-        filter_tags_engines_fun = lambda x: not tags or is_engine_ok(x) or is_tags_ok(x) or is_protocol_in_tags(x)
+        filter_tags_engines_fun = (
+            lambda x: not tags
+            or is_engine_ok(x)
+            or is_tags_ok(x)
+            or is_protocol_in_tags(x)
+        )
         filter_names_fun = lambda x: not names or is_name_ok(x) or is_source_ok(x)
 
         filter_fun = (

tests/test_cli.py

@@ -33,6 +33,7 @@ DEFAULT_ARGS: Dict[str, Any] = {
     'tags': '',
     'timeout': 30,
     'tor_proxy': 'socks5://127.0.0.1:9050',
+    'i2p_proxy': 'http://127.0.0.1:4444',
     'top_sites': 500,
     'txt': False,
     'use_disabled_sites': False,