Merge pull request #110 from soxoj/0.1.19

Bump to 0.1.19

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [0.1.19] - 2021-04-14
+* added `--no-progressbar` option
+* fixed ascii tree bug
+* fixed `python -m maigret` run
+* fixed requests freeze with timeout async tasks
+
 ## [0.1.18] - 2021-03-30
 * some API improvements
 

maigret/__main__.py

@@ -6,7 +6,7 @@ Maigret entrypoint
 
 import asyncio
 
-import maigret
+from .maigret import main
 
 if __name__ == "__main__":
-    asyncio.run(maigret.main())
+    asyncio.run(main())

maigret/checking.py

@@ -46,6 +46,14 @@ QueryDraft = Tuple[Callable, Any, Any]
 QueriesDraft = Iterable[QueryDraft]
 
 
+def create_task_func():
+    if sys.version_info.minor > 6:
+        create_asyncio_task = asyncio.create_task
+    else:
+        loop = asyncio.get_event_loop()
+        create_asyncio_task = loop.create_task
+    return create_asyncio_task
+
 class AsyncExecutor:
     def __init__(self, *args, **kwargs):
         self.logger = kwargs['logger']
@@ -109,29 +117,38 @@ class AsyncioProgressbarQueueExecutor(AsyncExecutor):
         self.workers_count = kwargs.get('in_parallel', 10)
         self.progress_func = kwargs.get('progress_func', tqdm.tqdm)
         self.queue = asyncio.Queue(self.workers_count)
+        self.timeout = kwargs.get('timeout')
 
     async def worker(self):
         while True:
-            f, args, kwargs = await self.queue.get()
-            result = await f(*args, **kwargs)
+            try:
+                f, args, kwargs = self.queue.get_nowait()
+            except asyncio.QueueEmpty:
+                return
+
+            query_future = f(*args, **kwargs)
+            query_task = create_task_func()(query_future)
+            try:
+                result = await asyncio.wait_for(query_task, timeout=self.timeout)
+            except asyncio.TimeoutError:
+                result = None
+
             self.results.append(result)
             self.progress.update(1)
             self.queue.task_done()
 
-    async def _run(self, tasks: QueriesDraft):
+    async def _run(self, queries: QueriesDraft):
         self.results = []
 
-        if sys.version_info.minor > 6:
-            create_task = asyncio.create_task
-        else:
-            loop = asyncio.get_event_loop()
-            create_task = loop.create_task
+        queries_list = list(queries)
 
-        workers = [create_task(self.worker())
-                   for _ in range(self.workers_count)]
-        task_list = list(tasks)
-        self.progress = self.progress_func(total=len(task_list))
-        for t in task_list:
+        min_workers = min(len(queries_list), self.workers_count)
+
+        workers = [create_task_func()(self.worker())
+                   for _ in range(min_workers)]
+
+        self.progress = self.progress_func(total=len(queries_list))
+        for t in queries_list:
             await self.queue.put(t)
         await self.queue.join()
         for w in workers:

maigret/maigret.py

@@ -21,7 +21,7 @@ from .sites import MaigretDatabase
 from .submit import submit_dialog
 from .utils import get_dict_ascii_tree
 
-__version__ = '0.1.18'
+__version__ = '0.1.19'
 
 
 async def main():
@@ -104,6 +104,10 @@ async def main():
                         action="store_true", dest="no_color", default=False,
                         help="Don't color terminal output"
                         )
+    parser.add_argument("--no-progressbar",
+                        action="store_true", dest="no_progressbar", default=False,
+                        help="Don't show progressbar."
+                        )
     parser.add_argument("--browse", "-b",
                         action="store_true", dest="browse", default=False,
                         help="Browse to all results on default bowser."
@@ -353,6 +357,7 @@ async def main():
                                 cookies=args.cookie_file,
                                 forced=args.use_disabled_sites,
                                 max_connections=args.connections,
+                                no_progressbar=args.no_progressbar,
                                 )
 
         general_results.append((username, id_type, results))

maigret/resources/data.json

@@ -94,20 +94,6 @@
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
-        "1x": {
-            "tags": [
-                "ba",
-                "in",
-                "se"
-            ],
-            "checkType": "message",
-            "absenceStrs": "This user does not exist or is not approved yet. Come back later.",
-            "alexaRank": 134199,
-            "url": "https://1x.com/member/{username}",
-            "urlMain": "https://1x.com",
-            "usernameClaimed": "blue",
-            "usernameUnclaimed": "noonewouldeverusethis7"
-        },
         "1xforum": {
             "tags": [
                 "ru"
@@ -2078,6 +2064,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Brute": {
+            "disabled": true,
             "tags": [
                 "ru"
             ],
@@ -2675,7 +2662,12 @@
                 "in"
             ],
             "checkType": "message",
-            "absenceStrs": "Adapted from",
+            "presenseStrs": [
+                "<div class=\"title\">"
+            ],
+            "absenceStrs": [
+                "Adapted from"
+            ],
             "alexaRank": 7219,
             "url": "https://www.codementor.io/{username}",
             "urlMain": "https://www.codementor.io/",
@@ -3022,6 +3014,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Damochka": {
+            "disabled": true,
             "tags": [
                 "ru"
             ],
@@ -7311,6 +7304,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Lingvolive": {
+            "disabled": true,
             "tags": [
                 "it",
                 "ru"
@@ -12117,7 +12111,7 @@
                 "us"
             ],
             "headers": {
-                "authorization": "Bearer BQBgSgdI2iyspCVcbJZjVlMt3-84iAe7qyMh_ZO095wRaD68tB-Zz-US5cXPD0DSVfluwjZWn1Bf2EVJIN8"
+                "authorization": "Bearer BQD2HAPvSlZkuoFflR2L6YOns9-DRL_YSCCpmyozsml_FZYf8sICoKQpz3JqJ2JHrlUMm9K56bgw2BPbyJg"
             },
             "errors": {
                 "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
@@ -13419,7 +13413,7 @@
                 "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
                 "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
                 "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-                "x-guest-token": "1376637415348113408"
+                "x-guest-token": "1381699376133857282"
             },
             "errors": {
                 "Bad guest token": "x-guest-token update required"
@@ -13796,7 +13790,7 @@
                 "video"
             ],
             "headers": {
-                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTcwNTE4NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.iZzO-_VDARa_honzp7KvRSaSK0qMQ7n8dp9k6gah_bE"
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTgyNTg2ODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.gRqKPO3uS5pqVzYh0lHOGKSUAwq5dfIu7q7yfxoIgEE"
             },
             "activation": {
                 "url": "https://vimeo.com/_rv/viewer",
@@ -15095,6 +15089,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "australianfrequentflyer.com.au": {
+            "disabled": true,
             "tags": [
                 "au"
             ],
@@ -15966,6 +15961,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "forums.battlefield.com": {
+            "disabled": true,
             "tags": [
                 "gaming",
                 "gb",
@@ -23484,6 +23480,27 @@
             "urlMain": "https://codeberg.org",
             "usernameClaimed": "pcastela",
             "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "1x": {
+            "tags": [
+                "photo"
+            ],
+            "checkType": "message",
+            "presenseStrs": [
+                " onload=",
+                "photos-feed",
+                "gallery-loadmore",
+                "lm_mode",
+                "create_exhibition_name"
+            ],
+            "absenceStrs": [
+                "1x.com \u2022 In Pursuit of the Sublime",
+                " >404</div>"
+            ],
+            "url": "https://1x.com/{username}",
+            "urlMain": "https://1x.com",
+            "usernameClaimed": "michaelafiresova",
+            "usernameUnclaimed": "noonewouldeverusethis7"
         }
     },
     "engines": {

maigret/utils.py

@@ -67,7 +67,7 @@ def get_dict_ascii_tree(items, prepend='', new_line=True):
             if field_value.startswith('[\''):
                 is_last_item = num == len(items) - 1
                 prepend_symbols = ' ' * 3 if is_last_item else ' ┃ '
-                field_value = print_ascii_tree(eval(field_value), prepend_symbols)
+                field_value = get_dict_ascii_tree(eval(field_value), prepend_symbols)
             text += f'\n{prepend}{box_symbol}{field_name}: {field_value}'
         else:
             text += f'\n{prepend}{box_symbol} {item}'

setup.py

@@ -12,7 +12,7 @@ with open('requirements.txt') as rf:
     requires = rf.read().splitlines()
 
 setup(name='maigret',
-      version='0.1.18',
+      version='0.1.19',
       description='Collect a dossier on a person by username from a huge number of sites',
       long_description=long_description,
       long_description_content_type="text/markdown",