Merge pull request #110 from soxoj/0.1.19

Bump to 0.1.19

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## [Unreleased]
 
+## [0.1.19] - 2021-04-14
+* added `--no-progressbar` option
+* fixed ascii tree bug
+* fixed `python -m maigret` run
+* fixed requests freeze with timeout async tasks
+
+## [0.1.18] - 2021-03-30
+* some API improvements
+
 ## [0.1.17] - 2021-03-30
 * simplified maigret search API
 * improved documentation

maigret.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python3
+#!/usr/bin/env python3
 import asyncio
 import sys
 

maigret/__init__.py

@@ -1,3 +1,5 @@
 """Maigret"""
 
 from .checking import maigret as search
+from .sites import MaigretEngine, MaigretSite, MaigretDatabase
+from .notify import QueryNotifyPrint as Notifier

maigret/__main__.py

@@ -6,7 +6,7 @@ Maigret entrypoint
 
 import asyncio
 
-import maigret
+from .maigret import main
 
 if __name__ == "__main__":
-    asyncio.run(maigret.main())
+    asyncio.run(main())

maigret/checking.py

@@ -46,6 +46,14 @@ QueryDraft = Tuple[Callable, Any, Any]
 QueriesDraft = Iterable[QueryDraft]
 
 
+def create_task_func():
+    if sys.version_info.minor > 6:
+        create_asyncio_task = asyncio.create_task
+    else:
+        loop = asyncio.get_event_loop()
+        create_asyncio_task = loop.create_task
+    return create_asyncio_task
+
 class AsyncExecutor:
     def __init__(self, *args, **kwargs):
         self.logger = kwargs['logger']
@@ -109,29 +117,38 @@ class AsyncioProgressbarQueueExecutor(AsyncExecutor):
         self.workers_count = kwargs.get('in_parallel', 10)
         self.progress_func = kwargs.get('progress_func', tqdm.tqdm)
         self.queue = asyncio.Queue(self.workers_count)
+        self.timeout = kwargs.get('timeout')
 
     async def worker(self):
         while True:
-            f, args, kwargs = await self.queue.get()
-            result = await f(*args, **kwargs)
+            try:
+                f, args, kwargs = self.queue.get_nowait()
+            except asyncio.QueueEmpty:
+                return
+
+            query_future = f(*args, **kwargs)
+            query_task = create_task_func()(query_future)
+            try:
+                result = await asyncio.wait_for(query_task, timeout=self.timeout)
+            except asyncio.TimeoutError:
+                result = None
+
             self.results.append(result)
             self.progress.update(1)
             self.queue.task_done()
 
-    async def _run(self, tasks: QueriesDraft):
+    async def _run(self, queries: QueriesDraft):
         self.results = []
 
-        if sys.version_info.minor > 6:
-            create_task = asyncio.create_task
-        else:
-            loop = asyncio.get_event_loop()
-            create_task = loop.create_task
+        queries_list = list(queries)
 
-        workers = [create_task(self.worker())
-                   for _ in range(self.workers_count)]
-        task_list = list(tasks)
-        self.progress = self.progress_func(total=len(task_list))
-        for t in task_list:
+        min_workers = min(len(queries_list), self.workers_count)
+
+        workers = [create_task_func()(self.worker())
+                   for _ in range(min_workers)]
+
+        self.progress = self.progress_func(total=len(queries_list))
+        for t in queries_list:
             await self.queue.put(t)
         await self.queue.join()
         for w in workers:

maigret/maigret.py

@@ -21,7 +21,7 @@ from .sites import MaigretDatabase
 from .submit import submit_dialog
 from .utils import get_dict_ascii_tree
 
-__version__ = '0.1.17'
+__version__ = '0.1.19'
 
 
 async def main():
@@ -104,6 +104,10 @@ async def main():
                         action="store_true", dest="no_color", default=False,
                         help="Don't color terminal output"
                         )
+    parser.add_argument("--no-progressbar",
+                        action="store_true", dest="no_progressbar", default=False,
+                        help="Don't show progressbar."
+                        )
     parser.add_argument("--browse", "-b",
                         action="store_true", dest="browse", default=False,
                         help="Browse to all results on default bowser."
@@ -353,6 +357,7 @@ async def main():
                                 cookies=args.cookie_file,
                                 forced=args.use_disabled_sites,
                                 max_connections=args.connections,
+                                no_progressbar=args.no_progressbar,
                                 )
 
         general_results.append((username, id_type, results))

maigret/resources/data.json

@@ -94,20 +94,6 @@
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
-        "1x": {
-            "tags": [
-                "ba",
-                "in",
-                "se"
-            ],
-            "checkType": "message",
-            "absenceStrs": "This user does not exist or is not approved yet. Come back later.",
-            "alexaRank": 134199,
-            "url": "https://1x.com/member/{username}",
-            "urlMain": "https://1x.com",
-            "usernameClaimed": "blue",
-            "usernameUnclaimed": "noonewouldeverusethis7"
-        },
         "1xforum": {
             "tags": [
                 "ru"
@@ -2078,6 +2064,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Brute": {
+            "disabled": true,
             "tags": [
                 "ru"
             ],
@@ -2675,7 +2662,12 @@
                 "in"
             ],
             "checkType": "message",
-            "absenceStrs": "Adapted from",
+            "presenseStrs": [
+                "<div class=\"title\">"
+            ],
+            "absenceStrs": [
+                "Adapted from"
+            ],
             "alexaRank": 7219,
             "url": "https://www.codementor.io/{username}",
             "urlMain": "https://www.codementor.io/",
@@ -3022,6 +3014,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Damochka": {
+            "disabled": true,
             "tags": [
                 "ru"
             ],
@@ -7311,6 +7304,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Lingvolive": {
+            "disabled": true,
             "tags": [
                 "it",
                 "ru"
@@ -12117,7 +12111,7 @@
                 "us"
             ],
             "headers": {
-                "authorization": "Bearer BQBgSgdI2iyspCVcbJZjVlMt3-84iAe7qyMh_ZO095wRaD68tB-Zz-US5cXPD0DSVfluwjZWn1Bf2EVJIN8"
+                "authorization": "Bearer BQD2HAPvSlZkuoFflR2L6YOns9-DRL_YSCCpmyozsml_FZYf8sICoKQpz3JqJ2JHrlUMm9K56bgw2BPbyJg"
             },
             "errors": {
                 "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
@@ -13419,7 +13413,7 @@
                 "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
                 "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
                 "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-                "x-guest-token": "1376637415348113408"
+                "x-guest-token": "1381699376133857282"
             },
             "errors": {
                 "Bad guest token": "x-guest-token update required"
@@ -13796,7 +13790,7 @@
                 "video"
             ],
             "headers": {
-                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTcwNTE4NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.iZzO-_VDARa_honzp7KvRSaSK0qMQ7n8dp9k6gah_bE"
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTgyNTg2ODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.gRqKPO3uS5pqVzYh0lHOGKSUAwq5dfIu7q7yfxoIgEE"
             },
             "activation": {
                 "url": "https://vimeo.com/_rv/viewer",
@@ -15095,6 +15089,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "australianfrequentflyer.com.au": {
+            "disabled": true,
             "tags": [
                 "au"
             ],
@@ -15966,6 +15961,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "forums.battlefield.com": {
+            "disabled": true,
             "tags": [
                 "gaming",
                 "gb",
@@ -23484,6 +23480,27 @@
             "urlMain": "https://codeberg.org",
             "usernameClaimed": "pcastela",
             "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "1x": {
+            "tags": [
+                "photo"
+            ],
+            "checkType": "message",
+            "presenseStrs": [
+                " onload=",
+                "photos-feed",
+                "gallery-loadmore",
+                "lm_mode",
+                "create_exhibition_name"
+            ],
+            "absenceStrs": [
+                "1x.com \u2022 In Pursuit of the Sublime",
+                " >404</div>"
+            ],
+            "url": "https://1x.com/{username}",
+            "urlMain": "https://1x.com",
+            "usernameClaimed": "michaelafiresova",
+            "usernameUnclaimed": "noonewouldeverusethis7"
         }
     },
     "engines": {

maigret/result.py

@@ -84,6 +84,9 @@ class QueryResult():
             'tags': self.tags,
         }
 
+    def is_found(self):
+        return self.status == QueryStatus.CLAIMED
+
     def __str__(self):
         """Convert Object To String.
 

maigret/utils.py

@@ -67,7 +67,7 @@ def get_dict_ascii_tree(items, prepend='', new_line=True):
             if field_value.startswith('[\''):
                 is_last_item = num == len(items) - 1
                 prepend_symbols = ' ' * 3 if is_last_item else ' ┃ '
-                field_value = print_ascii_tree(eval(field_value), prepend_symbols)
+                field_value = get_dict_ascii_tree(eval(field_value), prepend_symbols)
             text += f'\n{prepend}{box_symbol}{field_name}: {field_value}'
         else:
             text += f'\n{prepend}{box_symbol} {item}'

setup.py

@@ -12,7 +12,7 @@ with open('requirements.txt') as rf:
     requires = rf.read().splitlines()
 
 setup(name='maigret',
-      version='0.1.17',
+      version='0.1.19',
       description='Collect a dossier on a person by username from a huge number of sites',
       long_description=long_description,
       long_description_content_type="text/markdown",

wizard.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+import asyncio
+import logging
+import maigret
+
+
+# top popular sites from the Maigret database
+TOP_SITES_COUNT = 300
+# Maigret HTTP requests timeout
+TIMEOUT = 10
+# max parallel requests
+MAX_CONNECTIONS = 50
+
+
+if __name__ == '__main__':
+    # setup logging and asyncio
+    logger = logging.getLogger('maigret')
+    logger.setLevel(logging.WARNING)
+    loop = asyncio.get_event_loop()
+
+    # setup Maigret
+    db = maigret.MaigretDatabase().load_from_file('./maigret/resources/data.json')
+    # also can be downloaded from web
+    # db = MaigretDatabase().load_from_url(MAIGRET_DB_URL)
+
+    # user input
+    username = input('Enter username to search: ')
+
+    sites_count_raw = input(f'Select the number of sites to search ({TOP_SITES_COUNT} for default, {len(db.sites_dict)} max): ')
+    sites_count = int(sites_count_raw) or TOP_SITES_COUNT
+
+    sites = db.ranked_sites_dict(top=sites_count)
+
+    show_progressbar_raw = input('Do you want to show a progressbar? [Yn] ')
+    show_progressbar = show_progressbar_raw.lower() != 'n' 
+
+    extract_info_raw = input('Do you want to extract additional info from accounts\' pages? [Yn] ')
+    extract_info = extract_info_raw.lower() != 'n' 
+
+    use_notifier_raw = input('Do you want to use notifier for displaying results while searching? [Yn] ')
+    use_notifier = use_notifier_raw.lower() != 'n'
+
+    notifier = None
+    if use_notifier:
+        notifier = maigret.Notifier(print_found_only=True, skip_check_errors=True)
+
+    # search!
+    search_func = maigret.search(username=username,
+                                 site_dict=sites,
+                                 timeout=TIMEOUT,
+                                 logger=logger,
+                                 max_connections=MAX_CONNECTIONS,
+                                 query_notify=notifier,
+                                 no_progressbar=(not show_progressbar),
+                                 is_parsing_enabled=extract_info,
+                                 )
+
+    results = loop.run_until_complete(search_func)
+
+    input('Search completed. Press any key to show results.')
+
+    for sitename, data in results.items():
+        is_found = data['status'].is_found()
+        print(f'{sitename} - {"Found!" if is_found else "Not found"}')