Merge pull request #110 from soxoj/0.1.19

Bump to 0.1.19

.github/workflows/python-package.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.6.9, 3.7, 3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 ## [Unreleased]
 
+## [0.1.19] - 2021-04-14
+* added `--no-progressbar` option
+* fixed ascii tree bug
+* fixed `python -m maigret` run
+* fixed requests freeze with timeout async tasks
+
+## [0.1.18] - 2021-03-30
+* some API improvements
+
+## [0.1.17] - 2021-03-30
+* simplified maigret search API
+* improved documentation
+* fixed 403 response code ignoring bug
+
 ## [0.1.16] - 2021-03-21
 * improved URL parsing mode
 * improved sites submit mode

maigret.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python3
+#!/usr/bin/env python3
 import asyncio
 import sys
 

maigret/__init__.py

@@ -1 +1,5 @@
 """Maigret"""
+
+from .checking import maigret as search
+from .sites import MaigretEngine, MaigretSite, MaigretDatabase
+from .notify import QueryNotifyPrint as Notifier

maigret/__main__.py

@@ -6,7 +6,7 @@ Maigret entrypoint
 
 import asyncio
 
-import maigret
+from .maigret import main
 
 if __name__ == "__main__":
-    asyncio.run(maigret.main())
+    asyncio.run(main())

maigret/checking.py

@@ -1,5 +1,6 @@
 import asyncio
 import logging
+from mock import Mock
 import re
 import ssl
 import sys
@@ -45,6 +46,14 @@ QueryDraft = Tuple[Callable, Any, Any]
 QueriesDraft = Iterable[QueryDraft]
 
 
+def create_task_func():
+    if sys.version_info.minor > 6:
+        create_asyncio_task = asyncio.create_task
+    else:
+        loop = asyncio.get_event_loop()
+        create_asyncio_task = loop.create_task
+    return create_asyncio_task
+
 class AsyncExecutor:
     def __init__(self, *args, **kwargs):
         self.logger = kwargs['logger']
@@ -108,22 +117,38 @@ class AsyncioProgressbarQueueExecutor(AsyncExecutor):
         self.workers_count = kwargs.get('in_parallel', 10)
         self.progress_func = kwargs.get('progress_func', tqdm.tqdm)
         self.queue = asyncio.Queue(self.workers_count)
+        self.timeout = kwargs.get('timeout')
 
     async def worker(self):
         while True:
-            f, args, kwargs = await self.queue.get()
-            result = await f(*args, **kwargs)
+            try:
+                f, args, kwargs = self.queue.get_nowait()
+            except asyncio.QueueEmpty:
+                return
+
+            query_future = f(*args, **kwargs)
+            query_task = create_task_func()(query_future)
+            try:
+                result = await asyncio.wait_for(query_task, timeout=self.timeout)
+            except asyncio.TimeoutError:
+                result = None
+
             self.results.append(result)
             self.progress.update(1)
             self.queue.task_done()
 
-    async def _run(self, tasks: QueriesDraft):
+    async def _run(self, queries: QueriesDraft):
         self.results = []
-        workers = [asyncio.create_task(self.worker())
-                   for _ in range(self.workers_count)]
-        task_list = list(tasks)
-        self.progress = self.progress_func(total=len(task_list))
-        for t in task_list:
+
+        queries_list = list(queries)
+
+        min_workers = min(len(queries_list), self.workers_count)
+
+        workers = [create_task_func()(self.worker())
+                   for _ in range(min_workers)]
+
+        self.progress = self.progress_func(total=len(queries_list))
+        for t in queries_list:
             await self.queue.put(t)
         await self.queue.join()
         for w in workers:
@@ -259,7 +284,7 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig
 
     if status_code and not error_text:
         error_text, site_error_text = detect_error_page(html_text, status_code, failure_errors,
-                                                        site.ignore_403)
+                                                        site.ignore403)
 
     if site.activation and html_text:
         is_need_activation = any([s for s in site.activation['marks'] if s in html_text])
@@ -387,26 +412,32 @@ def process_site_result(response, query_notify, logger, results_info, site: Maig
     return results_info
 
 
-async def maigret(username, site_dict, query_notify, logger,
+async def maigret(username, site_dict, logger, query_notify=None,
                   proxy=None, timeout=None, is_parsing_enabled=False,
                   id_type='username', debug=False, forced=False,
                   max_connections=100, no_progressbar=False,
                   cookies=None):
     """Main search func
 
-    Checks for existence of username on various social media sites.
+    Checks for existence of username on certain sites.
 
     Keyword Arguments:
-    username               -- String indicating username that report
-                              should be created against.
-    site_dict              -- Dictionary containing all of the site data.
+    username               -- Username string will be used for search.
+    site_dict              -- Dictionary containing sites data.
     query_notify           -- Object with base type of QueryNotify().
                               This will be used to notify the caller about
                               query results.
-    proxy                  -- String indicating the proxy URL
+    logger                 -- Standard Python logger object.
     timeout                -- Time in seconds to wait before timing out request.
                               Default is no timeout.
-    is_parsing_enabled     -- Search for other usernames in website pages.
+    is_parsing_enabled     -- Extract additional info from account pages.
+    id_type                -- Type of username to search.
+                              Default is 'username', see all supported here:
+                              https://github.com/soxoj/maigret/wiki/Supported-identifier-types
+    max_connections        -- Maximum number of concurrent connections allowed.
+                              Default is 100.
+    no_progressbar         -- Displaying of ASCII progressbar during scanner.
+    cookies                -- Filename of a cookie jar file to use for each request.
 
     Return Value:
     Dictionary containing results from report. Key of dictionary is the name
@@ -423,6 +454,9 @@ async def maigret(username, site_dict, query_notify, logger,
     """
 
     # Notify caller that we are starting the query.
+    if not query_notify:
+        query_notify = Mock()
+
     query_notify.start(username, id_type)
 
     # TODO: connector
@@ -602,7 +636,6 @@ def timeout_check(value):
 
 
 async def site_self_check(site, logger, semaphore, db: MaigretDatabase, silent=False):
-    query_notify = Mock()
     changes = {
         'disabled': False,
     }
@@ -622,10 +655,9 @@ async def site_self_check(site, logger, semaphore, db: MaigretDatabase, silent=F
     for username, status in check_data:
         async with semaphore:
             results_dict = await maigret(
-                username,
-                {site.name: site},
-                query_notify,
-                logger,
+                username=username,
+                site_dict={site.name: site},
+                logger=logger,
                 timeout=30,
                 id_type=site.type,
                 forced=True,

maigret/maigret.py

@@ -21,7 +21,7 @@ from .sites import MaigretDatabase
 from .submit import submit_dialog
 from .utils import get_dict_ascii_tree
 
-__version__ = '0.1.16'
+__version__ = '0.1.19'
 
 
 async def main():
@@ -104,6 +104,10 @@ async def main():
                         action="store_true", dest="no_color", default=False,
                         help="Don't color terminal output"
                         )
+    parser.add_argument("--no-progressbar",
+                        action="store_true", dest="no_progressbar", default=False,
+                        help="Don't show progressbar."
+                        )
     parser.add_argument("--browse", "-b",
                         action="store_true", dest="browse", default=False,
                         help="Browse to all results on default bowser."
@@ -341,9 +345,9 @@ async def main():
 
         sites_to_check = get_top_sites_for_id(id_type)
 
-        results = await maigret(username,
-                                dict(sites_to_check),
-                                query_notify,
+        results = await maigret(username=username,
+                                site_dict=dict(sites_to_check),
+                                query_notify=query_notify,
                                 proxy=args.proxy,
                                 timeout=args.timeout,
                                 is_parsing_enabled=parsing_enabled,
@@ -353,6 +357,7 @@ async def main():
                                 cookies=args.cookie_file,
                                 forced=args.use_disabled_sites,
                                 max_connections=args.connections,
+                                no_progressbar=args.no_progressbar,
                                 )
 
         general_results.append((username, id_type, results))

maigret/resources/data.json

@@ -94,20 +94,6 @@
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
-        "1x": {
-            "tags": [
-                "ba",
-                "in",
-                "se"
-            ],
-            "checkType": "message",
-            "absenceStrs": "This user does not exist or is not approved yet. Come back later.",
-            "alexaRank": 134199,
-            "url": "https://1x.com/member/{username}",
-            "urlMain": "https://1x.com",
-            "usernameClaimed": "blue",
-            "usernameUnclaimed": "noonewouldeverusethis7"
-        },
         "1xforum": {
             "tags": [
                 "ru"
@@ -1413,22 +1399,22 @@
             "usernameUnclaimed": "noonewouldeverusethis"
         },
         "Avto-forum.name": {
+            "ignore403": true,
             "tags": [
                 "ru"
             ],
             "engine": "XenForo",
             "alexaRank": 716960,
-            "ignore403": true,
             "urlMain": "https://avto-forum.name",
             "usernameClaimed": "mariya",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Avtoforum": {
+            "ignore403": true,
             "tags": [
                 "ru"
             ],
             "engine": "XenForo",
-            "ignore403": true,
             "urlMain": "https://avtoforum.org",
             "usernameClaimed": "tim",
             "usernameUnclaimed": "noonewouldeverusethis7"
@@ -1566,25 +1552,13 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "XSS.is": {
+            "ignore403": true,
             "tags": [
-                "in",
+                "hacking",
                 "ru"
             ],
-            "activation": {
-                "method": "xssis",
-                "marks": [
-                    "errorHtml"
-                ],
-                "url": "https://xss.is/login/keep-alive",
-                "src": "csrf",
-                "dst": "x-guest-token"
-            },
-            "checkType": "status_code",
-            "getParams": {
-                "_xfToken": "1611179947,a2710362e45dad9aa1da381e21941a38"
-            },
+            "engine": "XenForo",
             "alexaRank": 165220,
-            "url": "https://xss.is/index.php?members/find&q={username}&_xfRequestUri=%2Fmembers%2F%3Fkey%3Dmost_messages&_xfWithData=1&_xfResponseType=json",
             "urlMain": "https://xss.is",
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
@@ -1654,6 +1628,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "BeerMoneyForum": {
+            "ignore403": true,
             "tags": [
                 "jp",
                 "ve"
@@ -1661,7 +1636,6 @@
             "checkType": "message",
             "absenceStrs": "The specified member cannot be found.",
             "alexaRank": 11581,
-            "ignore403": true,
             "url": "https://www.beermoneyforum.com/members/?username={username}",
             "urlMain": "https://www.beermoneyforum.com",
             "usernameClaimed": "Yugocean",
@@ -2031,6 +2005,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "BoomInfo": {
+            "ignore403": true,
             "tags": [
                 "ru",
                 "ua"
@@ -2038,7 +2013,6 @@
             "checkType": "message",
             "absenceStrs": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d. \u041f\u043e\u0436\u0430\u043b\u0443\u0439\u0441\u0442\u0430, \u0432\u0432\u0435\u0434\u0438\u0442\u0435 \u0434\u0440\u0443\u0433\u043e\u0435 \u0438\u043c\u044f.",
             "alexaRank": 1680672,
-            "ignore403": true,
             "url": "https://boominfo.ru/members/?username={username}",
             "urlMain": "https://boominfo.ru",
             "usernameClaimed": "boominfo",
@@ -2090,6 +2064,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Brute": {
+            "disabled": true,
             "tags": [
                 "ru"
             ],
@@ -2687,7 +2662,12 @@
                 "in"
             ],
             "checkType": "message",
-            "absenceStrs": "Adapted from",
+            "presenseStrs": [
+                "<div class=\"title\">"
+            ],
+            "absenceStrs": [
+                "Adapted from"
+            ],
             "alexaRank": 7219,
             "url": "https://www.codementor.io/{username}",
             "urlMain": "https://www.codementor.io/",
@@ -3034,6 +3014,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Damochka": {
+            "disabled": true,
             "tags": [
                 "ru"
             ],
@@ -3591,12 +3572,12 @@
             "usernameUnclaimed": "noonewouldeverusethis777"
         },
         "Dumpz": {
+            "ignore403": true,
             "tags": [
                 "ru"
             ],
             "engine": "XenForo",
             "alexaRank": 1291982,
-            "ignore403": true,
             "urlMain": "https://dumpz.ws",
             "usernameClaimed": "emailx45",
             "usernameUnclaimed": "noonewouldeverusethis7"
@@ -3848,13 +3829,13 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Erogen.club": {
+            "ignore403": true,
             "tags": [
                 "ru",
                 "ua"
             ],
             "engine": "XenForo",
             "alexaRank": 479929,
-            "ignore403": true,
             "urlMain": "https://erogen.club",
             "usernameClaimed": "yanok",
             "usernameUnclaimed": "noonewouldeverusethis7"
@@ -5814,18 +5795,19 @@
         },
         "Gunandgame": {
             "disabled": true,
+            "ignore403": true,
             "tags": [
                 "us"
             ],
             "checkType": "message",
             "absenceStrs": "The specified member cannot be found. Please enter a member's entire name.",
-            "ignore403": true,
             "url": "https://www.gunandgame.com/members/?username={username}",
             "urlMain": "https://www.gunandgame.co",
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Gunboards": {
+            "ignore403": true,
             "tags": [
                 "in",
                 "us"
@@ -5835,7 +5817,6 @@
             ],
             "engine": "XenForo",
             "alexaRank": 464194,
-            "ignore403": true,
             "urlMain": "https://forums.gunboards.com",
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
@@ -7323,6 +7304,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Lingvolive": {
+            "disabled": true,
             "tags": [
                 "it",
                 "ru"
@@ -8503,12 +8485,12 @@
         },
         "Musiker-board": {
             "disabled": true,
+            "ignore403": true,
             "tags": [
                 "de"
             ],
             "engine": "XenForo",
             "alexaRank": 151707,
-            "ignore403": true,
             "urlMain": "https://www.musiker-board.de",
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
@@ -9053,13 +9035,13 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Niketalk": {
+            "ignore403": true,
             "tags": [
                 "us"
             ],
             "checkType": "message",
             "absenceStrs": "The specified member cannot be found",
             "alexaRank": 165332,
-            "ignore403": true,
             "url": "https://niketalk.com/members/?username={username}",
             "urlMain": "https://niketalk.com",
             "usernameClaimed": "adam",
@@ -9636,6 +9618,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Pbnation": {
+            "ignore403": true,
             "tags": [
                 "ca",
                 "us"
@@ -9643,7 +9626,6 @@
             "checkType": "message",
             "absenceStrs": "This user has not registered",
             "alexaRank": 107502,
-            "ignore403": true,
             "url": "https://www.pbnation.com/member.php?username={username}",
             "urlMain": "https://www.pbnation.com/",
             "usernameClaimed": "adam",
@@ -10314,6 +10296,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Prosvetlenie": {
+            "ignore403": true,
             "tags": [
                 "kg",
                 "ru"
@@ -10321,7 +10304,6 @@
             "checkType": "message",
             "absenceStrs": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d",
             "alexaRank": 2256482,
-            "ignore403": true,
             "url": "http://www.prosvetlenie.org/forum/members/?username={username}",
             "urlMain": "http://www.prosvetlenie.org",
             "usernameClaimed": "odin",
@@ -10533,6 +10515,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "RPGGeek": {
+            "ignore403": true,
             "tags": [
                 "gaming",
                 "us"
@@ -10540,7 +10523,6 @@
             "checkType": "message",
             "absenceStrs": "User does not exist",
             "alexaRank": 177522,
-            "ignore403": true,
             "url": "https://rpggeek.com/user/{username}",
             "urlMain": "https://rpggeek.com",
             "usernameClaimed": "adam",
@@ -10548,13 +10530,13 @@
         },
         "RPGRussia": {
             "disabled": true,
+            "ignore403": true,
             "tags": [
                 "ru",
                 "us"
             ],
             "engine": "XenForo",
             "alexaRank": 256354,
-            "ignore403": true,
             "urlMain": "https://rpgrussia.com",
             "usernameClaimed": "adam",
             "usernameUnclaimed": "noonewouldeverusethis7"
@@ -11240,13 +11222,13 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Rusfishing": {
+            "ignore403": true,
             "tags": [
                 "ru"
             ],
             "checkType": "message",
             "absenceStrs": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d",
             "alexaRank": 77147,
-            "ignore403": true,
             "url": "https://www.rusfishing.ru/forum/members/?username={username}",
             "urlMain": "https://www.rusfishing.ru",
             "usernameClaimed": "ale8443",
@@ -11545,12 +11527,12 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Sexforum.ws": {
+            "ignore403": true,
             "tags": [
                 "ru"
             ],
             "engine": "XenForo",
             "alexaRank": 1815966,
-            "ignore403": true,
             "urlMain": "http://sexforum.ws",
             "usernameClaimed": "katrin1988",
             "usernameUnclaimed": "noonewouldeverusethis7"
@@ -12129,7 +12111,7 @@
                 "us"
             ],
             "headers": {
-                "authorization": "Bearer BQCyNy7oN-nOZ-rNK_jL0R4GHS7f3uWdK8AdxZ9m-pg9dbm7t-mLNaaHCG2BpDeV4AekGfl4AscbXLMqeB4"
+                "authorization": "Bearer BQD2HAPvSlZkuoFflR2L6YOns9-DRL_YSCCpmyozsml_FZYf8sICoKQpz3JqJ2JHrlUMm9K56bgw2BPbyJg"
             },
             "errors": {
                 "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
@@ -13095,13 +13077,13 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "TotalStavki": {
+            "ignore403": true,
             "tags": [
                 "ru"
             ],
             "checkType": "message",
             "absenceStrs": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d",
             "alexaRank": 5459551,
-            "ignore403": true,
             "url": "https://totalstavki.ru/forum/members/?username={username}",
             "urlMain": "https://totalstavki.ru",
             "usernameClaimed": "turbo",
@@ -13431,7 +13413,7 @@
                 "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
                 "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
                 "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-                "x-guest-token": "1373570939997335554"
+                "x-guest-token": "1381699376133857282"
             },
             "errors": {
                 "Bad guest token": "x-guest-token update required"
@@ -13778,6 +13760,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "VideogameGeek": {
+            "ignore403": true,
             "tags": [
                 "gaming",
                 "us"
@@ -13785,7 +13768,6 @@
             "checkType": "message",
             "absenceStrs": "User does not exist",
             "alexaRank": 719092,
-            "ignore403": true,
             "url": "https://videogamegeek.com/user/{username}",
             "urlMain": "https://videogamegeek.com",
             "usernameClaimed": "adam",
@@ -13808,7 +13790,7 @@
                 "video"
             ],
             "headers": {
-                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTYzMjE3MDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.yS9-vO94E_Ad_e2krnmOoqXWcaLxhZSrRvjUVHkgIPk"
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MTgyNTg2ODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.gRqKPO3uS5pqVzYh0lHOGKSUAwq5dfIu7q7yfxoIgEE"
             },
             "activation": {
                 "url": "https://vimeo.com/_rv/viewer",
@@ -13893,13 +13875,13 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Vlmi": {
+            "ignore403": true,
             "tags": [
                 "ru",
                 "ua"
             ],
             "engine": "XenForo",
             "alexaRank": 725829,
-            "ignore403": true,
             "urlMain": "https://vlmi.biz",
             "usernameClaimed": "mixa",
             "usernameUnclaimed": "noonewouldeverusethis7"
@@ -14516,6 +14498,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Wuz": {
+            "ignore403": true,
             "tags": [
                 "by",
                 "ru"
@@ -14523,7 +14506,6 @@
             "checkType": "message",
             "absenceStrs": "\u0423\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d",
             "alexaRank": 1927898,
-            "ignore403": true,
             "url": "http://wuz.by/forum/members/?username={username}",
             "urlMain": "http://wuz.by",
             "usernameClaimed": "adam",
@@ -15107,6 +15089,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "australianfrequentflyer.com.au": {
+            "disabled": true,
             "tags": [
                 "au"
             ],
@@ -15978,6 +15961,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "forums.battlefield.com": {
+            "disabled": true,
             "tags": [
                 "gaming",
                 "gb",
@@ -16069,6 +16053,7 @@
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "forums.overclockers.co.uk": {
+            "ignore403": true,
             "tags": [
                 "gb",
                 "uk"
@@ -16076,7 +16061,6 @@
             "checkType": "message",
             "absenceStrs": "The specified member cannot be found. Please enter a member's entire name.",
             "alexaRank": 10013,
-            "ignore403": true,
             "url": "https://forums.overclockers.co.uk/members/?username={username}",
             "urlMain": "https://forums.overclockers.co.uk",
             "usernameClaimed": "adam",
@@ -23481,6 +23465,42 @@
             "urlMain": "https://skyblock.net",
             "usernameClaimed": "alex",
             "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "codeberg.org": {
+            "checkType": "message",
+            "presenseStrs": [
+                "user profile",
+                " username text center"
+            ],
+            "absenceStrs": [
+                "og:description",
+                " ui centered image"
+            ],
+            "url": "https://codeberg.org/{username}",
+            "urlMain": "https://codeberg.org",
+            "usernameClaimed": "pcastela",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "1x": {
+            "tags": [
+                "photo"
+            ],
+            "checkType": "message",
+            "presenseStrs": [
+                " onload=",
+                "photos-feed",
+                "gallery-loadmore",
+                "lm_mode",
+                "create_exhibition_name"
+            ],
+            "absenceStrs": [
+                "1x.com \u2022 In Pursuit of the Sublime",
+                " >404</div>"
+            ],
+            "url": "https://1x.com/{username}",
+            "urlMain": "https://1x.com",
+            "usernameClaimed": "michaelafiresova",
+            "usernameUnclaimed": "noonewouldeverusethis7"
         }
     },
     "engines": {

maigret/result.py

@@ -84,6 +84,9 @@ class QueryResult():
             'tags': self.tags,
         }
 
+    def is_found(self):
+        return self.status == QueryStatus.CLAIMED
+
     def __str__(self):
         """Convert Object To String.
 

maigret/sites.py

@@ -46,7 +46,7 @@ class MaigretSite:
 
         self.disabled = False
         self.similar_search = False
-        self.ignore_403 = False
+        self.ignore403 = False
         self.tags = []
 
         self.type = 'username'

maigret/submit.py

@@ -27,7 +27,6 @@ def extract_mainpage_url(url):
 
 
 async def site_self_check(site, logger, semaphore, db: MaigretDatabase, silent=False):
-    query_notify = Mock()
     changes = {
         'disabled': False,
     }
@@ -41,10 +40,9 @@ async def site_self_check(site, logger, semaphore, db: MaigretDatabase, silent=F
 
     for username, status in check_data:
         results_dict = await maigret(
-            username,
-            {site.name: site},
-            query_notify,
-            logger,
+            username=username,
+            site_dict={site.name: site},
+            logger=logger,
             timeout=30,
             id_type=site.type,
             forced=True,

maigret/utils.py

@@ -67,7 +67,7 @@ def get_dict_ascii_tree(items, prepend='', new_line=True):
             if field_value.startswith('[\''):
                 is_last_item = num == len(items) - 1
                 prepend_symbols = ' ' * 3 if is_last_item else ' ┃ '
-                field_value = print_ascii_tree(eval(field_value), prepend_symbols)
+                field_value = get_dict_ascii_tree(eval(field_value), prepend_symbols)
             text += f'\n{prepend}{box_symbol}{field_name}: {field_value}'
         else:
             text += f'\n{prepend}{box_symbol} {item}'

requirements.txt

@@ -14,21 +14,19 @@ future-annotations==1.0.0
 html5lib==1.1
 idna==2.10
 Jinja2==2.11.3
-lxml==4.6.2
+lxml==4.6.3
 MarkupSafe==1.1.1
 mock==4.0.2
 multidict==5.1.0
-Pillow==8.1.1
 pycountry==20.7.3
 PyPDF2==1.26.0
 PySocks==1.7.1
 python-bidi==0.4.2
 python-socks==1.1.2
-reportlab==3.5.59
 requests>=2.24.0
 requests-futures==1.0.0
 six==1.15.0
-socid-extractor>=0.0.15
+socid-extractor>=0.0.16
 soupsieve==2.1
 stem==1.8.0
 torrequest==0.1.0

setup.py

@@ -12,7 +12,7 @@ with open('requirements.txt') as rf:
     requires = rf.read().splitlines()
 
 setup(name='maigret',
-      version='0.1.15',
+      version='0.1.19',
       description='Collect a dossier on a person by username from a huge number of sites',
       long_description=long_description,
       long_description_content_type="text/markdown",

tests/test_utils.py

@@ -26,6 +26,13 @@ def test_case_convert_snake_to_title():
     assert b == 'Camel cased string'
 
 
+def test_case_convert_camel_with_digits_to_snake():
+    a = 'ignore403'
+    b = CaseConverter.camel_to_snake(a)
+
+    assert b == 'ignore403'
+
+
 def test_is_country_tag():
     assert is_country_tag('ru') == True
     assert is_country_tag('FR') == True

wizard.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+import asyncio
+import logging
+import maigret
+
+
+# top popular sites from the Maigret database
+TOP_SITES_COUNT = 300
+# Maigret HTTP requests timeout
+TIMEOUT = 10
+# max parallel requests
+MAX_CONNECTIONS = 50
+
+
+if __name__ == '__main__':
+    # setup logging and asyncio
+    logger = logging.getLogger('maigret')
+    logger.setLevel(logging.WARNING)
+    loop = asyncio.get_event_loop()
+
+    # setup Maigret
+    db = maigret.MaigretDatabase().load_from_file('./maigret/resources/data.json')
+    # also can be downloaded from web
+    # db = MaigretDatabase().load_from_url(MAIGRET_DB_URL)
+
+    # user input
+    username = input('Enter username to search: ')
+
+    sites_count_raw = input(f'Select the number of sites to search ({TOP_SITES_COUNT} for default, {len(db.sites_dict)} max): ')
+    sites_count = int(sites_count_raw) or TOP_SITES_COUNT
+
+    sites = db.ranked_sites_dict(top=sites_count)
+
+    show_progressbar_raw = input('Do you want to show a progressbar? [Yn] ')
+    show_progressbar = show_progressbar_raw.lower() != 'n' 
+
+    extract_info_raw = input('Do you want to extract additional info from accounts\' pages? [Yn] ')
+    extract_info = extract_info_raw.lower() != 'n' 
+
+    use_notifier_raw = input('Do you want to use notifier for displaying results while searching? [Yn] ')
+    use_notifier = use_notifier_raw.lower() != 'n'
+
+    notifier = None
+    if use_notifier:
+        notifier = maigret.Notifier(print_found_only=True, skip_check_errors=True)
+
+    # search!
+    search_func = maigret.search(username=username,
+                                 site_dict=sites,
+                                 timeout=TIMEOUT,
+                                 logger=logger,
+                                 max_connections=MAX_CONNECTIONS,
+                                 query_notify=notifier,
+                                 no_progressbar=(not show_progressbar),
+                                 is_parsing_enabled=extract_info,
+                                 )
+
+    results = loop.run_until_complete(search_func)
+
+    input('Search completed. Press any key to show results.')
+
+    for sitename, data in results.items():
+        is_found = data['status'].is_found()
+        print(f'{sitename} - {"Found!" if is_found else "Not found"}')