Box drawing logic improved, added new settings (#217)

maigret/checking.py

@@ -28,6 +28,7 @@ from .executors import (
     AsyncioSimpleExecutor,
     AsyncioProgressbarQueueExecutor,
 )
+
 from .result import QueryResult, QueryStatus
 from .sites import MaigretDatabase, MaigretSite
 from .types import QueryOptions, QueryResultWrapper

maigret/maigret.py

@@ -252,7 +252,7 @@ def setup_arguments_parser(settings: Settings):
     parser.add_argument(
         "--with-domains",
         action="store_true",
-        default=False,
+        default=settings.domain_search,
         help="Enable (experimental) feature of checking domains on usernames.",
     )
 
@@ -264,13 +264,13 @@ def setup_arguments_parser(settings: Settings):
         "--all-sites",
         action="store_true",
         dest="all_sites",
-        default=False,
+        default=settings.scan_all_sites,
         help="Use all sites for scan.",
     )
     filter_group.add_argument(
         "--top-sites",
         action="store",
-        default=500,
+        default=settings.top_sites_count,
         metavar="N",
         type=int,
         help="Count of sites for scan ranked by Alexa Top (default: 500).",
@@ -283,13 +283,13 @@ def setup_arguments_parser(settings: Settings):
         action="append",
         metavar='SITE_NAME',
         dest="site_list",
-        default=[],
+        default=settings.scan_sites_list,
         help="Limit analysis to just the specified sites (multiple option).",
     )
     filter_group.add_argument(
         "--use-disabled-sites",
         action="store_true",
-        default=False,
+        default=settings.scan_disabled_sites,
         help="Use disabled sites to search (may cause many false positives).",
     )
 
@@ -316,7 +316,7 @@ def setup_arguments_parser(settings: Settings):
     modes_group.add_argument(
         "--self-check",
         action="store_true",
-        default=False,
+        default=settings.self_check_enabled,
         help="Do self check for sites and database and disable non-working ones.",
     )
     modes_group.add_argument(
@@ -333,14 +333,14 @@ def setup_arguments_parser(settings: Settings):
         "--print-not-found",
         action="store_true",
         dest="print_not_found",
-        default=False,
+        default=settings.print_not_found,
         help="Print sites where the username was not found.",
     )
     output_group.add_argument(
         "--print-errors",
         action="store_true",
         dest="print_check_errors",
-        default=False,
+        default=settings.print_check_errors,
         help="Print errors messages: connection, captcha, site country ban, etc.",
     )
     output_group.add_argument(
@@ -372,14 +372,14 @@ def setup_arguments_parser(settings: Settings):
         "--no-color",
         action="store_true",
         dest="no_color",
-        default=False,
+        default=(not settings.colored_print),
         help="Don't color terminal output",
     )
     output_group.add_argument(
         "--no-progressbar",
         action="store_true",
         dest="no_progressbar",
-        default=False,
+        default=(not settings.show_progressbar),
         help="Don't show progressbar.",
     )
 
@@ -391,7 +391,7 @@ def setup_arguments_parser(settings: Settings):
         "--txt",
         action="store_true",
         dest="txt",
-        default=False,
+        default=settings.txt_report,
         help="Create a TXT report (one report per username).",
     )
     report_group.add_argument(
@@ -399,7 +399,7 @@ def setup_arguments_parser(settings: Settings):
         "--csv",
         action="store_true",
         dest="csv",
-        default=False,
+        default=settings.csv_report,
         help="Create a CSV report (one report per username).",
     )
     report_group.add_argument(
@@ -407,7 +407,7 @@ def setup_arguments_parser(settings: Settings):
         "--html",
         action="store_true",
         dest="html",
-        default=False,
+        default=settings.html_report,
         help="Create an HTML report file (general report on all usernames).",
     )
     report_group.add_argument(
@@ -415,7 +415,7 @@ def setup_arguments_parser(settings: Settings):
         "--xmind",
         action="store_true",
         dest="xmind",
-        default=False,
+        default=settings.xmind_report,
         help="Generate an XMind 8 mindmap report (one report per username).",
     )
     report_group.add_argument(
@@ -423,7 +423,7 @@ def setup_arguments_parser(settings: Settings):
         "--pdf",
         action="store_true",
         dest="pdf",
-        default=False,
+        default=settings.pdf_report,
         help="Generate a PDF report (general report on all usernames).",
     )
     report_group.add_argument(
@@ -431,7 +431,7 @@ def setup_arguments_parser(settings: Settings):
         "--graph",
         action="store_true",
         dest="graph",
-        default=False,
+        default=settings.graph_report,
         help="Generate a graph report (general report on all usernames).",
     )
     report_group.add_argument(
@@ -440,7 +440,7 @@ def setup_arguments_parser(settings: Settings):
         action="store",
         metavar='TYPE',
         dest="json",
-        default='',
+        default=settings.json_report_type,
         choices=SUPPORTED_JSON_REPORT_FORMATS,
         help=f"Generate a JSON report of specific type: {', '.join(SUPPORTED_JSON_REPORT_FORMATS)}"
         " (one report per username).",
@@ -448,7 +448,7 @@ def setup_arguments_parser(settings: Settings):
 
     parser.add_argument(
         "--reports-sorting",
-        default='default',
+        default=settings.report_sorting,
         choices=('default', 'data'),
         help="Method of results sorting in reports (default: in order of getting the result)",
     )
@@ -689,7 +689,9 @@ async def main():
         username = report_context['username']
 
         if args.html:
-            filename = report_filepath_tpl.format(username=username, postfix='_plain.html')
+            filename = report_filepath_tpl.format(
+                username=username, postfix='_plain.html'
+            )
             save_html_report(filename, report_context)
             query_notify.warning(f'HTML report on all usernames saved in {filename}')
 
@@ -699,7 +701,9 @@ async def main():
             query_notify.warning(f'PDF report on all usernames saved in {filename}')
 
         if args.graph:
-            filename = report_filepath_tpl.format(username=username, postfix='_graph.html')
+            filename = report_filepath_tpl.format(
+                username=username, postfix='_graph.html'
+            )
             save_graph_report(filename, general_results, db)
             query_notify.warning(f'Graph report on all usernames saved in {filename}')
 

maigret/resources/data.json

@@ -13078,7 +13078,7 @@
                 "us"
             ],
             "headers": {
-                "authorization": "Bearer BQB8QPkkvz_PhWGy4sSY4ijssYjumEHJgJJBFu3VX2Sm4XIoT9jp0eFZrYL3TayY4QZGHmMiz3BCPLcAth4"
+                "authorization": "Bearer BQB8pjfUq_eeuyy-yvDy5ptoq01QtMbLG0uPW_0t8La7CHd8l4pN3iY4rm5ZBkCEdeWmXyF313-0w9FSp1E"
             },
             "errors": {
                 "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
@@ -14913,7 +14913,7 @@
                 "video"
             ],
             "headers": {
-                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzU2OTI0NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.KZHo96wUe5__rTqZQqAWiJKPKOy2-sjyxRjhOuuhyEc"
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzY0MDY3NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.7tNEeKdpPtvT3k4G-7neQhO1SniqondSkOQ50SUkuNc"
             },
             "activation": {
                 "url": "https://vimeo.com/_rv/viewer",

maigret/resources/settings.json

@@ -26,5 +26,23 @@
     "reports_path": "reports",
     "proxy_url": null,
     "tor_proxy_url": "socks5://127.0.0.1:9050",
-    "i2p_proxy_url": "http://127.0.0.1:4444"
+    "i2p_proxy_url": "http://127.0.0.1:4444",
+    "domain_search": false,
+    "scan_all_sites": false,
+    "top_sites_count": 500,
+    "scan_disabled_sites": false,
+    "scan_sites_list": [],
+    "self_check_enabled": false,
+    "print_not_found": false,
+    "print_check_errors": false,
+    "colored_print": true,
+    "show_progressbar": true,
+    "report_sorting": "default",
+    "json_report_type": "",
+    "txt_report": false,
+    "csv_report": false,
+    "xmind_report": false,
+    "graph_report": false,
+    "pdf_report": false,
+    "html_report": false
 }

maigret/settings.py

@@ -24,6 +24,24 @@ class Settings:
     proxy_url: str
     tor_proxy_url: str
     i2p_proxy_url: str
+    domain_search: bool
+    scan_all_sites: bool
+    top_sites_count: int
+    scan_disabled_sites: bool
+    scan_sites_list: List
+    self_check_enabled: bool
+    print_not_found: bool
+    print_check_errors: bool
+    colored_print: bool
+    show_progressbar: bool
+    report_sorting: str
+    json_report_type: str
+    txt_report: bool
+    csv_report: bool
+    xmind_report: bool
+    pdf_report: bool
+    html_report: bool
+    graph_report: bool
 
     # submit mode settings
     presence_strings: list

maigret/utils.py

@@ -1,3 +1,4 @@
+# coding: utf8
 import ast
 import difflib
 import re
@@ -73,15 +74,22 @@ def ascii_data_display(data: str) -> Any:
 
 
 def get_dict_ascii_tree(items, prepend="", new_line=True):
+    new_result = b'\xe2\x94\x9c'.decode()
+    new_line = b'\xe2\x94\x80'.decode()
+    last_result = b'\xe2\x94\x94'.decode()
+    skip_result = b'\xe2\x94\x82'.decode()
+
     text = ""
     for num, item in enumerate(items):
-        box_symbol = "┣╸" if num != len(items) - 1 else "┗╸"
+        box_symbol = (
+            new_result + new_line if num != len(items) - 1 else last_result + new_line
+        )
 
         if type(item) == tuple:
             field_name, field_value = item
             if field_value.startswith("['"):
                 is_last_item = num == len(items) - 1
-                prepend_symbols = " " * 3 if is_last_item else " ┃ "
+                prepend_symbols = " " * 3 if is_last_item else f" {skip_result} "
                 data = ascii_data_display(field_value)
                 field_value = get_dict_ascii_tree(data, prepend_symbols)
             text += f"\n{prepend}{box_symbol}{field_name}: {field_value}"

tests/test_utils.py

@@ -123,19 +123,19 @@ def test_get_dict_ascii_tree():
     assert (
         ascii_tree
         == """
- ┣╸uid: dXJpOm5vZGU6VXNlcjoyNjQwMzQxNQ==
+ ├─uid: dXJpOm5vZGU6VXNlcjoyNjQwMzQxNQ==
- ┣╸legacy_id: 26403415
+ ├─legacy_id: 26403415
- ┣╸username: alexaimephotographycars
+ ├─username: alexaimephotographycars
- ┣╸name: Alex Aimé
+ ├─name: Alex Aimé
- ┣╸links: 
+ ├─links: 
- ┃ ┗╸ www.instagram.com/street.reality.photography/
+ │ └─ www.instagram.com/street.reality.photography/
- ┣╸created_at: 2018-05-04T10:17:01.000+0000
+ ├─created_at: 2018-05-04T10:17:01.000+0000
- ┣╸image: https://drscdn.500px.org/user_avatar/26403415/q%3D85_w%3D300_h%3D300/v2?webp=true&v=2&sig=0235678a4f7b65e007e864033ebfaf5ef6d87fad34f80a8639d985320c20fe3b
+ ├─image: https://drscdn.500px.org/user_avatar/26403415/q%3D85_w%3D300_h%3D300/v2?webp=true&v=2&sig=0235678a4f7b65e007e864033ebfaf5ef6d87fad34f80a8639d985320c20fe3b
- ┣╸image_bg: https://drscdn.500px.org/user_cover/26403415/q%3D65_m%3D2048/v2?webp=true&v=1&sig=bea411fb158391a4fdad498874ff17088f91257e59dfb376ff67e3a44c3a4201
+ ├─image_bg: https://drscdn.500px.org/user_cover/26403415/q%3D65_m%3D2048/v2?webp=true&v=1&sig=bea411fb158391a4fdad498874ff17088f91257e59dfb376ff67e3a44c3a4201
- ┣╸website: www.instagram.com/street.reality.photography/
+ ├─website: www.instagram.com/street.reality.photography/
- ┣╸facebook_link:  www.instagram.com/street.reality.photography/
+ ├─facebook_link:  www.instagram.com/street.reality.photography/
- ┣╸instagram_username: Street.Reality.Photography
+ ├─instagram_username: Street.Reality.Photography
- ┗╸twitter_username: Alexaimephotogr"""
+ └─twitter_username: Alexaimephotogr"""
     )