Merge branch 'main' into settings-new-configs

README.md

@@ -23,7 +23,7 @@
 
 **Maigret** collect a dossier on a person **by username only**, checking for accounts on a huge number of sites and gathering all the available information from web pages. No API keys required. Maigret is an easy-to-use and powerful fork of [Sherlock](https://github.com/sherlock-project/sherlock).
 
-Currently supported more than 2000 sites ([full list](https://raw.githubusercontent.com/soxoj/maigret/main/sites.md)), search is launched against 500 popular sites in descending order of popularity by default. Also supported checking of Tor sites, I2P sites, and domains (via DNS resolving).
+Currently supported more than 2500 sites ([full list](https://github.com/soxoj/maigret/blob/main/sites.md)), search is launched against 500 popular sites in descending order of popularity by default. Also supported checking of Tor sites, I2P sites, and domains (via DNS resolving).
 
 ## Main features
 

docs/source/command-line-options.rst

@@ -23,7 +23,6 @@ extracted username and ids. :doc:`Examples <extracting-information-from-pages>`.
 Main options
 ------------
 
-=======
 Options are also configurable through settings files, see
 :doc:`settings section <settings>`.
 

docs/source/index.rst

@@ -27,3 +27,4 @@ You may be interested in:
    supported-identifier-types
    tags
    usage-examples
+   settings

docs/source/settings.rst

@@ -0,0 +1,26 @@
+.. _settings:
+
+Settings
+==============
+
+Options are also configurable through settings files. See
+`settings.json <https://github.com/soxoj/maigret/blob/main/maigret/resources/settings.json>`
+for the list of currently supported options.
+
+After start Maigret tries to load configuration from the following sources in exactly the same order:
+
+.. code-block:: console
+
+  # relative path, based on installed package path
+  resources/settings.json
+
+  # absolute path, configuration file in home directory
+  ~/.maigret/settings.json
+
+  # relative path, based on current working directory
+  settings.json
+
+Missing any of these files is not an error.
+If the next settings file contains already known option,
+this option will be rewrited. So it is possible to make
+custom configuration for different users and directories.

maigret/maigret.py

@@ -129,7 +129,9 @@ def setup_arguments_parser(settings: Settings):
 
     parser = ArgumentParser(
         formatter_class=RawDescriptionHelpFormatter,
-        description=f"Maigret v{__version__}",
+        description=f"Maigret v{__version__}\n"
+        "Documentation: https://maigret.readthedocs.io/\n"
+        "All settings are also configurable through files, see docs.",
     )
     parser.add_argument(
         "username",
@@ -149,9 +151,9 @@ def setup_arguments_parser(settings: Settings):
         metavar='TIMEOUT',
         dest="timeout",
         type=timeout_check,
-        default=30,
+        default=settings.timeout,
-        help="Time in seconds to wait for response to requests. "
+        help="Time in seconds to wait for response to requests "
-        "Default timeout of 30.0s. "
+        f"(default {settings.timeout}s). "
         "A longer timeout will be more likely to get results from slow sites. "
         "On the other hand, this may cause a long delay to gather all results. ",
     )
@@ -169,21 +171,21 @@ def setup_arguments_parser(settings: Settings):
         action="store",
         type=int,
         dest="connections",
-        default=100,
+        default=settings.max_connections,
         help="Allowed number of concurrent connections.",
     )
     parser.add_argument(
         "--no-recursion",
         action="store_true",
         dest="disable_recursive_search",
-        default=False,
+        default=(not settings.recursive_search),
         help="Disable recursive search by additional data extracted from pages.",
     )
     parser.add_argument(
         "--no-extracting",
         action="store_true",
         dest="disable_extracting",
-        default=False,
+        default=(not settings.info_extracting),
         help="Disable parsing pages for additional data and other usernames.",
     )
     parser.add_argument(
@@ -197,7 +199,7 @@ def setup_arguments_parser(settings: Settings):
         "--db",
         metavar="DB_FILE",
         dest="db_file",
-        default=None,
+        default=settings.sites_db_path,
         help="Load Maigret database from a JSON file or HTTP web resource.",
     )
     parser.add_argument(
@@ -507,10 +509,7 @@ async def main():
     if args.tags:
         args.tags = list(set(str(args.tags).split(',')))
 
-    if args.db_file is None:
+    db_file = path.join(path.dirname(path.realpath(__file__)), args.db_file)
-        args.db_file = path.join(
-            path.dirname(path.realpath(__file__)), "resources/data.json"
-        )
 
     if args.top_sites == 0 or args.all_sites:
         args.top_sites = sys.maxsize
@@ -525,7 +524,7 @@ async def main():
     )
 
     # Create object with all information about sites we are aware of.
-    db = MaigretDatabase().load_from_path(args.db_file)
+    db = MaigretDatabase().load_from_path(db_file)
     get_top_sites_for_id = lambda x: db.ranked_sites_dict(
         top=args.top_sites,
         tags=args.tags,
@@ -540,7 +539,7 @@ async def main():
         submitter = Submitter(db=db, logger=logger, settings=settings)
         is_submitted = await submitter.dialog(args.new_site_to_submit, args.cookie_file)
         if is_submitted:
-            db.save_to_file(args.db_file)
+            db.save_to_file(db_file)
 
     # Database self-checking
     if args.self_check:
@@ -558,7 +557,7 @@ async def main():
                 'y',
                 '',
             ):
-                db.save_to_file(args.db_file)
+                db.save_to_file(db_file)
                 print('Database was successfully updated.')
             else:
                 print('Updates will be applied only for current search session.')
@@ -708,7 +707,7 @@ async def main():
             print(text_report)
 
     # update database
-    db.save_to_file(args.db_file)
+    db.save_to_file(db_file)
 
 
 def run():

maigret/resources/data.json

@@ -13078,7 +13078,7 @@
                 "us"
             ],
             "headers": {
-                "authorization": "Bearer BQAE_L1qFR4DChej3IyXOV0Qvon6anKopqdcnvO6TjZOBa09RuXRGSy29RLO2Pt9DoQezHJCYd6C7a-hzk8"
+                "authorization": "Bearer BQB8QPkkvz_PhWGy4sSY4ijssYjumEHJgJJBFu3VX2Sm4XIoT9jp0eFZrYL3TayY4QZGHmMiz3BCPLcAth4"
             },
             "errors": {
                 "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
@@ -14913,7 +14913,7 @@
                 "video"
             ],
             "headers": {
-                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzYwNjY5ODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.FI1YY8XtBtG4YJsPoV3V-ykYRNcesKI5EymKj1QBRtg"
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzU2OTI0NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.KZHo96wUe5__rTqZQqAWiJKPKOy2-sjyxRjhOuuhyEc"
             },
             "activation": {
                 "url": "https://vimeo.com/_rv/viewer",

tests/test_cli.py

@@ -7,7 +7,7 @@ DEFAULT_ARGS: Dict[str, Any] = {
     'connections': 100,
     'cookie_file': None,
     'csv': False,
-    'db_file': None,
+    'db_file': 'resources/data.json',
     'debug': False,
     'disable_extracting': False,
     'disable_recursive_search': False,