Added cascade settings loading from ~/.maigret/settings.json and ./settings.json

.gitignore

@@ -30,4 +30,7 @@ src/
 .coverage
 dist/
 htmlcov/
-/test_*
+/test_*
+
+# Maigret files
+settings.json

Makefile

@@ -13,7 +13,7 @@ lint:
 	flake8 --count --select=E9,F63,F7,F82 --show-source --statistics ${LINT_FILES} maigret.py
 
 	@echo 'warning'
-	flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=E731,W503 ${LINT_FILES} maigret.py
+	flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=E731,W503,E501 ${LINT_FILES} maigret.py
 
 	@echo 'mypy'
 	mypy ${LINT_FILES}

maigret/maigret.py

@@ -8,6 +8,7 @@ import sys
 import platform
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 from typing import List, Tuple
+import os.path as path
 
 from socid_extractor import extract, parse
 
@@ -111,7 +112,7 @@ def extract_ids_from_results(results: QueryResultWrapper, db: MaigretDatabase) -
     return ids_results
 
 
-def setup_arguments_parser():
+def setup_arguments_parser(settings: Settings):
     from aiohttp import __version__ as aiohttp_version
     from requests import __version__ as requests_version
     from socid_extractor import __version__ as socid_version
@@ -159,7 +160,7 @@ def setup_arguments_parser():
         action="store",
         type=int,
         metavar='RETRIES',
-        default=1,
+        default=settings.retries_count,
         help="Attempts to restart temporarily failed requests.",
     )
     parser.add_argument(
@@ -453,9 +454,6 @@ def setup_arguments_parser():
 
 
 async def main():
-    arg_parser = setup_arguments_parser()
-    args = arg_parser.parse_args()
-
     # Logging
     log_level = logging.ERROR
     logging.basicConfig(
@@ -463,15 +461,27 @@ async def main():
         datefmt='%H:%M:%S',
         level=log_level,
     )
+    logger = logging.getLogger('maigret')
+    logger.setLevel(log_level)
 
+    # Load settings
+    settings = Settings()
+    settings_loaded, err = settings.load()
+
+    if not settings_loaded:
+        logger.error(err)
+        sys.exit(3)
+
+    arg_parser = setup_arguments_parser(settings)
+    args = arg_parser.parse_args()
+
+    # Re-set loggging level based on args
     if args.debug:
         log_level = logging.DEBUG
     elif args.info:
         log_level = logging.INFO
     elif args.verbose:
         log_level = logging.WARNING
-
-    logger = logging.getLogger('maigret')
     logger.setLevel(log_level)
 
     # Usernames initial list
@@ -497,15 +507,9 @@ async def main():
     if args.tags:
         args.tags = list(set(str(args.tags).split(',')))
 
-    settings = Settings(
-        os.path.join(
-            os.path.dirname(os.path.realpath(__file__)), "resources/settings.json"
-        )
-    )
-
     if args.db_file is None:
-        args.db_file = os.path.join(
-            os.path.dirname(os.path.realpath(__file__)), "resources/data.json"
+        args.db_file = path.join(
+            path.dirname(path.realpath(__file__)), "resources/data.json"
         )
 
     if args.top_sites == 0 or args.all_sites:
@@ -568,7 +572,7 @@ async def main():
     os.makedirs(args.folderoutput, exist_ok=True)
 
     # Define one report filename template
-    report_filepath_tpl = os.path.join(args.folderoutput, 'report_{username}{postfix}')
+    report_filepath_tpl = path.join(args.folderoutput, 'report_{username}{postfix}')
 
     if usernames == {}:
         # magic params to exit after init

maigret/resources/data.json

@@ -13078,7 +13078,7 @@
                 "us"
             ],
             "headers": {
-                "authorization": "Bearer BQB8QPkkvz_PhWGy4sSY4ijssYjumEHJgJJBFu3VX2Sm4XIoT9jp0eFZrYL3TayY4QZGHmMiz3BCPLcAth4"
+                "authorization": "Bearer BQCJhizJUIWlnA6-hlMrHeTsb0jCi067H73LzQ1bWk8Iz1tQ6DIE6WyvbJ2x1bapJhZ_MLNiQbTp-dCxz7g"
             },
             "errors": {
                 "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
@@ -14913,7 +14913,7 @@
                 "video"
             ],
             "headers": {
-                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzU2OTI0NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.KZHo96wUe5__rTqZQqAWiJKPKOy2-sjyxRjhOuuhyEc"
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzYxMDg1NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.kqIzH7O99ov6IrTfeQoA7NNaSZe_7czUgrwKxHnBs3o"
             },
             "activation": {
                 "url": "https://vimeo.com/_rv/viewer",

maigret/resources/settings.json

@@ -13,5 +13,7 @@
         "e-mail"
     ],
     "supposed_usernames": [
-        "alex", "god", "admin", "red", "blue", "john"]
+        "alex", "god", "admin", "red", "blue", "john"
+    ],
+    "retries_count": 1
 }

maigret/settings.py

@@ -1,28 +1,57 @@
+import os
+import os.path as path
 import json
 
+SETTINGS_FILES_PATHS = [
+    path.join(path.dirname(path.realpath(__file__)), "resources/settings.json"),
+    '~/.maigret/settings.json',
+    path.join(os.getcwd(), 'settings.json'),
+]
+
 
 class Settings:
+    # main maigret setting
+    retries_count: int
+
+    # submit mode settings
     presence_strings: list
     supposed_usernames: list
 
-    def __init__(self, filename):
-        data = {}
+    def __init__(self):
+        pass
 
-        try:
-            with open(filename, "r", encoding="utf-8") as file:
-                try:
+    def load(self, paths=None):
+        was_inited = False
+
+        if not paths:
+            paths = SETTINGS_FILES_PATHS
+
+        for filename in paths:
+            data = {}
+
+            try:
+                with open(filename, "r", encoding="utf-8") as file:
                     data = json.load(file)
-                except Exception as error:
-                    raise ValueError(
-                        f"Problem with parsing json contents of "
-                        f"settings file '{filename}':  {str(error)}."
-                    )
-        except FileNotFoundError as error:
-            raise FileNotFoundError(
-                f"Problem while attempting to access settings file '{filename}'."
-            ) from error
+            except FileNotFoundError:
+                # treast as a normal situation
+                pass
+            except Exception as error:
+                return False, ValueError(
+                    f"Problem with parsing json contents of "
+                    f"settings file '{filename}':  {str(error)}."
+                )
 
-        self.__dict__.update(data)
+            self.__dict__.update(data)
+            import logging
+
+            logging.error(data)
+            if data:
+                was_inited = True
+
+        return (
+            was_inited,
+            f'None of the default settings files found: {", ".join(paths)}',
+        )
 
     @property
     def json(self):

tests/conftest.py

@@ -7,10 +7,12 @@ from _pytest.mark import Mark
 
 from maigret.sites import MaigretDatabase
 from maigret.maigret import setup_arguments_parser
+from maigret.settings import Settings
 
 
 CUR_PATH = os.path.dirname(os.path.realpath(__file__))
 JSON_FILE = os.path.join(CUR_PATH, '../maigret/resources/data.json')
+SETTINGS_FILE = os.path.join(CUR_PATH, '../maigret/resources/settings.json')
 TEST_JSON_FILE = os.path.join(CUR_PATH, 'db.json')
 LOCAL_TEST_JSON_FILE = os.path.join(CUR_PATH, 'local.json')
 empty_mark = Mark('', (), {})
@@ -59,7 +61,9 @@ def reports_autoclean():
 
 @pytest.fixture(scope='session')
 def argparser():
-    return setup_arguments_parser()
+    settings = Settings()
+    settings.load([SETTINGS_FILE])
+    return setup_arguments_parser(settings)
 
 
 @pytest.fixture(scope="session")

tests/test_sites.py

@@ -1,6 +1,5 @@
 """Maigret Database test functions"""
 from maigret.sites import MaigretDatabase, MaigretSite
-from maigret.utils import URLMatcher
 
 EXAMPLE_DB = {
     'engines': {