diff --git a/.gitignore b/.gitignore index 3be27da..862ac39 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,7 @@ src/ .coverage dist/ htmlcov/ -/test_* \ No newline at end of file +/test_* + +# Maigret files +settings.json diff --git a/Makefile b/Makefile index 9034543..fa161a6 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ lint: flake8 --count --select=E9,F63,F7,F82 --show-source --statistics ${LINT_FILES} maigret.py @echo 'warning' - flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=E731,W503 ${LINT_FILES} maigret.py + flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=E731,W503,E501 ${LINT_FILES} maigret.py @echo 'mypy' mypy ${LINT_FILES} diff --git a/maigret/maigret.py b/maigret/maigret.py index 27a7919..515fffa 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -8,6 +8,7 @@ import sys import platform from argparse import ArgumentParser, RawDescriptionHelpFormatter from typing import List, Tuple +import os.path as path from socid_extractor import extract, parse @@ -111,7 +112,7 @@ def extract_ids_from_results(results: QueryResultWrapper, db: MaigretDatabase) - return ids_results -def setup_arguments_parser(): +def setup_arguments_parser(settings: Settings): from aiohttp import __version__ as aiohttp_version from requests import __version__ as requests_version from socid_extractor import __version__ as socid_version @@ -159,7 +160,7 @@ def setup_arguments_parser(): action="store", type=int, metavar='RETRIES', - default=1, + default=settings.retries_count, help="Attempts to restart temporarily failed requests.", ) parser.add_argument( @@ -453,9 +454,6 @@ def setup_arguments_parser(): async def main(): - arg_parser = setup_arguments_parser() - args = arg_parser.parse_args() - # Logging log_level = logging.ERROR logging.basicConfig( @@ -463,15 +461,27 @@ async def main(): datefmt='%H:%M:%S', level=log_level, ) + logger = logging.getLogger('maigret') + logger.setLevel(log_level) + # Load settings + settings = Settings() + settings_loaded, err = settings.load() + + if not settings_loaded: + logger.error(err) + sys.exit(3) + + arg_parser = setup_arguments_parser(settings) + args = arg_parser.parse_args() + + # Re-set loggging level based on args if args.debug: log_level = logging.DEBUG elif args.info: log_level = logging.INFO elif args.verbose: log_level = logging.WARNING - - logger = logging.getLogger('maigret') logger.setLevel(log_level) # Usernames initial list @@ -497,15 +507,9 @@ async def main(): if args.tags: args.tags = list(set(str(args.tags).split(','))) - settings = Settings( - os.path.join( - os.path.dirname(os.path.realpath(__file__)), "resources/settings.json" - ) - ) - if args.db_file is None: - args.db_file = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "resources/data.json" + args.db_file = path.join( + path.dirname(path.realpath(__file__)), "resources/data.json" ) if args.top_sites == 0 or args.all_sites: @@ -568,7 +572,7 @@ async def main(): os.makedirs(args.folderoutput, exist_ok=True) # Define one report filename template - report_filepath_tpl = os.path.join(args.folderoutput, 'report_{username}{postfix}') + report_filepath_tpl = path.join(args.folderoutput, 'report_{username}{postfix}') if usernames == {}: # magic params to exit after init diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 5a4b51a..cfa2706 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -13078,7 +13078,7 @@ "us" ], "headers": { - "authorization": "Bearer BQB8QPkkvz_PhWGy4sSY4ijssYjumEHJgJJBFu3VX2Sm4XIoT9jp0eFZrYL3TayY4QZGHmMiz3BCPLcAth4" + "authorization": "Bearer BQCJhizJUIWlnA6-hlMrHeTsb0jCi067H73LzQ1bWk8Iz1tQ6DIE6WyvbJ2x1bapJhZ_MLNiQbTp-dCxz7g" }, "errors": { "Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn" @@ -14913,7 +14913,7 @@ "video" ], "headers": { - "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzU2OTI0NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.KZHo96wUe5__rTqZQqAWiJKPKOy2-sjyxRjhOuuhyEc" + "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2MzYxMDg1NjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.kqIzH7O99ov6IrTfeQoA7NNaSZe_7czUgrwKxHnBs3o" }, "activation": { "url": "https://vimeo.com/_rv/viewer", diff --git a/maigret/resources/settings.json b/maigret/resources/settings.json index 1dbe06e..e395855 100644 --- a/maigret/resources/settings.json +++ b/maigret/resources/settings.json @@ -13,5 +13,7 @@ "e-mail" ], "supposed_usernames": [ - "alex", "god", "admin", "red", "blue", "john"] + "alex", "god", "admin", "red", "blue", "john" + ], + "retries_count": 1 } \ No newline at end of file diff --git a/maigret/settings.py b/maigret/settings.py index 730c2ef..0f6fd77 100644 --- a/maigret/settings.py +++ b/maigret/settings.py @@ -1,28 +1,57 @@ +import os +import os.path as path import json +SETTINGS_FILES_PATHS = [ + path.join(path.dirname(path.realpath(__file__)), "resources/settings.json"), + '~/.maigret/settings.json', + path.join(os.getcwd(), 'settings.json'), +] + class Settings: + # main maigret setting + retries_count: int + + # submit mode settings presence_strings: list supposed_usernames: list - def __init__(self, filename): - data = {} + def __init__(self): + pass - try: - with open(filename, "r", encoding="utf-8") as file: - try: + def load(self, paths=None): + was_inited = False + + if not paths: + paths = SETTINGS_FILES_PATHS + + for filename in paths: + data = {} + + try: + with open(filename, "r", encoding="utf-8") as file: data = json.load(file) - except Exception as error: - raise ValueError( - f"Problem with parsing json contents of " - f"settings file '{filename}': {str(error)}." - ) - except FileNotFoundError as error: - raise FileNotFoundError( - f"Problem while attempting to access settings file '{filename}'." - ) from error + except FileNotFoundError: + # treast as a normal situation + pass + except Exception as error: + return False, ValueError( + f"Problem with parsing json contents of " + f"settings file '{filename}': {str(error)}." + ) - self.__dict__.update(data) + self.__dict__.update(data) + import logging + + logging.error(data) + if data: + was_inited = True + + return ( + was_inited, + f'None of the default settings files found: {", ".join(paths)}', + ) @property def json(self): diff --git a/tests/conftest.py b/tests/conftest.py index 2210eaa..4bbc8c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,10 +7,12 @@ from _pytest.mark import Mark from maigret.sites import MaigretDatabase from maigret.maigret import setup_arguments_parser +from maigret.settings import Settings CUR_PATH = os.path.dirname(os.path.realpath(__file__)) JSON_FILE = os.path.join(CUR_PATH, '../maigret/resources/data.json') +SETTINGS_FILE = os.path.join(CUR_PATH, '../maigret/resources/settings.json') TEST_JSON_FILE = os.path.join(CUR_PATH, 'db.json') LOCAL_TEST_JSON_FILE = os.path.join(CUR_PATH, 'local.json') empty_mark = Mark('', (), {}) @@ -59,7 +61,9 @@ def reports_autoclean(): @pytest.fixture(scope='session') def argparser(): - return setup_arguments_parser() + settings = Settings() + settings.load([SETTINGS_FILE]) + return setup_arguments_parser(settings) @pytest.fixture(scope="session") diff --git a/tests/test_sites.py b/tests/test_sites.py index f494922..23610c6 100644 --- a/tests/test_sites.py +++ b/tests/test_sites.py @@ -1,6 +1,5 @@ """Maigret Database test functions""" from maigret.sites import MaigretDatabase, MaigretSite -from maigret.utils import URLMatcher EXAMPLE_DB = { 'engines': {