CLI test fixes

merge upstream

README.md

@@ -75,7 +75,6 @@ You can launch Maigret using cloud shells and Jupyter notebooks. Press one of th
 
 Maigret can be installed using pip, Docker, or simply can be launched from the cloned repo.
 
-
 **NOTE**: Python 3.10 or higher and pip is required, **Python 3.11 is recommended.**
 
 ```bash
@@ -132,30 +131,6 @@ maigret user1 user2 user3 -a
 
 Use `maigret --help` to get full options description. Also options [are documented](https://maigret.readthedocs.io/en/latest/command-line-options.html).
 
-### Web interface
-
-You can run Maigret with a web interface, where you can view the graph with results and download reports of all formats on a single page.
-
-<details>
-<summary>Web Interface Screenshots</summary>
-
-![Web interface: how to start](https://raw.githubusercontent.com/soxoj/maigret/main/static/web_interface_screenshot_start.png)
-
-![Web interface: results](https://raw.githubusercontent.com/soxoj/maigret/main/static/web_interface_screenshot.png)
-
-</details>
-
-Instructions:
-
-1. Run Maigret with the ``--web`` flag and specify the port number.
-
-```console
-maigret --web 5000
-```
-2. Open http://127.0.0.1:5000 in your browser and enter one or more usernames to make a search.
-
-3. Wait a bit for the search to complete and view the graph with results, the table with all accounts found, and download reports of all formats.
-
 ## Contributing
 
 Maigret has open-source code, so you may contribute your own sites by adding them to `data.json` file, or bring changes to it's code!

docs/source/features.rst

@@ -5,34 +5,6 @@ Features
 
 This is the list of Maigret features.
 
-.. _web-interface:
-
-Web Interface
--------------
-
-You can run Maigret with a web interface, where you can view the graph with results and download reports of all formats on a single page.
-
-
-.. image:: https://raw.githubusercontent.com/soxoj/maigret/main/static/web_interface_screenshot_start.png
-   :alt: Web interface: how to start
-
-
-.. image:: https://raw.githubusercontent.com/soxoj/maigret/main/static/web_interface_screenshot.png
-   :alt: Web interface: results
-
-
-Instructions:
-
-1. Run Maigret with the ``--web`` flag and specify the port number.
-
-.. code-block:: console
-
-  maigret --web 5000
-
-2. Open http://127.0.0.1:5000 in your browser and enter one or more usernames to make a search.
-
-3. Wait a bit for the search to complete and view the graph with results, the table with all accounts found, and download reports of all formats.
-
 Personal info gathering
 -----------------------
 

docs/source/usage-examples.rst

@@ -3,16 +3,6 @@
 Usage examples
 ==============
 
-You can use Maigret as:
-
-- a command line tool: inital and a default mode
-- a `web interface <#web-interface>`_: view the graph with results and download all report formats on a single page
-- a library: integrate Maigret into your own project
-
-Use Cases
----------
-
-
 1. Search for accounts with username ``machine42`` on top 500 sites (by default, according to Alexa rank) from the Maigret DB.
 
 .. code-block:: console

maigret/checking.py

@@ -26,7 +26,11 @@ except ImportError:
 from . import errors
 from .activation import ParsingActivator, import_aiohttp_cookies
 from .errors import CheckError
-from .executors import AsyncioQueueGeneratorExecutor
+from .executors import (
+    AsyncExecutor,
+    AsyncioSimpleExecutor,
+    AsyncioProgressbarQueueExecutor,
+)
 from .result import MaigretCheckResult, MaigretCheckStatus
 from .sites import MaigretDatabase, MaigretSite
 from .types import QueryOptions, QueryResultWrapper
@@ -48,15 +52,6 @@ SUPPORTED_IDS = (
 BAD_CHARS = "#"
 
 
-def is_cloudflare_bypass_active(value) -> bool:
-    """True if Cloudflare webgate URL rewrite should run (``--cloudflare-bypass`` or settings)."""
-    if value is True:
-        return True
-    if isinstance(value, dict):
-        return bool(value.get("enabled", False))
-    return False
-
-
 class CheckerBase:
     pass
 
@@ -440,11 +435,6 @@ def make_site_result(
 
     # workaround to prevent slash errors
     url = re.sub("(?<!:)/+", "/", url)
-    url_probe = site.url_probe
-
-    if 'cloudflare' in site.tags and options.get("cloudflare_bypass"):
-        url_probe = 'http://localhost:8000/html?url=' + url
-        logger.info(f"Using cloudflare proxy for {site.name}")
 
     # always clearweb_checker for now
     checker = options["checkers"][site.protocol]
@@ -486,6 +476,7 @@ def make_site_result(
     else:
         # URL of user on site (if it exists)
         results_site["url_user"] = url
+        url_probe = site.url_probe
         if url_probe is None:
             # Probe URL is normal one seen by people out on the web.
             url_probe = url
@@ -602,7 +593,6 @@ async def maigret(
     cookies=None,
     retries=0,
     check_domains=False,
-    cloudflare_bypass=False,
     *args,
     **kwargs,
 ) -> QueryResultWrapper:
@@ -680,13 +670,18 @@ async def maigret(
         await debug_ip_request(clearweb_checker, logger)
 
     # setup parallel executor
-    executor = AsyncioQueueGeneratorExecutor(
-        logger=logger,
-        in_parallel=max_connections,
-        timeout=timeout + 0.5,
-        *args,
-        **kwargs,
-    )
+    executor: Optional[AsyncExecutor] = None
+    if no_progressbar:
+        # TODO: switch to AsyncioProgressbarQueueExecutor with progress object mock
+        executor = AsyncioSimpleExecutor(logger=logger)
+    else:
+        executor = AsyncioProgressbarQueueExecutor(
+            logger=logger,
+            in_parallel=max_connections,
+            timeout=timeout + 0.5,
+            *args,
+            **kwargs,
+        )
 
     # make options objects for all the requests
     options: QueryOptions = {}
@@ -701,14 +696,12 @@ async def maigret(
     options["timeout"] = timeout
     options["id_type"] = id_type
     options["forced"] = forced
-    options["cloudflare_bypass"] = is_cloudflare_bypass_active(cloudflare_bypass)
 
     # results from analysis of all sites
     all_results: Dict[str, QueryResultWrapper] = {}
 
     sites = list(site_dict.keys())
 
-    executor_limit = timeout + 0.5
     attempts = retries + 1
     while attempts:
         tasks_dict = {}
@@ -723,11 +716,7 @@ async def maigret(
                     sitename,
                     '',
                     MaigretCheckStatus.UNKNOWN,
-                    error=CheckError(
-                        'Request timeout',
-                        f'No response within {executor_limit:.1f}s per site '
-                        f'(increase --timeout or use --no-progressbar)',
-                    ),
+                    error=CheckError('Request failed'),
                 ),
             }
             tasks_dict[sitename] = (
@@ -739,17 +728,13 @@ async def maigret(
                 },
             )
 
-        cur_results = []
-        with alive_bar(
-            len(tasks_dict), title="Searching", force_tty=True, disable=no_progressbar
-        ) as progress:
-            async for result in executor.run(tasks_dict.values()):
-                cur_results.append(result)
-                progress()
+        cur_results = await executor.run(tasks_dict.values())
+
+        # wait for executor timeout errors
+        await asyncio.sleep(1)
 
         all_results.update(cur_results)
 
-        # rerun for failed sites
         sites = get_failed_sites(dict(cur_results))
         attempts -= 1
 
@@ -808,7 +793,6 @@ async def site_self_check(
     i2p_proxy=None,
     skip_errors=False,
     cookies=None,
-    cloudflare_bypass=False,
 ):
     changes = {
         "disabled": False,
@@ -836,7 +820,6 @@ async def site_self_check(
                 tor_proxy=tor_proxy,
                 i2p_proxy=i2p_proxy,
                 cookies=cookies,
-                cloudflare_bypass=cloudflare_bypass,
             )
 
             # don't disable entries with other ids types
@@ -908,7 +891,6 @@ async def self_check(
     proxy=None,
     tor_proxy=None,
     i2p_proxy=None,
-    cloudflare_bypass=False,
 ) -> bool:
     sem = asyncio.Semaphore(max_connections)
     tasks = []
@@ -924,17 +906,7 @@ async def self_check(
 
     for _, site in all_sites.items():
         check_coro = site_self_check(
-            site,
-            logger,
-            sem,
-            db,
-            silent,
-            proxy,
-            tor_proxy,
-            i2p_proxy,
-            skip_errors=True,
-            cookies=None,
-            cloudflare_bypass=cloudflare_bypass,
+            site, logger, sem, db, silent, proxy, tor_proxy, i2p_proxy, skip_errors=True
         )
         future = asyncio.ensure_future(check_coro)
         tasks.append(future)

maigret/errors.py

@@ -136,10 +136,7 @@ def extract_and_group(search_res: QueryResultWrapper) -> List[Dict[str, Any]]:
 
 
 def notify_about_errors(
-    search_results: QueryResultWrapper,
-    query_notify,
-    show_statistics=False,
-    print_check_errors=False,
+    search_results: QueryResultWrapper, query_notify, show_statistics=False
 ) -> List[Tuple]:
     """
     Prepare error notifications in search results, text + symbol,
@@ -172,7 +169,7 @@ def notify_about_errors(
             text = f'{e["err"]}: {round(e["perc"],2)}%'
             results.append((text, '!'))
 
-    if was_errs_displayed and not print_check_errors:
+    if was_errs_displayed:
         results.append(
             ('You can see detailed site check errors with a flag `--print-errors`', '-')
         )

maigret/executors.py

@@ -1,7 +1,7 @@
 import asyncio
 import sys
 import time
-from typing import Any, Iterable, List, Callable
+from typing import Any, Iterable, List
 
 import alive_progress
 from alive_progress import alive_bar
@@ -19,7 +19,6 @@ def create_task_func():
 
 
 class AsyncExecutor:
-    # Deprecated: will be removed soon, don't use it
     def __init__(self, *args, **kwargs):
         self.logger = kwargs['logger']
 
@@ -35,7 +34,6 @@ class AsyncExecutor:
 
 
 class AsyncioSimpleExecutor(AsyncExecutor):
-    # Deprecated: will be removed soon, don't use it
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.semaphore = asyncio.Semaphore(kwargs.get('in_parallel', 100))
@@ -50,7 +48,6 @@ class AsyncioSimpleExecutor(AsyncExecutor):
 
 
 class AsyncioProgressbarExecutor(AsyncExecutor):
-    # Deprecated: will be removed soon, don't use it
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -74,7 +71,6 @@ class AsyncioProgressbarExecutor(AsyncExecutor):
 
 
 class AsyncioProgressbarSemaphoreExecutor(AsyncExecutor):
-    # Deprecated: will be removed soon, don't use it
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.semaphore = asyncio.Semaphore(kwargs.get('in_parallel', 1))
@@ -110,28 +106,6 @@ class AsyncioProgressbarQueueExecutor(AsyncExecutor):
         self.progress = None
 
     # TODO: tests
-    @staticmethod
-    def _emit_timeout_notify(args, default_fallback):
-        """Print per-site line when wait_for kills check_site_for_username (no query_notify.update ran)."""
-        if (
-            not default_fallback
-            or not isinstance(default_fallback, (tuple, list))
-            or len(default_fallback) < 2
-        ):
-            return
-        _, results_dict = default_fallback[0], default_fallback[1]
-        if not isinstance(results_dict, dict):
-            return
-        status = results_dict.get('status')
-        if status is None:
-            return
-        if len(args) < 5:
-            return
-        query_notify = args[4]
-        site = results_dict.get('site')
-        similar = bool(getattr(site, 'similar_search', False)) if site is not None else False
-        query_notify.update(status, similar)
-
     async def increment_progress(self, count):
         """Update progress by calling the provided progress function."""
         if self.progress:
@@ -166,7 +140,6 @@ class AsyncioProgressbarQueueExecutor(AsyncExecutor):
                 result = await asyncio.wait_for(query_task, timeout=self.timeout)
             except asyncio.TimeoutError:
                 result = kwargs.get('default')
-                self._emit_timeout_notify(args, result)
 
             self.results.append(result)
 
@@ -201,68 +174,3 @@ class AsyncioProgressbarQueueExecutor(AsyncExecutor):
                     w.cancel()
 
         return self.results
-
-
-class AsyncioQueueGeneratorExecutor:
-    # Deprecated: will be removed soon, don't use it
-    def __init__(self, *args, **kwargs):
-        self.workers_count = kwargs.get('in_parallel', 10)
-        self.queue = asyncio.Queue()
-        self.timeout = kwargs.get('timeout')
-        self.logger = kwargs['logger']
-        self._results = asyncio.Queue()
-        self._stop_signal = object()
-
-    async def worker(self):
-        """Process tasks from the queue and put results into the results queue."""
-        while True:
-            task = await self.queue.get()
-            if task is self._stop_signal:
-                self.queue.task_done()
-                break
-
-            try:
-                f, args, kwargs = task
-                query_future = f(*args, **kwargs)
-                query_task = create_task_func()(query_future)
-
-                try:
-                    result = await asyncio.wait_for(query_task, timeout=self.timeout)
-                except asyncio.TimeoutError:
-                    result = kwargs.get('default')
-                    AsyncioProgressbarQueueExecutor._emit_timeout_notify(args, result)
-                await self._results.put(result)
-            except Exception as e:
-                self.logger.error(f"Error in worker: {e}")
-            finally:
-                self.queue.task_done()
-
-    async def run(self, queries: Iterable[Callable[..., Any]]):
-        """Run workers to process queries in parallel."""
-        start_time = time.time()
-
-        # Add tasks to the queue
-        for t in queries:
-            await self.queue.put(t)
-
-        # Create workers
-        workers = [
-            asyncio.create_task(self.worker()) for _ in range(self.workers_count)
-        ]
-
-        # Add stop signals
-        for _ in range(self.workers_count):
-            await self.queue.put(self._stop_signal)
-
-        try:
-            while any(w.done() is False for w in workers) or not self._results.empty():
-                try:
-                    result = await asyncio.wait_for(self._results.get(), timeout=1)
-                    yield result
-                except asyncio.TimeoutError:
-                    pass
-        finally:
-            # Ensure all workers are awaited
-            await asyncio.gather(*workers)
-            self.execution_time = time.time() - start_time
-            self.logger.debug(f"Spent time: {self.execution_time}")

maigret/maigret.py

@@ -254,12 +254,6 @@ def setup_arguments_parser(settings: Settings):
         default=settings.domain_search,
         help="Enable (experimental) feature of checking domains on usernames.",
     )
-    parser.add_argument(
-        "--cloudflare-bypass",
-        action="store_true",
-        default=settings.cloudflare_bypass,
-        help="Enable Cloudflare bypass (edit settings.json to configure)",
-    )
 
     filter_group = parser.add_argument_group(
         'Site filtering', 'Options to set site search scope'
@@ -334,10 +328,10 @@ def setup_arguments_parser(settings: Settings):
         "--web",
         metavar='PORT',
         type=int,
-        nargs='?',  # Optional PORT value
-        const=5000,  # Default PORT if `--web` is provided without a value
-        default=None,  # Explicitly set default to None
-        help="Launch the web interface on the specified port (default: 5000 if no PORT is provided).",
+        nargs='?',
+        const=5000,  # default if --web is provided without a port
+        default=settings.web_interface_port,
+        help="Launches the web interface on the specified port (default: 5000 if no PORT is provided).",
     )
     output_group = parser.add_argument_group(
         'Output options', 'Options to change verbosity and view of the console output'
@@ -504,9 +498,8 @@ async def main():
 
         port = (
             args.web if args.web else 5000
-        )  # args.web is either the specified port or 5000 by default
+        )  # args.web is either the specified port or 5000 by const
         app.run(port=port)
-        return
 
     # Usernames initial list
     usernames = {
@@ -587,7 +580,6 @@ async def main():
             max_connections=args.connections,
             tor_proxy=args.tor_proxy,
             i2p_proxy=args.i2p_proxy,
-            cloudflare_bypass=args.cloudflare_bypass,
         )
         if is_need_update:
             if input('Do you want to save changes permanently? [Yn]\n').lower() in (
@@ -688,14 +680,10 @@ async def main():
             no_progressbar=args.no_progressbar,
             retries=args.retries,
             check_domains=args.with_domains,
-            cloudflare_bypass=args.cloudflare_bypass,
         )
 
         errs = errors.notify_about_errors(
-            results,
-            query_notify,
-            show_statistics=args.verbose,
-            print_check_errors=args.print_check_errors,
+            results, query_notify, show_statistics=args.verbose
         )
         for e in errs:
             query_notify.warning(*e)

maigret/report.py

@@ -98,20 +98,21 @@ class MaigretGraph:
     def __init__(self, graph):
         self.G = graph
 
-    def add_node(self, key, value, color=None):
+    def add_node(self, key, value):
         node_name = f'{key}: {value}'
 
-        params = dict(self.other_params)
+        params = self.other_params
         if key in SUPPORTED_IDS:
-            params = dict(self.username_params)
+            params = self.username_params
         elif value.startswith('http'):
-            params = dict(self.site_params)
-            
-        params['title'] = node_name
-        if color:
-            params['color'] = color
+            params = self.site_params
+
+        self.G.add_node(node_name, title=node_name, **params)
+
+        if value != value.lower():
+            normalized_node_name = self.add_node(key, value.lower())
+            self.link(node_name, normalized_node_name)
 
-        self.G.add_node(node_name, **params)
         return node_name
 
     def link(self, node1_name, node2_name):
@@ -119,108 +120,96 @@ class MaigretGraph:
 
 
 def save_graph_report(filename: str, username_results: list, db: MaigretDatabase):
+    # moved here to speed up the launch of Maigret
     import networkx as nx
 
     G = nx.Graph()
     graph = MaigretGraph(G)
 
-    base_site_nodes = {}
-    site_account_nodes = {}
-    processed_values = {}  # Track processed values to avoid duplicates
-
     for username, id_type, results in username_results:
-        # Add username node, using normalized version directly if different
-        norm_username = username.lower()
-        username_node_name = graph.add_node(id_type, norm_username)
+        username_node_name = graph.add_node(id_type, username)
 
-        for website_name, dictionary in results.items():
-            if not dictionary or dictionary.get("is_similar"):
+        for website_name in results:
+            dictionary = results[website_name]
+            # TODO: fix no site data issue
+            if not dictionary:
+                continue
+
+            if dictionary.get("is_similar"):
                 continue
 
             status = dictionary.get("status")
-            if not status or status.status != MaigretCheckStatus.CLAIMED:
+            if not status:  # FIXME: currently in case of timeout
                 continue
 
-            # base site node 
-            site_base_url = website_name
-            if site_base_url not in base_site_nodes:
-                base_site_nodes[site_base_url] = graph.add_node('site', site_base_url, color='#28a745')  # Green color
+            if dictionary["status"].status != MaigretCheckStatus.CLAIMED:
+                continue
 
-            site_base_node_name = base_site_nodes[site_base_url]
-
-            # account node
-            account_url = dictionary.get('url_user', f'{site_base_url}/{norm_username}')
-            account_node_id = f"{site_base_url}: {account_url}"
-            if account_node_id not in site_account_nodes:
-                site_account_nodes[account_node_id] = graph.add_node('account', account_url)
-
-            account_node_name = site_account_nodes[account_node_id]
-
-            # link username → account → site
-            graph.link(username_node_name, account_node_name)
-            graph.link(account_node_name, site_base_node_name)
+            site_fallback_name = dictionary.get(
+                'url_user', f'{website_name}: {username.lower()}'
+            )
+            # site_node_name = dictionary.get('url_user', f'{website_name}: {username.lower()}')
+            site_node_name = graph.add_node('site', site_fallback_name)
+            graph.link(username_node_name, site_node_name)
 
             def process_ids(parent_node, ids):
                 for k, v in ids.items():
-                    if k.endswith('_count') or k.startswith('is_') or k.endswith('_at') or k in 'image':
+                    if k.endswith('_count') or k.startswith('is_') or k.endswith('_at'):
+                        continue
+                    if k in 'image':
                         continue
 
-                    # Normalize value if string
-                    norm_v = v.lower() if isinstance(v, str) else v
-                    value_key = f"{k}:{norm_v}"
-                    
-                    if value_key in processed_values:
-                        ids_data_name = processed_values[value_key]
-                    else:
-                        v_data = v
-                        if isinstance(v, str) and v.startswith('['):
-                            try:
-                                v_data = ast.literal_eval(v)
-                            except Exception as e:
-                                logging.error(e)
-                                continue
+                    v_data = v
+                    if v.startswith('['):
+                        try:
+                            v_data = ast.literal_eval(v)
+                        except Exception as e:
+                            logging.error(e)
 
-                        if isinstance(v_data, list):
-                            list_node_name = graph.add_node(k, site_base_url)
-                            processed_values[value_key] = list_node_name
-                            for vv in v_data:
-                                data_node_name = graph.add_node(vv, site_base_url)
-                                graph.link(list_node_name, data_node_name)
+                    # value is a list
+                    if isinstance(v_data, list):
+                        list_node_name = graph.add_node(k, site_fallback_name)
+                        for vv in v_data:
+                            data_node_name = graph.add_node(vv, site_fallback_name)
+                            graph.link(list_node_name, data_node_name)
 
-                                add_ids = {a: b for b, a in db.extract_ids_from_url(vv).items()}
-                                if add_ids:
-                                    process_ids(data_node_name, add_ids)
-                            ids_data_name = list_node_name
-                        else:
-                            ids_data_name = graph.add_node(k, norm_v)
-                            processed_values[value_key] = ids_data_name
-
-                            if 'username' in k or k in SUPPORTED_IDS:
-                                new_username_key = f"username:{norm_v}"
-                                if new_username_key not in processed_values:
-                                    new_username_node_name = graph.add_node('username', norm_v)
-                                    processed_values[new_username_key] = new_username_node_name
-                                    graph.link(ids_data_name, new_username_node_name)
-
-                            add_ids = {k: v for v, k in db.extract_ids_from_url(v).items()}
+                            add_ids = {
+                                a: b for b, a in db.extract_ids_from_url(vv).items()
+                            }
                             if add_ids:
-                                process_ids(ids_data_name, add_ids)
+                                process_ids(data_node_name, add_ids)
+                    else:
+                        # value is just a string
+                        # ids_data_name = f'{k}: {v}'
+                        # if ids_data_name == parent_node:
+                        #     continue
 
-                    graph.link(parent_node, ids_data_name)
+                        ids_data_name = graph.add_node(k, v)
+                        # G.add_node(ids_data_name, size=10, title=ids_data_name, group=3)
+                        graph.link(parent_node, ids_data_name)
+
+                        # check for username
+                        if 'username' in k or k in SUPPORTED_IDS:
+                            new_username_node_name = graph.add_node('username', v)
+                            graph.link(ids_data_name, new_username_node_name)
+
+                        add_ids = {k: v for v, k in db.extract_ids_from_url(v).items()}
+                        if add_ids:
+                            process_ids(ids_data_name, add_ids)
 
             if status.ids_data:
-                process_ids(account_node_name, status.ids_data)
+                process_ids(site_node_name, status.ids_data)
 
-    # Remove overly long nodes
-    nodes_to_remove = [node for node in G.nodes if len(str(node)) > 100]
-    G.remove_nodes_from(nodes_to_remove)
+    nodes_to_remove = []
+    for node in G.nodes:
+        if len(str(node)) > 100:
+            nodes_to_remove.append(node)
 
-    # Remove site nodes with only one connection
-    single_degree_sites = [n for n, deg in G.degree() if n.startswith("site:") and deg <= 1]
-    G.remove_nodes_from(single_degree_sites)
+    [G.remove_node(node) for node in nodes_to_remove]
 
-    # Generate interactive visualization
+    # moved here to speed up the launch of Maigret
     from pyvis.network import Network
+
     nt = Network(notebook=True, height="750px", width="100%")
     nt.from_nx(G)
     nt.show(filename)

maigret/resources/data.json

@@ -3149,7 +3149,6 @@
         },
         "ChaturBate": {
             "tags": [
-                "cloudflare",
                 "us"
             ],
             "checkType": "status_code",
@@ -3329,7 +3328,6 @@
         },
         "CloudflareCommunity": {
             "tags": [
-                "cloudflare",
                 "forum",
                 "tech"
             ],
@@ -4204,7 +4202,6 @@
         },
         "Discogs": {
             "tags": [
-                "cloudflare",
                 "music",
                 "us"
             ],
@@ -4921,7 +4918,6 @@
         },
         "Etsy": {
             "tags": [
-                "cloudflare",
                 "shopping",
                 "us"
             ],
@@ -5041,7 +5037,6 @@
             "usernameUnclaimed": "noonewouldeverusethis7",
             "alexaRank": 240,
             "tags": [
-                "cloudflare",
                 "design"
             ]
         },
@@ -6275,7 +6270,6 @@
         "Freepik": {
             "tags": [
                 "art",
-                "cloudflare",
                 "photo",
                 "stock"
             ],
@@ -7224,8 +7218,7 @@
             "url": "https://gramho.com/explore-hashtag/{username}",
             "source": "Instagram",
             "usernameClaimed": "adam",
-            "usernameUnclaimed": "noonewouldeverusethis7",
-            "disabled": true
+            "usernameUnclaimed": "noonewouldeverusethis7"
         },
         "Gravatar": {
             "tags": [
@@ -8673,7 +8666,6 @@
         },
         "Kickstarter": {
             "tags": [
-                "cloudflare",
                 "finance",
                 "us"
             ],
@@ -12029,7 +12021,6 @@
         },
         "Patreon": {
             "tags": [
-                "cloudflare",
                 "finance"
             ],
             "checkType": "status_code",
@@ -13522,7 +13513,6 @@
         },
         "Redbubble": {
             "tags": [
-                "cloudflare",
                 "shopping",
                 "us"
             ],
@@ -15067,7 +15057,6 @@
         },
         "SourceForge": {
             "tags": [
-                "cloudflare",
                 "coding",
                 "us"
             ],
@@ -16920,14 +16909,10 @@
         },
         "Twitch": {
             "tags": [
-                "cloudflare",
                 "streaming",
                 "us"
             ],
-            "headers": {
-                "cookies": "_gat=1; _ga_4ZM72D0Y59=GS1.2.1734305902.1.0.1734305902.0.0.0; _ga=GA1.2.1051951095.1734305902; _gid=GA1.2.30506583.1734305902; cf_clearance=xo5uTOkBRsAYb4so4QSu1h8tlFcFJyYSA2SBEHyYA2U-1734305900-1.2.1.1-l9mQ677uPsBenPceAasuW_ZVBqRgQqy4df.13gRl6y4aFBf._3bLo1c3.uVZOXwMxL_iVN.EvEHEBiNczBNMM6riJrVWgiLx1O1jGRbhIiGP.tsomZgyl_bNupNbWxZNzHy454hC0iUigDrE5jkJJoazDRJNc5532wj9nT.U9DDBxW3RplVCdj4x5sMt3K3IXADYvAGabBQnzvS3rEr_w66KClwAehy69tWHVSPDkc.ww7QnxdDItYqmtL8bz9IScdouTAvU_MWK6oxvxcLc6GQFCQZnoToeX8Fgeui2flhV.kXXjEQ4NjypxSFakcCPIysHZOUjKfv93.W9Vfl7id.Y8DUpsmxEPVOpfcrGY6YvFtk6yJhvUQryJftS5b7E5P5jVPW_pPlMWSTWL9IaysG7INm6ZjDyjsVG7OBJIUujSSMlyoKiR8sv0L2ueHt6",
-                "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
-            },
+            "urlProbe": "https://twitchtracker.com/{username}",
             "checkType": "status_code",
             "alexaRank": 34,
             "urlMain": "https://www.twitch.tv/",
@@ -17093,7 +17078,7 @@
         },
         "Udemy": {
             "tags": [
-                "cloudflare"
+                "in"
             ],
             "checkType": "response_url",
             "alexaRank": 131,
@@ -17490,6 +17475,9 @@
                 ],
                 "method": "vimeo"
             },
+            "headers": {
+                "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3MzM5Njc3MjAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbCwianRpIjoiNGJkNDE4NzktM2VhOS00ZWRiLWIzZDUtNjAyNjQ3YjMyNTVhIn0.kPbKREujSfYsisyF0pS_HskTapRlHBfVLRw4cis1ezk"
+            },
             "urlProbe": "https://api.vimeo.com/users/{username}?fields=name%2Cgender%2Cbio%2Curi%2Clink%2Cbackground_video%2Clocation_details%2Cpictures%2Cverified%2Cmetadata.public_videos.total%2Cavailable_for_hire%2Ccan_work_remotely%2Cmetadata.connections.videos.total%2Cmetadata.connections.albums.total%2Cmetadata.connections.followers.total%2Cmetadata.connections.following.total%2Cmetadata.public_videos.total%2Cmetadata.connections.vimeo_experts.is_enrolled%2Ctotal_collection_count%2Ccreated_time%2Cprofile_preferences%2Cmembership%2Cclients%2Cskills%2Cproject_types%2Crates%2Ccategories%2Cis_expert%2Cprofile_discovery%2Cwebsites%2Ccontact_emails&fetch_user_profile=1",
             "checkType": "status_code",
             "alexaRank": 148,
@@ -27255,7 +27243,6 @@
         },
         "upwork.com": {
             "tags": [
-                "cloudflare",
                 "us"
             ],
             "engine": "engine404",
@@ -35781,7 +35768,6 @@
     "tags": [
         "gaming",
         "coding",
-        "cloudflare",
         "photo",
         "music",
         "blog",

maigret/resources/settings.json

@@ -54,15 +54,5 @@
     "graph_report": false,
     "pdf_report": false,
     "html_report": false,
-    "web_interface_port": 5000,
-    "cloudflare_bypass": {
-        "enabled": true,
-        "modules": [
-            {
-                "name": "chrome_webgate",
-                "method": "url_rewrite",
-                "url": "http://localhost:8000/html?url={url}&retries=1"
-            }
-        ]
-    }
+    "web_interface_port": 5000
 }

maigret/submit.py

@@ -110,7 +110,6 @@ class Submitter:
             cookies=self.args.cookie_file,
             # Don't skip errors in submit mode - we need check both false positives/true negatives
             skip_errors=False,
-            cloudflare_bypass=getattr(self.args, 'cloudflare_bypass', False),
         )
         return changes
 
@@ -189,7 +188,6 @@ class Submitter:
         )
         return entered_username if entered_username else supposed_username
 
-    # TODO: replace with checking.py/SimpleAiohttpChecker call
     @staticmethod
     async def get_html_response_to_compare(
         url: str, session: ClientSession = None, redirects=False, headers: Dict = None

maigret/web/app.py

@@ -1,3 +1,4 @@
+# app.py
 from flask import (
     Flask,
     render_template,
@@ -21,7 +22,7 @@ from maigret.report import generate_report_context
 app = Flask(__name__)
 app.secret_key = 'your-secret-key-here'
 
-#add background job tracking
+# Add background job tracking
 background_jobs = {}
 job_results = {}
 
@@ -45,38 +46,15 @@ async def maigret_search(username, options):
     logger = setup_logger(logging.WARNING, 'maigret')
     try:
         db = MaigretDatabase().load_from_path(MAIGRET_DB_FILE)
-        
-        top_sites = int(options.get('top_sites') or 500) 
-        if options.get('all_sites'):
-            top_sites = 999999999  # effectively all
-        
-        tags = options.get('tags', [])
-        site_list= options.get('site_list', [])
-        logger.info(f"Filtering sites by tags: {tags}")
-        
-        sites = db.ranked_sites_dict(
-            top=top_sites,
-            tags=tags,
-            names=site_list,
-            disabled=False,
-            id_type='username'
-        )
-        
-        logger.info(f"Found {len(sites)} sites matching the tag criteria")
+        sites = db.ranked_sites_dict(top=int(options.get('top_sites', 500)))
 
         results = await maigret.search(
             username=username,
             site_dict=sites,
             timeout=int(options.get('timeout', 30)),
             logger=logger,
-            id_type='username',
+            id_type=options.get('id_type', 'username'),
             cookies=COOKIES_FILE if options.get('use_cookies') else None,
-            is_parsing_enabled=(not options.get('disable_extracting', False)),  
-            recursive_search_enabled=(not options.get('disable_recursive_search', False)),
-            check_domains=options.get('with_domains', False),
-            proxy=options.get('proxy', None),
-            tor_proxy=options.get('tor_proxy', None),
-            i2p_proxy=options.get('i2p_proxy', None),
         )
         return results
     except Exception as e:
@@ -89,7 +67,7 @@ async def search_multiple_usernames(usernames, options):
     for username in usernames:
         try:
             search_results = await maigret_search(username.strip(), options)
-            results.append((username.strip(), 'username', search_results))
+            results.append((username.strip(), options['id_type'], search_results))
         except Exception as e:
             logging.error(f"Error searching username {username}: {str(e)}")
     return results
@@ -97,16 +75,20 @@ async def search_multiple_usernames(usernames, options):
 
 def process_search_task(usernames, options, timestamp):
     try:
+        # Setup event loop for async operations
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
 
+        # Run the search
         general_results = loop.run_until_complete(
             search_multiple_usernames(usernames, options)
         )
 
+        # Create session folder
         session_folder = os.path.join(REPORTS_FOLDER, f"search_{timestamp}")
         os.makedirs(session_folder, exist_ok=True)
 
+        # Save the combined graph
         graph_path = os.path.join(session_folder, "combined_graph.html")
         maigret.report.save_graph_report(
             graph_path,
@@ -114,6 +96,7 @@ def process_search_task(usernames, options, timestamp):
             MaigretDatabase().load_from_path(MAIGRET_DB_FILE),
         )
 
+        # Save individual reports
         individual_reports = []
         for username, id_type, results in general_results:
             report_base = os.path.join(session_folder, f"report_{username}")
@@ -170,7 +153,7 @@ def process_search_task(usernames, options, timestamp):
                 }
             )
 
-        # save results and mark job as complete using timestamp as key
+        # Save results and mark job as complete
         job_results[timestamp] = {
             'status': 'completed',
             'session_folder': f"search_{timestamp}",
@@ -178,9 +161,7 @@ def process_search_task(usernames, options, timestamp):
             'usernames': usernames,
             'individual_reports': individual_reports,
         }
-
     except Exception as e:
-        logging.error(f"Error in search task for timestamp {timestamp}: {str(e)}")
         job_results[timestamp] = {'status': 'failed', 'error': str(e)}
     finally:
         background_jobs[timestamp]['completed'] = True
@@ -188,24 +169,9 @@ def process_search_task(usernames, options, timestamp):
 
 @app.route('/')
 def index():
-    #load site data for autocomplete
-    db = MaigretDatabase().load_from_path(MAIGRET_DB_FILE)
-    site_options = []
-    
-    for site in db.sites:
-        #add main site name
-        site_options.append(site.name)
-        #add URL if different from name
-        if site.url_main and site.url_main not in site_options:
-            site_options.append(site.url_main)
-    
-    #sort and deduplicate
-    site_options = sorted(set(site_options))
-    
-    return render_template('index.html', site_options=site_options)
+    return render_template('index.html')
 
 
-# Modified search route
 @app.route('/search', methods=['POST'])
 def search():
     usernames_input = request.form.get('usernames', '').strip()
@@ -220,28 +186,15 @@ def search():
     # Create timestamp for this search session
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
 
-    # Get selected tags - ensure it's a list
-    selected_tags = request.form.getlist('tags')
-    logging.info(f"Selected tags: {selected_tags}")
+    logging.info(f"Starting search for usernames: {usernames}")
 
     options = {
-        'top_sites': request.form.get('top_sites') or '500',
-        'timeout': request.form.get('timeout') or '30',
+        'top_sites': request.form.get('top_sites', '500'),
+        'timeout': request.form.get('timeout', '30'),
+        'id_type': 'username',  # fixed as username
         'use_cookies': 'use_cookies' in request.form,
-        'all_sites': 'all_sites' in request.form,
-        'disable_recursive_search': 'disable_recursive_search' in request.form,
-        'disable_extracting': 'disable_extracting' in request.form,
-        'with_domains': 'with_domains' in request.form,
-        'proxy': request.form.get('proxy', None) or None,
-        'tor_proxy': request.form.get('tor_proxy', None) or None,
-        'i2p_proxy': request.form.get('i2p_proxy', None) or None,
-        'permute': 'permute' in request.form,
-        'tags': selected_tags,  # Pass selected tags as a list
-        'site_list': [s.strip() for s in request.form.get('site', '').split(',') if s.strip()],
     }
 
-    logging.info(f"Starting search for usernames: {usernames} with tags: {selected_tags}")
-
     # Start background job
     background_jobs[timestamp] = {
         'completed': False,
@@ -251,42 +204,46 @@ def search():
     }
     background_jobs[timestamp]['thread'].start()
 
+    logging.info(f"Search job started with timestamp: {timestamp}")
+
+    # Redirect to status page
     return redirect(url_for('status', timestamp=timestamp))
 
+
 @app.route('/status/<timestamp>')
 def status(timestamp):
     logging.info(f"Status check for timestamp: {timestamp}")
 
     # Validate timestamp
     if timestamp not in background_jobs:
-        flash('Invalid search session.', 'danger')
-        logging.error(f"Invalid search session: {timestamp}")
+        flash('Invalid search session', 'danger')
         return redirect(url_for('index'))
 
     # Check if job is completed
     if background_jobs[timestamp]['completed']:
         result = job_results.get(timestamp)
         if not result:
-            flash('No results found for this search session.', 'warning')
-            logging.error(f"No results found for completed session: {timestamp}")
+            flash('No results found for this search session', 'warning')
             return redirect(url_for('index'))
 
         if result['status'] == 'completed':
-            # Note: use the session_folder from the results to redirect
+            # Redirect to results page once done
             return redirect(url_for('results', session_id=result['session_folder']))
         else:
-            error_msg = result.get('error', 'Unknown error occurred.')
+            error_msg = result.get('error', 'Unknown error occurred')
             flash(f'Search failed: {error_msg}', 'danger')
-            logging.error(f"Search failed for session {timestamp}: {error_msg}")
             return redirect(url_for('index'))
 
-    # If job is still running, show a status page
+    # If job is still running, show status page with a simple spinner
     return render_template('status.html', timestamp=timestamp)
 
 
 @app.route('/results/<session_id>')
 def results(session_id):
-    # Find completed results that match this session_folder
+    if not session_id.startswith('search_'):
+        flash('Invalid results session format', 'danger')
+        return redirect(url_for('index'))
+
     result_data = next(
         (
             r
@@ -296,11 +253,6 @@ def results(session_id):
         None,
     )
 
-    if not result_data:
-        flash('No results found for this session ID.', 'danger')
-        logging.error(f"Results for session {session_id} not found in job_results.")
-        return redirect(url_for('index'))
-
     return render_template(
         'results.html',
         usernames=result_data['usernames'],
@@ -313,9 +265,7 @@ def results(session_id):
 @app.route('/reports/<path:filename>')
 def download_report(filename):
     try:
-        file_path = os.path.normpath(os.path.join(REPORTS_FOLDER, filename))
-        if not file_path.startswith(REPORTS_FOLDER):
-            raise Exception("Invalid file path")
+        file_path = os.path.join(REPORTS_FOLDER, filename)
         return send_file(file_path)
     except Exception as e:
         logging.error(f"Error serving file {filename}: {str(e)}")
@@ -327,5 +277,4 @@ if __name__ == '__main__':
         level=logging.INFO,
         format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
     )
-    debug_mode = os.getenv('FLASK_DEBUG', 'False').lower() in ['true', '1', 't']
-    app.run(debug=debug_mode)
+    app.run(debug=True)

maigret/web/templates/base.html

@@ -1,118 +1,44 @@
+<!-- templates/base.html -->
 <!DOCTYPE html>
 <html lang="en" data-bs-theme="dark">
-
 <head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Maigret Web Interface</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
-    <style>
-        body {
-            min-height: 100vh;
-            display: flex;
-            flex-direction: column;
-        }
-
-        .main-container {
-            flex: 1;
-            padding-top: 2rem;
-        }
-
-        .form-container {
-            max-width: auto;
-            margin: auto;
-            padding-bottom: 2rem;
-        }
-
-        [data-bs-theme="dark"] {
-            --bs-body-bg: #212529;
-            --bs-body-color: #dee2e6;
-        }
-
-        .header {
-            padding: 1rem 0;
-            margin-bottom: 2rem;
-            border-bottom: 1px solid var(--bs-border-color);
-        }
-
-        .header-content {
-            display: flex;
-            align-items: center;
-            justify-content: space-between;
-        }
-
-        .logo-container {
-            display: flex;
-            align-items: center;
-            gap: 1rem;
-        }
-
-        .logo {
-            height: 40px;
-            width: auto;
-        }
-
-        .footer {
-            margin-top: auto;
-            padding: 1rem 0;
-            text-align: center;
-            border-top: 1px solid var(--bs-border-color);
-            font-size: 0.9rem;
-        }
-
-        .footer a {
-            color: inherit;
-            text-decoration: none;
-        }
-
-        .footer a:hover {
-            text-decoration: underline;
-        }
-    </style>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>Maigret Web Interface</title>
+<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
+<style>
+    body { 
+        padding-top: 2rem;
+    }
+    .form-container { 
+        max-width: auto; 
+        margin: auto; 
+    }
+    [data-bs-theme="dark"] {
+        --bs-body-bg: #212529;
+        --bs-body-color: #dee2e6;
+    }
+</style>
 </head>
-
 <body>
-    <div class="header">
-        <div class="container">
-            <div class="header-content">
-                <div class="logo-container">
-                    <img src="{{ url_for('static', filename='maigret.png') }}" alt="Maigret Logo" class="logo">
-                    <h1 class="h4 mb-0">Maigret Web Interface</h1>
-                </div>
-                <button class="btn btn-outline-secondary" id="theme-toggle">
-                    Toggle Dark/Light Mode
-                </button>
-            </div>
-        </div>
+<div class="container">
+    <div class="mb-3">
+        <button class="btn btn-outline-secondary" id="theme-toggle">
+            Toggle Dark/Light Mode
+        </button>
     </div>
-
-    <div class="main-container">
-        <div class="container">
-            {% block content %}{% endblock %}
-        </div>
-    </div>
-
-    <footer class="footer">
-        <div class="container">
-            <p class="mb-0">
-                Powered by <a href="https://github.com/soxoj/maigret" target="_blank">Maigret</a> |
-                Licensed under <a href="https://github.com/soxoj/maigret/blob/main/LICENSE" target="_blank">MIT
-                    License</a>
-            </p>
-        </div>
-    </footer>
-
-    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
-    <script>
-        document.getElementById('theme-toggle').addEventListener('click', function () {
-            const html = document.documentElement;
-            if (html.getAttribute('data-bs-theme') === 'dark') {
-                html.setAttribute('data-bs-theme', 'light');
-            } else {
-                html.setAttribute('data-bs-theme', 'dark');
-            }
-        });
-    </script>
+    {% block content %}{% endblock %}
+</div>
+<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+<script>
+document.getElementById('theme-toggle').addEventListener('click', function() {
+    const html = document.documentElement;
+    if (html.getAttribute('data-bs-theme') === 'dark') {
+        html.setAttribute('data-bs-theme', 'light');
+    } else {
+        html.setAttribute('data-bs-theme', 'dark');
+    }
+});
+</script>
 </body>
-
-</html>
+</html>

maigret/web/templates/index.html

@@ -1,383 +1,35 @@
 {% extends "base.html" %}
-
 {% block content %}
-<style>
-    .tag-cloud {
-        display: flex;
-        flex-wrap: wrap;
-        gap: 8px;
-        padding: 15px;
-        border-radius: 8px;
-        background: rgba(0, 0, 0, 0.05);
-        margin-bottom: 20px;
-    }
-
-    .tag {
-        display: inline-block;
-        padding: 5px 10px;
-        border-radius: 15px;
-        background-color: #dc3545;
-        color: white;
-        cursor: pointer;
-        font-size: 14px;
-        transition: all 0.3s ease;
-        user-select: none;
-    }
-
-    .tag.selected {
-        background-color: #28a745;
-    }
-
-    .tag:hover {
-        transform: translateY(-2px);
-        box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2);
-    }
-
-    .hidden-select {
-        display: none !important;
-    }
-
-    .site-input-container {
-        position: relative;
-    }
-
-    .site-input {
-        width: 100%;
-    }
-
-    .selected-sites {
-        display: flex;
-        flex-wrap: wrap;
-        gap: 8px;
-        padding: 10px 0;
-    }
-
-    .selected-site {
-        background-color: #214e7b;
-        padding: 2px 8px;
-        border-radius: 12px;
-        font-size: 14px;
-        display: inline-flex;
-        align-items: center;
-        gap: 5px;
-    }
-
-    .remove-site {
-        cursor: pointer;
-        color: #dc3545;
-        font-weight: bold;
-    }
-
-    .section-header {
-        cursor: pointer;
-        padding: 1rem;
-        background: rgba(255, 255, 255, 0.05);
-        border-radius: 4px;
-        margin-bottom: 0.5rem;
-        display: flex;
-        justify-content: space-between;
-        align-items: center;
-    }
-
-    .section-content {
-        padding: 1rem;
-        display: none;
-    }
-
-    .section-content.show {
-        display: block;
-    }
-
-    .chevron::after {
-        content: '▼';
-        transition: transform 0.2s;
-    }
-
-    .chevron.collapsed::after {
-        transform: rotate(-90deg);
-    }
-
-    .main-search-section {
-        background: rgba(255, 255, 255, 0.03);
-        padding: 2rem;
-        border-radius: 8px;
-        margin-bottom: 2rem;
-    }
-
-    .search-button {
-        width: 100%;
-        padding: 1rem;
-        font-size: 1.2rem;
-        margin-top: 2rem;
-    }
-</style>
-
 <div class="form-container">
+    <h1 class="mb-4">Maigret Web Interface</h1>
+    
     {% if error %}
     <div class="alert alert-danger">{{ error }}</div>
     {% endif %}
-
+    
     <form method="POST" action="{{ url_for('search') }}" class="mb-4">
-        <!-- Main Search Section -->
-        <div class="main-search-section">
-            <div class="mb-4">
-                <label for="usernames" class="form-label h5">Usernames to Search</label>
-                <textarea class="form-control" id="usernames" name="usernames" rows="3" required
-                    placeholder="Enter one or more usernames (separated by spaces or commas)..."></textarea>
-            </div>
-
-            <div class="row align-items-center">
-                <div class="col-md-6">
-                    <label for="top_sites" class="form-label">Number of Sites</label>
-                    <input type="number" class="form-control" id="top_sites" name="top_sites" min="1" max="10000"
-                        placeholder="Default: 500">
-                </div>
-                <div class="col-md-6">
-                    <label for="timeout" class="form-label">Timeout (seconds)</label>
-                    <input type="number" class="form-control" id="timeout" name="timeout" min="1"
-                        placeholder="Default: 30">
-                </div>
-                <div class="col-12 mt-3">
-                    <div class="form-check">
-                        <input type="checkbox" class="form-check-input" id="all_sites" name="all_sites"
-                            onchange="document.getElementById('top_sites').disabled = this.checked;">
-                        <label class="form-check-label" for="all_sites">Search All Sites</label>
-                    </div>
-                </div>
-            </div>
+        <div class="mb-3">
+            <label for="usernames" class="form-label">Usernames to Search</label>
+            <textarea class="form-control" id="usernames" name="usernames" rows="3" required 
+                      placeholder="Enter one or more usernames (separated by spaces or commas)"></textarea>
         </div>
-
-        <!-- Filters Section -->
-        <div class="mb-4">
-            <div class="section-header" onclick="toggleSection('filters')">
-                <h5 class="mb-0">Filters</h5>
-                <span class="chevron"></span>
-            </div>
-            <div id="filters" class="section-content">
-                <div class="mb-3 site-input-container">
-                    <label for="site" class="form-label">Specify Sites (Optional)</label>
-                    <input type="text" class="form-control site-input" id="siteInput"
-                        placeholder="Type to search for sites..." list="siteOptions">
-                    <input type="hidden" id="site" name="site">
-                    <datalist id="siteOptions">
-                        {% for site in site_options %}
-                        <option value="{{ site }}">
-                            {% endfor %}
-                    </datalist>
-                    <div class="selected-sites" id="selectedSites"></div>
-                </div>
-
-                <div class="mb-3">
-                    <label class="form-label">Tags (click to select)</label>
-                    <div class="tag-cloud" id="tagCloud"></div>
-                    <select multiple class="hidden-select" id="tags" name="tags">
-                        <option value="gaming">Gaming</option>
-                        <option value="coding">Coding</option>
-                        <option value="photo">Photo</option>
-                        <option value="music">Music</option>
-                        <option value="blog">Blog</option>
-                        <option value="finance">Finance</option>
-                        <option value="freelance">Freelance</option>
-                        <option value="dating">Dating</option>
-                        <option value="tech">Tech</option>
-                        <option value="forum">Forum</option>
-                        <option value="porn">Porn</option>
-                        <option value="erotic">Erotic</option>
-                        <option value="webcam">Webcam</option>
-                        <option value="video">Video</option>
-                        <option value="movies">Movies</option>
-                        <option value="hacking">Hacking</option>
-                        <option value="art">Art</option>
-                        <option value="discussion">Discussion</option>
-                        <option value="sharing">Sharing</option>
-                        <option value="writing">Writing</option>
-                        <option value="wiki">Wiki</option>
-                        <option value="business">Business</option>
-                        <option value="shopping">Shopping</option>
-                        <option value="sport">Sport</option>
-                        <option value="books">Books</option>
-                        <option value="news">News</option>
-                        <option value="documents">Documents</option>
-                        <option value="travel">Travel</option>
-                        <option value="maps">Maps</option>
-                        <option value="hobby">Hobby</option>
-                        <option value="apps">Apps</option>
-                        <option value="classified">Classified</option>
-                        <option value="career">Career</option>
-                        <option value="geosocial">Geosocial</option>
-                        <option value="streaming">Streaming</option>
-                        <option value="education">Education</option>
-                        <option value="networking">Networking</option>
-                        <option value="torrent">Torrent</option>
-                        <option value="science">Science</option>
-                        <option value="medicine">Medicine</option>
-                        <option value="reading">Reading</option>
-                        <option value="stock">Stock</option>
-                        <option value="messaging">Messaging</option>
-                        <option value="trading">Trading</option>
-                        <option value="links">Links</option>
-                        <option value="fashion">Fashion</option>
-                        <option value="tasks">Tasks</option>
-                        <option value="military">Military</option>
-                        <option value="auto">Auto</option>
-                        <option value="gambling">Gambling</option>
-                        <option value="cybercriminal">Cybercriminal</option>
-                        <option value="review">Review</option>
-                        <option value="bookmarks">Bookmarks</option>
-                        <option value="design">Design</option>
-                        <option value="tor">Tor</option>
-                        <option value="i2p">I2P</option>
-                        <option value="q&a">Q&A</option>
-                        <option value="crypto">Crypto</option>
-                        <option value="ai">AI</option>
-                    </select>
-                </div>
-            </div>
+        
+        <div class="mb-3">
+            <label for="top_sites" class="form-label">Number of Top Sites to Check</label>
+            <input type="number" class="form-control" id="top_sites" name="top_sites" value="500" min="1" max="10000">
         </div>
-
-        <!-- Advanced Options Section -->
-        <div class="mb-4">
-            <div class="section-header" onclick="toggleSection('advanced')">
-                <h5 class="mb-0">Advanced Options</h5>
-                <span class="chevron"></span>
-            </div>
-            <div id="advanced" class="section-content">
-                <div class="mb-3 form-check">
-                    <input type="checkbox" class="form-check-input" id="permute" name="permute">
-                    <label class="form-check-label" for="permute">Enable Username Permutations</label>
-                </div>
-                <div class="mb-3 form-check">
-                    <input type="checkbox" class="form-check-input" id="disable_recursive_search"
-                        name="disable_recursive_search">
-                    <label class="form-check-label" for="disable_recursive_search">Disable Recursive Search</label>
-                </div>
-                <div class="mb-3 form-check">
-                    <input type="checkbox" class="form-check-input" id="disable_extracting" name="disable_extracting">
-                    <label class="form-check-label" for="disable_extracting">Disable Information Extraction</label>
-                </div>
-                <div class="mb-3 form-check">
-                    <input type="checkbox" class="form-check-input" id="with_domains" name="with_domains">
-                    <label class="form-check-label" for="with_domains">Check Domains</label>
-                </div>
-                <div class="mb-3">
-                    <label for="proxy" class="form-label">Proxy URL</label>
-                    <input type="text" class="form-control" id="proxy" name="proxy"
-                        placeholder="e.g., 127.0.0.1:1080">
-                </div>
-                <div class="mb-3">
-                    <label for="tor_proxy" class="form-label">TOR Proxy URL</label>
-                    <input type="text" class="form-control" id="tor_proxy" name="tor_proxy"
-                        placeholder="Default: 127.0.0.1:9050">
-                </div>
-                <div class="mb-3">
-                    <label for="i2p_proxy" class="form-label">I2P Proxy URL</label>
-                    <input type="text" class="form-control" id="i2p_proxy" name="i2p_proxy"
-                        placeholder="Default: 127.0.0.1:4444">
-                </div>
-            </div>
+        
+        <div class="mb-3">
+            <label for="timeout" class="form-label">Timeout (seconds)</label>
+            <input type="number" class="form-control" id="timeout" name="timeout" value="30" min="1" max="120">
         </div>
-
-        <button type="submit" class="btn search-button" style="background-color: rgb(249, 207, 0); color: black;">
-            Start Search
-        </button>
+        
+        <div class="mb-3 form-check">
+            <input type="checkbox" class="form-check-input" id="use_cookies" name="use_cookies">
+            <label class="form-check-label" for="use_cookies">Use Cookies File</label>
+        </div>
+        
+        <button type="submit" class="btn btn-primary">Search</button>
     </form>
 </div>
-
-<script>
-    function toggleSection(sectionId) {
-        const content = document.getElementById(sectionId);
-        const header = content.previousElementSibling;
-        content.classList.toggle('show');
-        header.querySelector('.chevron').classList.toggle('collapsed');
-    }
-
-    document.addEventListener('DOMContentLoaded', function () {
-        // Tag cloud functionality
-        const tagCloud = document.getElementById('tagCloud');
-        const hiddenSelect = document.getElementById('tags');
-        const allTags = Array.from(hiddenSelect.options).map(opt => ({
-            value: opt.value,
-            label: opt.text
-        }));
-
-        allTags.forEach(tag => {
-            const tagElement = document.createElement('span');
-            tagElement.className = 'tag';
-            tagElement.textContent = tag.label;
-            tagElement.dataset.value = tag.value;
-
-            tagElement.addEventListener('click', function () {
-                const isSelected = this.classList.toggle('selected');
-                const option = Array.from(hiddenSelect.options).find(opt => opt.value === tag.value);
-                if (option) {
-                    option.selected = isSelected;
-                }
-            });
-
-            tagCloud.appendChild(tagElement);
-        });
-
-        // Site selection functionality
-        const siteInput = document.getElementById('siteInput');
-        const hiddenInput = document.getElementById('site');
-        const selectedSitesContainer = document.getElementById('selectedSites');
-        let selectedSites = new Set();
-
-        function updateHiddenInput() {
-            hiddenInput.value = Array.from(selectedSites).join(',');
-        }
-
-        function addSite(site) {
-            if (site && !selectedSites.has(site)) {
-                selectedSites.add(site);
-                updateHiddenInput();
-                const siteElement = document.createElement('span');
-                siteElement.className = 'selected-site';
-                siteElement.innerHTML = `${site}<span class="remove-site" data-site="${site}">&times;</span>`;
-                selectedSitesContainer.appendChild(siteElement);
-            }
-        }
-
-        function removeSite(site) {
-            selectedSites.delete(site);
-            updateHiddenInput();
-            const siteElements = selectedSitesContainer.querySelectorAll('.selected-site');
-            siteElements.forEach(el => {
-                if (el.querySelector('.remove-site').dataset.site === site) {
-                    el.remove();
-                }
-            });
-        }
-
-        siteInput.addEventListener('change', function (e) {
-            const value = this.value.trim();
-            if (value) {
-                addSite(value);
-                this.value = '';
-            }
-        });
-
-        selectedSitesContainer.addEventListener('click', function (e) {
-            if (e.target.classList.contains('remove-site')) {
-                removeSite(e.target.dataset.site);
-            }
-        });
-
-        siteInput.addEventListener('paste', function (e) {
-            e.preventDefault();
-            const paste = (e.clipboardData || window.clipboardData).getData('text');
-            const sites = paste.split(',').map(site => site.trim()).filter(site => site);
-            sites.forEach(addSite);
-        });
-
-        const form = document.querySelector('form');
-        form.addEventListener('submit', function (e) {
-            const selectedTags = Array.from(tagCloud.querySelectorAll('.tag.selected'));
-            Array.from(hiddenSelect.options).forEach(opt => {
-                opt.selected = selectedTags.some(tag => tag.dataset.value === opt.value);
-            });
-            updateHiddenInput();
-        });
-    });
-</script>
 {% endblock %}

maigret/web/templates/results.html

@@ -1,156 +1,56 @@
 {% extends "base.html" %}
 {% block content %}
-<style>
-    .tag-badge {
-       background-color: #214e7b;
-       padding: 2px 8px;
-       border-radius: 12px;
-       font-size: 14px;
-       display: inline-flex;
-       align-items: center;
-       gap: 5px;
-       margin: 2px;
-       color: white;
-    }
+<div class="form-container">
+    <h1 class="mb-4">Search Results</h1>
     
-    .profile-list {
-       list-style: none;
-       padding: 0;
-    }
-    
-    .profile-item {
-       margin-bottom: 10px;
-       padding: 10px;
-       display: flex;
-       justify-content: space-between;
-       align-items: center;
-       border-bottom: 1px solid rgba(255, 255, 255, 0.1);
-    }
-    
-    .profile-link {
-       display: flex;
-       align-items: center;
-       gap: 8px;
-    }
-    
-    .favicon {
-       width: 16px;
-       height: 16px;
-    }
-    
-    .tag-container {
-       display: flex;
-       flex-wrap: wrap;
-       gap: 5px;
-       justify-content: flex-end;
-    }
-    
-    .report-container {
-       margin-bottom: 1rem;
-    }
-    
-    .report-header {
-       cursor: pointer;
-       padding: 1rem;
-       background: rgba(255, 255, 255, 0.05);
-       border-radius: 4px;
-       margin-bottom: 0.5rem;
-    }
-    
-    .report-content {
-       display: none;
-    }
-    
-    .report-content.show {
-       display: block;
-    }
-    
-    .chevron::after {
-       content: '▼';
-       margin-left: 8px;
-       transition: transform 0.2s;
-    }
-    
-    .chevron.collapsed::after {
-       transform: rotate(-90deg);
-    }
-    </style>
-
-    <div class="form-container">
-        <h1 class="mb-4">Search Results</h1>
-        <!-- Flash messages -->
-        {% with messages = get_flashed_messages() %}
-            {% if messages %}
-                {% for message in messages %}
+    {% with messages = get_flashed_messages() %}
+        {% if messages %}
+            {% for message in messages %}
                 <div class="alert alert-info">{{ message }}</div>
-                {% endfor %}
-            {% endif %}
-        {% endwith %}
-     
-        <p>The search has completed. <a href="{{ url_for('index')}}">Back to start.</a></p>
-     
-        {% if graph_file %}
-        <h3>Combined Graph</h3>
-        <iframe src="{{ url_for('download_report', filename=graph_file) }}" style="width:100%; height:600px; border:none;"></iframe>
-        {% endif %}
-     
-        <hr>
-     
-        {% if individual_reports %}
-        <h3>Individual Reports</h3>
-        <div class="reports-list">
-            {% for report in individual_reports %}
-            <div class="report-container">
-                <div class="report-header" onclick="toggleReport(this)" data-target="report-{{ loop.index }}">
-                    <h5 class="mb-0 d-flex align-items-center">
-                        <span>{{ report.username }}</span>
-                        <span class="chevron"></span>
-                    </h5>
-                </div>
-                <div id="report-{{ loop.index }}" class="report-content">
-                    <p>
-                        <a href="{{ url_for('download_report', filename=report.csv_file) }}">CSV Report</a> |
-                        <a href="{{ url_for('download_report', filename=report.json_file) }}">JSON Report</a> |
-                        <a href="{{ url_for('download_report', filename=report.pdf_file) }}">PDF Report</a> |
-                        <a href="{{ url_for('download_report', filename=report.html_file) }}">HTML Report</a>
-                    </p>
-                    {% if report.claimed_profiles %}
-                    <strong>Claimed Profiles:</strong>
-                    <ul class="profile-list">
-                        {% for profile in report.claimed_profiles %}
-                        <li class="profile-item">
-                            <div class="profile-link">
-                                <img class="favicon" src="https://www.google.com/s2/favicons?domain={{ profile.url }}" onerror="this.style.display='none'" alt="">
-                                <a href="{{ profile.url }}" target="_blank">{{ profile.site_name }}</a>
-                            </div>
-                            {% if profile.tags %}
-                            <div class="tag-container">
-                                {% for tag in profile.tags %}
-                                <span class="tag-badge">{{ tag }}</span>
-                                {% endfor %}
-                            </div>
-                            {% endif %}
-                        </li>
-                        {% endfor %}
-                    </ul>
-                    {% else %}
-                    <p>No claimed profiles found.</p>
-                    {% endif %}
-                </div>
-            </div>
             {% endfor %}
-        </div>
-        {% else %}
-        <p>No individual reports available.</p>
         {% endif %}
-     </div>
-     
-     <script>
-     function toggleReport(header) {
-        const reportId = header.getAttribute('data-target');
-        const content = document.getElementById(reportId);
-        content.classList.toggle('show');
-        header.querySelector('.chevron').classList.toggle('collapsed');
-     }
-     </script>
-{% endblock %}
+    {% endwith %}
+
+    <p>The search has completed. Below are the results:</p>
+
+    <!-- Display the combined graph if available -->
+    {% if graph_file %}
+    <h3>Combined Graph</h3>
+    <iframe src="{{ url_for('download_report', filename=graph_file) }}" style="width:100%; height:600px; border:none;"></iframe>
+    {% endif %}
+
+    <hr>
+
+    <!-- Display individual reports -->
+    {% if individual_reports %}
+        <h3>Individual Reports</h3>
+        <ul class="list-group">
+        {% for report in individual_reports %}
+            <li class="list-group-item">
+                <h5>{{ report.username }}</h5>
+                <p>
+                    <a href="{{ url_for('download_report', filename=report.csv_file) }}">CSV Report</a> |
+                    <a href="{{ url_for('download_report', filename=report.json_file) }}">JSON Report</a> |
+                    <a href="{{ url_for('download_report', filename=report.pdf_file) }}">PDF Report</a> |
+                    <a href="{{ url_for('download_report', filename=report.html_file) }}">HTML Report</a>
+                </p>
+                {% if report.claimed_profiles %}
+                    <strong>Claimed Profiles:</strong>
+                    <ul>
+                    {% for profile in report.claimed_profiles %}
+                        <li>
+                            <a href="{{ profile.url }}" target="_blank">{{ profile.site_name }}</a> (Tags: {{ profile.tags|join(', ') }})
+                        </li>
+                    {% endfor %}
+                    </ul>
+                {% else %}
+                    <p>No claimed profiles found.</p>
+                {% endif %}
+            </li>
+        {% endfor %}
+        </ul>
+    {% else %}
+        <p>No individual reports available.</p>
+    {% endif %}
+</div>
+{% endblock %}

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
 
 [[package]]
 name = "about-time"
@@ -38,87 +38,87 @@ files = [
 
 [[package]]
 name = "aiohttp"
-version = "3.11.11"
+version = "3.11.10"
 description = "Async http client/server framework (asyncio)"
 optional = false
 python-versions = ">=3.9"
 files = [
-    {file = "aiohttp-3.11.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a60804bff28662cbcf340a4d61598891f12eea3a66af48ecfdc975ceec21e3c8"},
-    {file = "aiohttp-3.11.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4b4fa1cb5f270fb3eab079536b764ad740bb749ce69a94d4ec30ceee1b5940d5"},
-    {file = "aiohttp-3.11.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:731468f555656767cda219ab42e033355fe48c85fbe3ba83a349631541715ba2"},
-    {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb23d8bb86282b342481cad4370ea0853a39e4a32a0042bb52ca6bdde132df43"},
-    {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f047569d655f81cb70ea5be942ee5d4421b6219c3f05d131f64088c73bb0917f"},
-    {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd7659baae9ccf94ae5fe8bfaa2c7bc2e94d24611528395ce88d009107e00c6d"},
-    {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af01e42ad87ae24932138f154105e88da13ce7d202a6de93fafdafb2883a00ef"},
-    {file = "aiohttp-3.11.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5854be2f3e5a729800bac57a8d76af464e160f19676ab6aea74bde18ad19d438"},
-    {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6526e5fb4e14f4bbf30411216780c9967c20c5a55f2f51d3abd6de68320cc2f3"},
-    {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:85992ee30a31835fc482468637b3e5bd085fa8fe9392ba0bdcbdc1ef5e9e3c55"},
-    {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:88a12ad8ccf325a8a5ed80e6d7c3bdc247d66175afedbe104ee2aaca72960d8e"},
-    {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0a6d3fbf2232e3a08c41eca81ae4f1dff3d8f1a30bae415ebe0af2d2458b8a33"},
-    {file = "aiohttp-3.11.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:84a585799c58b795573c7fa9b84c455adf3e1d72f19a2bf498b54a95ae0d194c"},
-    {file = "aiohttp-3.11.11-cp310-cp310-win32.whl", hash = "sha256:bfde76a8f430cf5c5584553adf9926534352251d379dcb266ad2b93c54a29745"},
-    {file = "aiohttp-3.11.11-cp310-cp310-win_amd64.whl", hash = "sha256:0fd82b8e9c383af11d2b26f27a478640b6b83d669440c0a71481f7c865a51da9"},
-    {file = "aiohttp-3.11.11-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ba74ec819177af1ef7f59063c6d35a214a8fde6f987f7661f4f0eecc468a8f76"},
-    {file = "aiohttp-3.11.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4af57160800b7a815f3fe0eba9b46bf28aafc195555f1824555fa2cfab6c1538"},
-    {file = "aiohttp-3.11.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ffa336210cf9cd8ed117011085817d00abe4c08f99968deef0013ea283547204"},
-    {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81b8fe282183e4a3c7a1b72f5ade1094ed1c6345a8f153506d114af5bf8accd9"},
-    {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af41686ccec6a0f2bdc66686dc0f403c41ac2089f80e2214a0f82d001052c03"},
-    {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70d1f9dde0e5dd9e292a6d4d00058737052b01f3532f69c0c65818dac26dc287"},
-    {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:249cc6912405917344192b9f9ea5cd5b139d49e0d2f5c7f70bdfaf6b4dbf3a2e"},
-    {file = "aiohttp-3.11.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0eb98d90b6690827dcc84c246811feeb4e1eea683c0eac6caed7549be9c84665"},
-    {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ec82bf1fda6cecce7f7b915f9196601a1bd1a3079796b76d16ae4cce6d0ef89b"},
-    {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:9fd46ce0845cfe28f108888b3ab17abff84ff695e01e73657eec3f96d72eef34"},
-    {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bd176afcf8f5d2aed50c3647d4925d0db0579d96f75a31e77cbaf67d8a87742d"},
-    {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ec2aa89305006fba9ffb98970db6c8221541be7bee4c1d027421d6f6df7d1ce2"},
-    {file = "aiohttp-3.11.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:92cde43018a2e17d48bb09c79e4d4cb0e236de5063ce897a5e40ac7cb4878773"},
-    {file = "aiohttp-3.11.11-cp311-cp311-win32.whl", hash = "sha256:aba807f9569455cba566882c8938f1a549f205ee43c27b126e5450dc9f83cc62"},
-    {file = "aiohttp-3.11.11-cp311-cp311-win_amd64.whl", hash = "sha256:ae545f31489548c87b0cced5755cfe5a5308d00407000e72c4fa30b19c3220ac"},
-    {file = "aiohttp-3.11.11-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e595c591a48bbc295ebf47cb91aebf9bd32f3ff76749ecf282ea7f9f6bb73886"},
-    {file = "aiohttp-3.11.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3ea1b59dc06396b0b424740a10a0a63974c725b1c64736ff788a3689d36c02d2"},
-    {file = "aiohttp-3.11.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8811f3f098a78ffa16e0ea36dffd577eb031aea797cbdba81be039a4169e242c"},
-    {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7227b87a355ce1f4bf83bfae4399b1f5bb42e0259cb9405824bd03d2f4336a"},
-    {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d40f9da8cabbf295d3a9dae1295c69975b86d941bc20f0a087f0477fa0a66231"},
-    {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ffb3dc385f6bb1568aa974fe65da84723210e5d9707e360e9ecb51f59406cd2e"},
-    {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8f5f7515f3552d899c61202d99dcb17d6e3b0de777900405611cd747cecd1b8"},
-    {file = "aiohttp-3.11.11-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3499c7ffbfd9c6a3d8d6a2b01c26639da7e43d47c7b4f788016226b1e711caa8"},
-    {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8e2bf8029dbf0810c7bfbc3e594b51c4cc9101fbffb583a3923aea184724203c"},
-    {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b6212a60e5c482ef90f2d788835387070a88d52cf6241d3916733c9176d39eab"},
-    {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d119fafe7b634dbfa25a8c597718e69a930e4847f0b88e172744be24515140da"},
-    {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:6fba278063559acc730abf49845d0e9a9e1ba74f85f0ee6efd5803f08b285853"},
-    {file = "aiohttp-3.11.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:92fc484e34b733704ad77210c7957679c5c3877bd1e6b6d74b185e9320cc716e"},
-    {file = "aiohttp-3.11.11-cp312-cp312-win32.whl", hash = "sha256:9f5b3c1ed63c8fa937a920b6c1bec78b74ee09593b3f5b979ab2ae5ef60d7600"},
-    {file = "aiohttp-3.11.11-cp312-cp312-win_amd64.whl", hash = "sha256:1e69966ea6ef0c14ee53ef7a3d68b564cc408121ea56c0caa2dc918c1b2f553d"},
-    {file = "aiohttp-3.11.11-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:541d823548ab69d13d23730a06f97460f4238ad2e5ed966aaf850d7c369782d9"},
-    {file = "aiohttp-3.11.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:929f3ed33743a49ab127c58c3e0a827de0664bfcda566108989a14068f820194"},
-    {file = "aiohttp-3.11.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0882c2820fd0132240edbb4a51eb8ceb6eef8181db9ad5291ab3332e0d71df5f"},
-    {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b63de12e44935d5aca7ed7ed98a255a11e5cb47f83a9fded7a5e41c40277d104"},
-    {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa54f8ef31d23c506910c21163f22b124facb573bff73930735cf9fe38bf7dff"},
-    {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a344d5dc18074e3872777b62f5f7d584ae4344cd6006c17ba12103759d407af3"},
-    {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b7fb429ab1aafa1f48578eb315ca45bd46e9c37de11fe45c7f5f4138091e2f1"},
-    {file = "aiohttp-3.11.11-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c341c7d868750e31961d6d8e60ff040fb9d3d3a46d77fd85e1ab8e76c3e9a5c4"},
-    {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ed9ee95614a71e87f1a70bc81603f6c6760128b140bc4030abe6abaa988f1c3d"},
-    {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:de8d38f1c2810fa2a4f1d995a2e9c70bb8737b18da04ac2afbf3971f65781d87"},
-    {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a9b7371665d4f00deb8f32208c7c5e652059b0fda41cf6dbcac6114a041f1cc2"},
-    {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:620598717fce1b3bd14dd09947ea53e1ad510317c85dda2c9c65b622edc96b12"},
-    {file = "aiohttp-3.11.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bf8d9bfee991d8acc72d060d53860f356e07a50f0e0d09a8dfedea1c554dd0d5"},
-    {file = "aiohttp-3.11.11-cp313-cp313-win32.whl", hash = "sha256:9d73ee3725b7a737ad86c2eac5c57a4a97793d9f442599bea5ec67ac9f4bdc3d"},
-    {file = "aiohttp-3.11.11-cp313-cp313-win_amd64.whl", hash = "sha256:c7a06301c2fb096bdb0bd25fe2011531c1453b9f2c163c8031600ec73af1cc99"},
-    {file = "aiohttp-3.11.11-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3e23419d832d969f659c208557de4a123e30a10d26e1e14b73431d3c13444c2e"},
-    {file = "aiohttp-3.11.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:21fef42317cf02e05d3b09c028712e1d73a9606f02467fd803f7c1f39cc59add"},
-    {file = "aiohttp-3.11.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1f21bb8d0235fc10c09ce1d11ffbd40fc50d3f08a89e4cf3a0c503dc2562247a"},
-    {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1642eceeaa5ab6c9b6dfeaaa626ae314d808188ab23ae196a34c9d97efb68350"},
-    {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2170816e34e10f2fd120f603e951630f8a112e1be3b60963a1f159f5699059a6"},
-    {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8be8508d110d93061197fd2d6a74f7401f73b6d12f8822bbcd6d74f2b55d71b1"},
-    {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4eed954b161e6b9b65f6be446ed448ed3921763cc432053ceb606f89d793927e"},
-    {file = "aiohttp-3.11.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6c9af134da4bc9b3bd3e6a70072509f295d10ee60c697826225b60b9959acdd"},
-    {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:44167fc6a763d534a6908bdb2592269b4bf30a03239bcb1654781adf5e49caf1"},
-    {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:479b8c6ebd12aedfe64563b85920525d05d394b85f166b7873c8bde6da612f9c"},
-    {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:10b4ff0ad793d98605958089fabfa350e8e62bd5d40aa65cdc69d6785859f94e"},
-    {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:b540bd67cfb54e6f0865ceccd9979687210d7ed1a1cc8c01f8e67e2f1e883d28"},
-    {file = "aiohttp-3.11.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1dac54e8ce2ed83b1f6b1a54005c87dfed139cf3f777fdc8afc76e7841101226"},
-    {file = "aiohttp-3.11.11-cp39-cp39-win32.whl", hash = "sha256:568c1236b2fde93b7720f95a890741854c1200fba4a3471ff48b2934d2d93fd3"},
-    {file = "aiohttp-3.11.11-cp39-cp39-win_amd64.whl", hash = "sha256:943a8b052e54dfd6439fd7989f67fc6a7f2138d0a2cf0a7de5f18aa4fe7eb3b1"},
-    {file = "aiohttp-3.11.11.tar.gz", hash = "sha256:bb49c7f1e6ebf3821a42d81d494f538107610c3a705987f53068546b0e90303e"},
+    {file = "aiohttp-3.11.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cbad88a61fa743c5d283ad501b01c153820734118b65aee2bd7dbb735475ce0d"},
+    {file = "aiohttp-3.11.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:80886dac673ceaef499de2f393fc80bb4481a129e6cb29e624a12e3296cc088f"},
+    {file = "aiohttp-3.11.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:61b9bae80ed1f338c42f57c16918853dc51775fb5cb61da70d590de14d8b5fb4"},
+    {file = "aiohttp-3.11.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e2e576caec5c6a6b93f41626c9c02fc87cd91538b81a3670b2e04452a63def6"},
+    {file = "aiohttp-3.11.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02c13415b5732fb6ee7ff64583a5e6ed1c57aa68f17d2bda79c04888dfdc2769"},
+    {file = "aiohttp-3.11.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4cfce37f31f20800a6a6620ce2cdd6737b82e42e06e6e9bd1b36f546feb3c44f"},
+    {file = "aiohttp-3.11.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bbbfff4c679c64e6e23cb213f57cc2c9165c9a65d63717108a644eb5a7398df"},
+    {file = "aiohttp-3.11.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49c7dbbc1a559ae14fc48387a115b7d4bbc84b4a2c3b9299c31696953c2a5219"},
+    {file = "aiohttp-3.11.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:68386d78743e6570f054fe7949d6cb37ef2b672b4d3405ce91fafa996f7d9b4d"},
+    {file = "aiohttp-3.11.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9ef405356ba989fb57f84cac66f7b0260772836191ccefbb987f414bcd2979d9"},
+    {file = "aiohttp-3.11.10-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:5d6958671b296febe7f5f859bea581a21c1d05430d1bbdcf2b393599b1cdce77"},
+    {file = "aiohttp-3.11.10-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:99b7920e7165be5a9e9a3a7f1b680f06f68ff0d0328ff4079e5163990d046767"},
+    {file = "aiohttp-3.11.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0dc49f42422163efb7e6f1df2636fe3db72713f6cd94688e339dbe33fe06d61d"},
+    {file = "aiohttp-3.11.10-cp310-cp310-win32.whl", hash = "sha256:40d1c7a7f750b5648642586ba7206999650208dbe5afbcc5284bcec6579c9b91"},
+    {file = "aiohttp-3.11.10-cp310-cp310-win_amd64.whl", hash = "sha256:68ff6f48b51bd78ea92b31079817aff539f6c8fc80b6b8d6ca347d7c02384e33"},
+    {file = "aiohttp-3.11.10-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:77c4aa15a89847b9891abf97f3d4048f3c2d667e00f8a623c89ad2dccee6771b"},
+    {file = "aiohttp-3.11.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:909af95a72cedbefe5596f0bdf3055740f96c1a4baa0dd11fd74ca4de0b4e3f1"},
+    {file = "aiohttp-3.11.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:386fbe79863eb564e9f3615b959e28b222259da0c48fd1be5929ac838bc65683"},
+    {file = "aiohttp-3.11.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3de34936eb1a647aa919655ff8d38b618e9f6b7f250cc19a57a4bf7fd2062b6d"},
+    {file = "aiohttp-3.11.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0c9527819b29cd2b9f52033e7fb9ff08073df49b4799c89cb5754624ecd98299"},
+    {file = "aiohttp-3.11.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65a96e3e03300b41f261bbfd40dfdbf1c301e87eab7cd61c054b1f2e7c89b9e8"},
+    {file = "aiohttp-3.11.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98f5635f7b74bcd4f6f72fcd85bea2154b323a9f05226a80bc7398d0c90763b0"},
+    {file = "aiohttp-3.11.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:03b6002e20938fc6ee0918c81d9e776bebccc84690e2b03ed132331cca065ee5"},
+    {file = "aiohttp-3.11.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6362cc6c23c08d18ddbf0e8c4d5159b5df74fea1a5278ff4f2c79aed3f4e9f46"},
+    {file = "aiohttp-3.11.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:3691ed7726fef54e928fe26344d930c0c8575bc968c3e239c2e1a04bd8cf7838"},
+    {file = "aiohttp-3.11.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31d5093d3acd02b31c649d3a69bb072d539d4c7659b87caa4f6d2bcf57c2fa2b"},
+    {file = "aiohttp-3.11.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:8b3cf2dc0f0690a33f2d2b2cb15db87a65f1c609f53c37e226f84edb08d10f52"},
+    {file = "aiohttp-3.11.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:fbbaea811a2bba171197b08eea288b9402faa2bab2ba0858eecdd0a4105753a3"},
+    {file = "aiohttp-3.11.10-cp311-cp311-win32.whl", hash = "sha256:4b2c7ac59c5698a7a8207ba72d9e9c15b0fc484a560be0788b31312c2c5504e4"},
+    {file = "aiohttp-3.11.10-cp311-cp311-win_amd64.whl", hash = "sha256:974d3a2cce5fcfa32f06b13ccc8f20c6ad9c51802bb7f829eae8a1845c4019ec"},
+    {file = "aiohttp-3.11.10-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b78f053a7ecfc35f0451d961dacdc671f4bcbc2f58241a7c820e9d82559844cf"},
+    {file = "aiohttp-3.11.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab7485222db0959a87fbe8125e233b5a6f01f4400785b36e8a7878170d8c3138"},
+    {file = "aiohttp-3.11.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cf14627232dfa8730453752e9cdc210966490992234d77ff90bc8dc0dce361d5"},
+    {file = "aiohttp-3.11.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:076bc454a7e6fd646bc82ea7f98296be0b1219b5e3ef8a488afbdd8e81fbac50"},
+    {file = "aiohttp-3.11.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:482cafb7dc886bebeb6c9ba7925e03591a62ab34298ee70d3dd47ba966370d2c"},
+    {file = "aiohttp-3.11.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf3d1a519a324af764a46da4115bdbd566b3c73fb793ffb97f9111dbc684fc4d"},
+    {file = "aiohttp-3.11.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24213ba85a419103e641e55c27dc7ff03536c4873470c2478cce3311ba1eee7b"},
+    {file = "aiohttp-3.11.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b99acd4730ad1b196bfb03ee0803e4adac371ae8efa7e1cbc820200fc5ded109"},
+    {file = "aiohttp-3.11.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:14cdb5a9570be5a04eec2ace174a48ae85833c2aadc86de68f55541f66ce42ab"},
+    {file = "aiohttp-3.11.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:7e97d622cb083e86f18317282084bc9fbf261801b0192c34fe4b1febd9f7ae69"},
+    {file = "aiohttp-3.11.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:012f176945af138abc10c4a48743327a92b4ca9adc7a0e078077cdb5dbab7be0"},
+    {file = "aiohttp-3.11.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44224d815853962f48fe124748227773acd9686eba6dc102578defd6fc99e8d9"},
+    {file = "aiohttp-3.11.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c87bf31b7fdab94ae3adbe4a48e711bfc5f89d21cf4c197e75561def39e223bc"},
+    {file = "aiohttp-3.11.10-cp312-cp312-win32.whl", hash = "sha256:06a8e2ee1cbac16fe61e51e0b0c269400e781b13bcfc33f5425912391a542985"},
+    {file = "aiohttp-3.11.10-cp312-cp312-win_amd64.whl", hash = "sha256:be2b516f56ea883a3e14dda17059716593526e10fb6303189aaf5503937db408"},
+    {file = "aiohttp-3.11.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8cc5203b817b748adccb07f36390feb730b1bc5f56683445bfe924fc270b8816"},
+    {file = "aiohttp-3.11.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ef359ebc6949e3a34c65ce20230fae70920714367c63afd80ea0c2702902ccf"},
+    {file = "aiohttp-3.11.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9bca390cb247dbfaec3c664326e034ef23882c3f3bfa5fbf0b56cad0320aaca5"},
+    {file = "aiohttp-3.11.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:811f23b3351ca532af598405db1093f018edf81368e689d1b508c57dcc6b6a32"},
+    {file = "aiohttp-3.11.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddf5f7d877615f6a1e75971bfa5ac88609af3b74796ff3e06879e8422729fd01"},
+    {file = "aiohttp-3.11.10-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6ab29b8a0beb6f8eaf1e5049252cfe74adbaafd39ba91e10f18caeb0e99ffb34"},
+    {file = "aiohttp-3.11.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c49a76c1038c2dd116fa443eba26bbb8e6c37e924e2513574856de3b6516be99"},
+    {file = "aiohttp-3.11.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f3dc0e330575f5b134918976a645e79adf333c0a1439dcf6899a80776c9ab39"},
+    {file = "aiohttp-3.11.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:efb15a17a12497685304b2d976cb4939e55137df7b09fa53f1b6a023f01fcb4e"},
+    {file = "aiohttp-3.11.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:db1d0b28fcb7f1d35600150c3e4b490775251dea70f894bf15c678fdd84eda6a"},
+    {file = "aiohttp-3.11.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:15fccaf62a4889527539ecb86834084ecf6e9ea70588efde86e8bc775e0e7542"},
+    {file = "aiohttp-3.11.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:593c114a2221444f30749cc5e5f4012488f56bd14de2af44fe23e1e9894a9c60"},
+    {file = "aiohttp-3.11.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7852bbcb4d0d2f0c4d583f40c3bc750ee033265d80598d0f9cb6f372baa6b836"},
+    {file = "aiohttp-3.11.10-cp313-cp313-win32.whl", hash = "sha256:65e55ca7debae8faaffee0ebb4b47a51b4075f01e9b641c31e554fd376595c6c"},
+    {file = "aiohttp-3.11.10-cp313-cp313-win_amd64.whl", hash = "sha256:beb39a6d60a709ae3fb3516a1581777e7e8b76933bb88c8f4420d875bb0267c6"},
+    {file = "aiohttp-3.11.10-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0580f2e12de2138f34debcd5d88894786453a76e98febaf3e8fe5db62d01c9bf"},
+    {file = "aiohttp-3.11.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a55d2ad345684e7c3dd2c20d2f9572e9e1d5446d57200ff630e6ede7612e307f"},
+    {file = "aiohttp-3.11.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:04814571cb72d65a6899db6099e377ed00710bf2e3eafd2985166f2918beaf59"},
+    {file = "aiohttp-3.11.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e44a9a3c053b90c6f09b1bb4edd880959f5328cf63052503f892c41ea786d99f"},
+    {file = "aiohttp-3.11.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:502a1464ccbc800b4b1995b302efaf426e8763fadf185e933c2931df7db9a199"},
+    {file = "aiohttp-3.11.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:613e5169f8ae77b1933e42e418a95931fb4867b2991fc311430b15901ed67079"},
+    {file = "aiohttp-3.11.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cca22a61b7fe45da8fc73c3443150c3608750bbe27641fc7558ec5117b27fdf"},
+    {file = "aiohttp-3.11.10-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:86a5dfcc39309470bd7b68c591d84056d195428d5d2e0b5ccadfbaf25b026ebc"},
+    {file = "aiohttp-3.11.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:77ae58586930ee6b2b6f696c82cf8e78c8016ec4795c53e36718365f6959dc82"},
+    {file = "aiohttp-3.11.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:78153314f26d5abef3239b4a9af20c229c6f3ecb97d4c1c01b22c4f87669820c"},
+    {file = "aiohttp-3.11.10-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:98283b94cc0e11c73acaf1c9698dea80c830ca476492c0fe2622bd931f34b487"},
+    {file = "aiohttp-3.11.10-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:53bf2097e05c2accc166c142a2090e4c6fd86581bde3fd9b2d3f9e93dda66ac1"},
+    {file = "aiohttp-3.11.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c5532f0441fc09c119e1dca18fbc0687e64fbeb45aa4d6a87211ceaee50a74c4"},
+    {file = "aiohttp-3.11.10-cp39-cp39-win32.whl", hash = "sha256:47ad15a65fb41c570cd0ad9a9ff8012489e68176e7207ec7b82a0940dddfd8be"},
+    {file = "aiohttp-3.11.10-cp39-cp39-win_amd64.whl", hash = "sha256:c6b9e6d7e41656d78e37ce754813fa44b455c3d0d0dced2a047def7dc5570b74"},
+    {file = "aiohttp-3.11.10.tar.gz", hash = "sha256:b1fc6b45010a8d0ff9e88f9f2418c6fd408c99c211257334aff41597ebece42e"},
 ]
 
 [package.dependencies]
@@ -248,19 +248,19 @@ files = [
 
 [[package]]
 name = "attrs"
-version = "24.3.0"
+version = "24.2.0"
 description = "Classes Without Boilerplate"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.7"
 files = [
-    {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"},
-    {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"},
+    {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"},
+    {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"},
 ]
 
 [package.extras]
 benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
 cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
 docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
 tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
 tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"]
@@ -345,13 +345,13 @@ files = [
 
 [[package]]
 name = "certifi"
-version = "2024.12.14"
+version = "2024.8.30"
 description = "Python package for providing Mozilla's CA Bundle."
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"},
-    {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"},
+    {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"},
+    {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"},
 ]
 
 [[package]]
@@ -3012,4 +3012,4 @@ propcache = ">=0.2.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "348d13f268fd6c5689f440a956c4b378aef359b29b6a853a8a537633f2574316"
+content-hash = "b25ba6ce790999bbdbd4e6892dd56c84d359684824b49c4f0dd882e1dcbedc0d"

pyinstaller/requirements.txt

@@ -1,5 +1,5 @@
 maigret @ https://github.com/soxoj/maigret/archive/refs/heads/main.zip
 pefile==2023.2.7 # do not bump while pyinstaller is 6.11.1, there is a conflict
-psutil==6.1.1
+psutil==6.1.0
 pyinstaller==6.11.1
 pywin32-ctypes==0.2.3

pyproject.toml

@@ -32,12 +32,12 @@ classifiers = [
 # poetry install --with dev
 python = "^3.10"
 aiodns = "^3.0.0"
-aiohttp = "^3.11.11"
+aiohttp = "^3.11.10"
 aiohttp-socks = "^0.9.1"
 arabic-reshaper = "^3.0.0"
 async-timeout = "^5.0.1"
-attrs = "^24.3.0"
-certifi = "^2024.12.14"
+attrs = "^24.2.0"
+certifi = "^2024.8.30"
 chardet = "^5.0.0"
 colorama = "^0.4.6"
 future = "^1.0.0"

sites.md

@@ -21,7 +21,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://vk.com/) [VK (by id) (https://vk.com/)](https://vk.com/)*: top 50, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://sbongacams.com) [BongaCams (https://sbongacams.com)](https://sbongacams.com)*: top 50, cz, webcam*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.instagram.com/) [Instagram (https://www.instagram.com/)](https://www.instagram.com/)*: top 50, photo*, search is disabled
-1. ![](https://www.google.com/s2/favicons?domain=https://www.twitch.tv/) [Twitch (https://www.twitch.tv/)](https://www.twitch.tv/)*: top 50, cloudflare, streaming, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.twitch.tv/) [Twitch (https://www.twitch.tv/)](https://www.twitch.tv/)*: top 50, streaming, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://yandex.ru/collections/) [YandexCollections API (https://yandex.ru/collections/)](https://yandex.ru/collections/)*: top 50, ru, sharing*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://stackoverflow.com) [StackOverflow (https://stackoverflow.com)](https://stackoverflow.com)*: top 50, coding*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.ebay.com/) [Ebay (https://www.ebay.com/)](https://www.ebay.com/)*: top 50, shopping, us*
@@ -62,7 +62,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://community.adobe.com) [community.adobe.com (https://community.adobe.com)](https://community.adobe.com)*: top 100, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.tradingview.com/) [TradingView (https://www.tradingview.com/)](https://www.tradingview.com/)*: top 100, trading, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.aparat.com) [Aparat (https://www.aparat.com)](https://www.aparat.com)*: top 100, ir, video*
-1. ![](https://www.google.com/s2/favicons?domain=https://chaturbate.com) [ChaturBate (https://chaturbate.com)](https://chaturbate.com)*: top 100, cloudflare, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://chaturbate.com) [ChaturBate (https://chaturbate.com)](https://chaturbate.com)*: top 100, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://medium.com/) [Medium (https://medium.com/)](https://medium.com/)*: top 100, blog, us*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://www.livejasmin.com/) [Livejasmin (https://www.livejasmin.com/)](https://www.livejasmin.com/)*: top 100, us, webcam*
 1. ![](https://www.google.com/s2/favicons?domain=https://pornhub.com/) [Pornhub (https://pornhub.com/)](https://pornhub.com/)*: top 100, porn*
@@ -72,7 +72,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://bleach.fandom.com/ru) [BleachFandom (https://bleach.fandom.com/ru)](https://bleach.fandom.com/ru)*: top 100, ru, wiki*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.fandom.com/) [Fandom (https://www.fandom.com/)](https://www.fandom.com/)*: top 100, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://community.fandom.com) [FandomCommunityCentral (https://community.fandom.com)](https://community.fandom.com)*: top 100, wiki*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.etsy.com/) [Etsy (https://www.etsy.com/)](https://www.etsy.com/)*: top 100, cloudflare, shopping, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.etsy.com/) [Etsy (https://www.etsy.com/)](https://www.etsy.com/)*: top 100, shopping, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.github.com/) [GitHub (https://www.github.com/)](https://www.github.com/)*: top 100, coding*
 1. ![](https://www.google.com/s2/favicons?domain=https://open.spotify.com/) [Spotify (https://open.spotify.com/)](https://open.spotify.com/)*: top 100, music, us*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://www.tiktok.com/) [TikTok (https://www.tiktok.com/)](https://www.tiktok.com/)*: top 100, video*
@@ -80,14 +80,14 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://www.tumblr.com) [Tumblr (https://www.tumblr.com)](https://www.tumblr.com)*: top 500, blog*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.roblox.com/) [Roblox (https://www.roblox.com/)](https://www.roblox.com/)*: top 500, gaming, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://soundcloud.com/) [SoundCloud (https://soundcloud.com/)](https://soundcloud.com/)*: top 500, music*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.udemy.com) [Udemy (https://www.udemy.com)](https://www.udemy.com)*: top 500, cloudflare*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.udemy.com) [Udemy (https://www.udemy.com)](https://www.udemy.com)*: top 500, in*
 1. ![](https://www.google.com/s2/favicons?domain=https://discourse.mozilla.org) [discourse.mozilla.org (https://discourse.mozilla.org)](https://discourse.mozilla.org)*: top 500*
 1. ![](https://www.google.com/s2/favicons?domain=https://linktr.ee) [linktr.ee (https://linktr.ee)](https://linktr.ee)*: top 500, links*
 1. ![](https://www.google.com/s2/favicons?domain=https://xhamster.com) [xHamster (https://xhamster.com)](https://xhamster.com)*: top 500, porn, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.zhihu.com/) [Zhihu (https://www.zhihu.com/)](https://www.zhihu.com/)*: top 500, cn*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://www.blogger.com) [Blogger (by GAIA id) (https://www.blogger.com)](https://www.blogger.com)*: top 500, blog*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.researchgate.net/) [ResearchGate (https://www.researchgate.net/)](https://www.researchgate.net/)*: top 500, in, us*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.freepik.com) [Freepik (https://www.freepik.com)](https://www.freepik.com)*: top 500, art, cloudflare, photo, stock*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.freepik.com) [Freepik (https://www.freepik.com)](https://www.freepik.com)*: top 500, art, photo, stock*
 1. ![](https://www.google.com/s2/favicons?domain=https://vimeo.com) [Vimeo (https://vimeo.com)](https://vimeo.com)*: top 500, video*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.pinterest.com/) [Pinterest (https://www.pinterest.com/)](https://www.pinterest.com/)*: top 500, art, photo, sharing*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.fiverr.com/) [Fiverr (https://www.fiverr.com/)](https://www.fiverr.com/)*: top 500, shopping, us*
@@ -101,9 +101,9 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://wix.com/) [Wix (https://wix.com/)](https://wix.com/)*: top 500, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://slack.com) [Slack (https://slack.com)](https://slack.com)*: top 500, messaging*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.chess.com) [Chess (https://www.chess.com)](https://www.chess.com)*: top 500, gaming, hobby*
-1. ![](https://www.google.com/s2/favicons?domain=https://upwork.com) [upwork.com (https://upwork.com)](https://upwork.com)*: top 500, cloudflare, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://upwork.com) [upwork.com (https://upwork.com)](https://upwork.com)*: top 500, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://archive.org) [Archive.org (https://archive.org)](https://archive.org)*: top 500*, search is disabled
-1. ![](https://www.google.com/s2/favicons?domain=https://www.figma.com/) [Figma (https://www.figma.com/)](https://www.figma.com/)*: top 500, cloudflare, design*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.figma.com/) [Figma (https://www.figma.com/)](https://www.figma.com/)*: top 500, design*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.istockphoto.com) [iStock (https://www.istockphoto.com)](https://www.istockphoto.com)*: top 500, photo, stock*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.scribd.com/) [Scribd (https://www.scribd.com/)](https://www.scribd.com/)*: top 500, reading*
 1. ![](https://www.google.com/s2/favicons?domain=https://opensea.io) [opensea.io (https://opensea.io)](https://opensea.io)*: top 500, us*
@@ -112,7 +112,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=http://www.yelp.com) [Yelp (http://www.yelp.com)](http://www.yelp.com)*: top 500, review*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://www.yelp.com) [Yelp (by id) (https://www.yelp.com)](https://www.yelp.com)*: top 500, review*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.blogger.com/) [Blogger (https://www.blogger.com/)](https://www.blogger.com/)*: top 500, blog*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.patreon.com/) [Patreon (https://www.patreon.com/)](https://www.patreon.com/)*: top 500, cloudflare, finance*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.patreon.com/) [Patreon (https://www.patreon.com/)](https://www.patreon.com/)*: top 500, finance*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.goodreads.com/) [GoodReads (https://www.goodreads.com/)](https://www.goodreads.com/)*: top 500, books, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.op.gg/) [OP.GG [LeagueOfLegends] Brazil (https://www.op.gg/)](https://www.op.gg/)*: top 500, br, gaming*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.op.gg/) [OP.GG [LeagueOfLegends] North America (https://www.op.gg/)](https://www.op.gg/)*: top 500, gaming*
@@ -145,7 +145,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=http://weebly.com) [Weebly (http://weebly.com)](http://weebly.com)*: top 500, business*
 1. ![](https://www.google.com/s2/favicons?domain=https://dating.rambler.ru/) [RamblerDating (https://dating.rambler.ru/)](https://dating.rambler.ru/)*: top 500, dating, ru*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://www.livejournal.com/) [LiveJournal (https://www.livejournal.com/)](https://www.livejournal.com/)*: top 500, blog, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://sourceforge.net/) [SourceForge (https://sourceforge.net/)](https://sourceforge.net/)*: top 500, cloudflare, coding, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://sourceforge.net/) [SourceForge (https://sourceforge.net/)](https://sourceforge.net/)*: top 500, coding, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://genius.com/) [Genius (https://genius.com/)](https://genius.com/)*: top 500, music, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://issuu.com/) [Issuu (https://issuu.com/)](https://issuu.com/)*: top 500, business*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.9gag.com/) [9GAG (https://www.9gag.com/)](https://www.9gag.com/)*: top 500, sharing*
@@ -162,7 +162,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://cyber.harvard.edu) [cyber.harvard.edu (https://cyber.harvard.edu)](https://cyber.harvard.edu)*: top 1K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://duolingo.com/) [Duolingo (https://duolingo.com/)](https://duolingo.com/)*: top 1K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.rottentomatoes.com) [Rottentomatoes (https://www.rottentomatoes.com)](https://www.rottentomatoes.com)*: top 1K, movies, us*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.kickstarter.com) [Kickstarter (https://www.kickstarter.com)](https://www.kickstarter.com)*: top 1K, cloudflare, finance, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.kickstarter.com) [Kickstarter (https://www.kickstarter.com)](https://www.kickstarter.com)*: top 1K, finance, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://forums.ea.com) [forums.ea.com (https://forums.ea.com)](https://forums.ea.com)*: top 1K, forum, gaming, us*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://forums.envato.com) [Envato (https://forums.envato.com)](https://forums.envato.com)*: top 1K, au, forum, in*
 1. ![](https://www.google.com/s2/favicons?domain=https://ultimate-guitar.com/) [Ultimate-Guitar (https://ultimate-guitar.com/)](https://ultimate-guitar.com/)*: top 1K, us*
@@ -181,17 +181,17 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://www.gamespot.com/) [Gamespot (https://www.gamespot.com/)](https://www.gamespot.com/)*: top 1K, gaming, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://note.com/) [note (https://note.com/)](https://note.com/)*: top 1K, jp*
 1. ![](https://www.google.com/s2/favicons?domain=http://bjapi.afreecatv.com) [AfreecaTV (http://bjapi.afreecatv.com)](http://bjapi.afreecatv.com)*: top 1K, streaming*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.redbubble.com/) [Redbubble (https://www.redbubble.com/)](https://www.redbubble.com/)*: top 1K, cloudflare, shopping, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.redbubble.com/) [Redbubble (https://www.redbubble.com/)](https://www.redbubble.com/)*: top 1K, shopping, us*
 1. ![](https://www.google.com/s2/favicons?domain=http://forums.tomsguide.com) [Tom's guide (http://forums.tomsguide.com)](http://forums.tomsguide.com)*: top 1K, forum, tech*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.yumpu.com) [Yumpu (https://www.yumpu.com)](https://www.yumpu.com)*: top 1K, stock*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://community.brave.com) [community.brave.com (https://community.brave.com)](https://community.brave.com)*: top 1K, forum, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://tinder.com/) [Tinder (https://tinder.com/)](https://tinder.com/)*: top 1K, dating, us*
-1. ![](https://www.google.com/s2/favicons?domain=https://community.cloudflare.com/) [CloudflareCommunity (https://community.cloudflare.com/)](https://community.cloudflare.com/)*: top 1K, cloudflare, forum, tech*
+1. ![](https://www.google.com/s2/favicons?domain=https://community.cloudflare.com/) [CloudflareCommunity (https://community.cloudflare.com/)](https://community.cloudflare.com/)*: top 1K, forum, tech*
 1. ![](https://www.google.com/s2/favicons?domain=https://eksisozluk.com) [Eksisozluk (https://eksisozluk.com)](https://eksisozluk.com)*: top 1K, tr*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.allrecipes.com/) [AllRecipes (https://www.allrecipes.com/)](https://www.allrecipes.com/)*: top 1K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://support.t-mobile.com) [T-MobileSupport (https://support.t-mobile.com)](https://support.t-mobile.com)*: top 1K, us*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://www.tinkoff.ru/invest/) [Tinkoff Invest (https://www.tinkoff.ru/invest/)](https://www.tinkoff.ru/invest/)*: top 5K, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.discogs.com/) [Discogs (https://www.discogs.com/)](https://www.discogs.com/)*: top 5K, cloudflare, music, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.discogs.com/) [Discogs (https://www.discogs.com/)](https://www.discogs.com/)*: top 5K, music, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://discuss.python.org/) [DiscussPython (https://discuss.python.org/)](https://discuss.python.org/)*: top 5K, coding, forum, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.nairaland.com/) [Nairaland Forum (https://www.nairaland.com/)](https://www.nairaland.com/)*: top 5K, ng*
 1. ![](https://www.google.com/s2/favicons?domain=https://ru.redtube.com/) [Redtube (https://ru.redtube.com/)](https://ru.redtube.com/)*: top 5K, porn, us*
@@ -3141,7 +3141,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://pubg.op.gg) [OP.GG [PUBG] (https://pubg.op.gg)](https://pubg.op.gg)*: top 100M, gaming*
 1. ![](https://www.google.com/s2/favicons?domain=https://valorant.op.gg) [OP.GG [Valorant] (https://valorant.op.gg)](https://valorant.op.gg)*: top 100M, gaming*
 
-The list was updated at (2026-03-22)
+The list was updated at (2024-12-13)
 ## Statistics
 
 Enabled/total sites: 2684/3137 = 85.56%
@@ -3197,6 +3197,6 @@ Top 20 tags:
 - (15)	`shopping`
 - (13)	`sport`
 - (13)	`business`
-- (13)	`cloudflare` (non-standard)
 - (12)	`movies`
 - (11)	`hobby`
+- (11)	`education`

tests/test_cli.py

@@ -42,20 +42,9 @@ DEFAULT_ARGS: Dict[str, Any] = {
     'use_disabled_sites': False,
     'username': [],
     'verbose': False,
-    'web': None,
+    'web': 5000,
     'with_domains': False,
     'xmind': False,
-    # Mirrors maigret/resources/settings.json (flag --cloudflare-bypass overrides with True)
-    'cloudflare_bypass': {
-        "enabled": True,
-        "modules": [
-            {
-                "name": "chrome_webgate",
-                "method": "url_rewrite",
-                "url": "http://localhost:8000/html?url={url}&retries=1"
-            }
-        ]
-    }
 }
 
 
@@ -71,15 +60,6 @@ def test_args_search_mode(argparser):
         assert getattr(args, arg) == want_args[arg]
 
 
-def test_args_cloudflare_bypass_flag(argparser):
-    args = argparser.parse_args('--cloudflare-bypass username'.split())
-
-    want_args = dict(DEFAULT_ARGS)
-    want_args.update({'username': ['username'], 'cloudflare_bypass': True})
-
-    assert args == Namespace(**want_args)
-
-
 def test_args_search_mode_several_usernames(argparser):
     args = argparser.parse_args('username1 username2'.split())
 

tests/test_executors.py

@@ -8,7 +8,6 @@ from maigret.executors import (
     AsyncioProgressbarExecutor,
     AsyncioProgressbarSemaphoreExecutor,
     AsyncioProgressbarQueueExecutor,
-    AsyncioQueueGeneratorExecutor,
 )
 
 logger = logging.getLogger(__name__)
@@ -36,7 +35,7 @@ async def test_asyncio_progressbar_executor():
     # no guarantees for the results order
     assert sorted(await executor.run(tasks)) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
     assert executor.execution_time > 0.2
-    assert executor.execution_time < 0.6
+    assert executor.execution_time < 0.3
 
 
 @pytest.mark.asyncio
@@ -77,35 +76,3 @@ async def test_asyncio_progressbar_queue_executor():
     assert await executor.run(tasks) == [0, 3, 6, 9, 1, 4, 7, 2, 5, 8]
     assert executor.execution_time > 0.2
     assert executor.execution_time < 0.4
-
-
-@pytest.mark.asyncio
-async def test_asyncio_queue_generator_executor():
-    tasks = [(func, [n], {}) for n in range(10)]
-
-    executor = AsyncioQueueGeneratorExecutor(logger=logger, in_parallel=2)
-    results = [result async for result in executor.run(tasks)]
-    assert results == [0, 1, 3, 2, 4, 6, 7, 5, 9, 8]
-    assert executor.execution_time > 0.5
-    assert executor.execution_time < 0.6
-
-    executor = AsyncioQueueGeneratorExecutor(logger=logger, in_parallel=3)
-    results = [result async for result in executor.run(tasks)]
-    assert results == [0, 3, 1, 4, 6, 2, 7, 9, 5, 8]
-    assert executor.execution_time > 0.4
-    assert executor.execution_time < 0.5
-
-    executor = AsyncioQueueGeneratorExecutor(logger=logger, in_parallel=5)
-    results = [result async for result in executor.run(tasks)]
-    assert results in (
-        [0, 3, 6, 1, 4, 7, 9, 2, 5, 8],
-        [0, 3, 6, 1, 4, 9, 7, 2, 5, 8],
-    )
-    assert executor.execution_time > 0.3
-    assert executor.execution_time < 0.4
-
-    executor = AsyncioQueueGeneratorExecutor(logger=logger, in_parallel=10)
-    results = [result async for result in executor.run(tasks)]
-    assert results == [0, 3, 6, 9, 1, 4, 7, 2, 5, 8]
-    assert executor.execution_time > 0.2
-    assert executor.execution_time < 0.3

tests/test_submit.py

@@ -1,8 +1,9 @@
 import pytest
-from unittest.mock import MagicMock, patch
-from maigret.submit import Submitter
+from unittest.mock import AsyncMock, MagicMock, patch
+from maigret.submit import Submitter, MaigretSite, MaigretEngine
 from aiohttp import ClientSession
 from maigret.sites import MaigretDatabase
+from maigret.settings import Settings
 import logging