From 5830c9ce72718501ff0d6331f8f2b31acbc7c513 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Tue, 21 Apr 2026 02:02:24 +0200 Subject: [PATCH] Add response time measurement --- maigret/checking.py | 19 ++++++--- maigret/resources/db_meta.json | 2 +- sites.md | 2 +- tests/test_checking.py | 72 ++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/maigret/checking.py b/maigret/checking.py index 389d362..549ea35 100644 --- a/maigret/checking.py +++ b/maigret/checking.py @@ -6,6 +6,7 @@ import random import re import ssl import sys +import time from typing import Any, Dict, List, Optional, Tuple from urllib.parse import quote @@ -334,7 +335,12 @@ def debug_response_logging(url, html_text, status_code, check_error): def process_site_result( - response, query_notify, logger, results_info: QueryResultWrapper, site: MaigretSite + response, + query_notify, + logger, + results_info: QueryResultWrapper, + site: MaigretSite, + response_time: Optional[float] = None, ): if not response: return results_info @@ -362,9 +368,6 @@ def process_site_result( html_text, status_code, check_error = response - # TODO: add elapsed request time counting - response_time = None - if logger.level == logging.DEBUG: debug_response_logging(url, html_text, status_code, check_error) @@ -667,7 +670,10 @@ async def check_site_for_username( print(f"error, no checker for {site.name}") return site.name, default_result + elapsed = 0.0 + t0 = time.perf_counter() response = await checker.check() + elapsed += time.perf_counter() - t0 html_text = response[0] if response and response[0] else "" # Retry once after token-style activation (e.g. Twitter guest token refresh). @@ -700,10 +706,13 @@ async def check_site_for_username( method=checker.method, payload=getattr(checker, 'payload', None), ) + t1 = time.perf_counter() response = await checker.check() + elapsed += time.perf_counter() - t1 response_result = process_site_result( - response, query_notify, logger, default_result, site + response, query_notify, logger, default_result, site, + response_time=elapsed, ) query_notify.update(response_result['status'], site.similar_search) diff --git a/maigret/resources/db_meta.json b/maigret/resources/db_meta.json index f180e17..984e1e1 100644 --- a/maigret/resources/db_meta.json +++ b/maigret/resources/db_meta.json @@ -1,6 +1,6 @@ { "version": 1, - "updated_at": "2026-04-20T22:38:11Z", + "updated_at": "2026-04-21T00:02:26Z", "sites_count": 3141, "min_maigret_version": "0.6.0", "data_sha256": "d93fb2d051328b60126c98fbf02841a6974549f0c8c9220a207a9172b3ee0c90", diff --git a/sites.md b/sites.md index 1c9660b..afb8118 100644 --- a/sites.md +++ b/sites.md @@ -3145,7 +3145,7 @@ Rank data fetched from Majestic Million by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://flarum.es) [flarum.es (https://flarum.es)](https://flarum.es)*: top 100M, es, forum* 1. ![](https://www.google.com/s2/favicons?domain=https://forum.fibra.click) [forum.fibra.click (https://forum.fibra.click)](https://forum.fibra.click)*: top 100M, forum, it* -The list was updated at (2026-04-20) +The list was updated at (2026-04-21) ## Statistics Enabled/total sites: 2562/3141 = 81.57% diff --git a/tests/test_checking.py b/tests/test_checking.py index 98a354f..a2e7022 100644 --- a/tests/test_checking.py +++ b/tests/test_checking.py @@ -1,7 +1,12 @@ +import asyncio +import logging + from mock import Mock import pytest from maigret import search +from maigret.checking import check_site_for_username, process_site_result +from maigret.result import MaigretCheckResult, MaigretCheckStatus def site_result_except(server, username, **kwargs): @@ -67,3 +72,70 @@ async def test_checking_by_message_negative(httpserver, local_test_db): result = await search('unclaimed', site_dict=sites_dict, logger=Mock()) assert result['Message']['status'].is_found() is True + + +def test_process_site_result_threads_response_time(local_test_db): + """process_site_result must thread the response_time kwarg into the result's query_time.""" + site = local_test_db.sites_dict['StatusCode'] + results_info = { + 'username': 'claimed', + 'parsing_enabled': False, + 'url_user': site.url.replace('{username}', 'claimed'), + 'status': None, + 'rank': 0, + 'url_main': site.url_main, + 'ids_data': {}, + } + response = ('body', 200, None) + logger = logging.getLogger('test') + query_notify = Mock() + + out = process_site_result( + response, query_notify, logger, results_info, site, + response_time=1.234, + ) + assert out['status'].query_time == pytest.approx(1.234) + + +def test_process_site_result_defaults_response_time_to_none(local_test_db): + """Omitting response_time keeps query_time as None (backward compatible).""" + site = local_test_db.sites_dict['StatusCode'] + results_info = { + 'username': 'claimed', + 'parsing_enabled': False, + 'url_user': site.url.replace('{username}', 'claimed'), + 'status': None, + 'rank': 0, + 'url_main': site.url_main, + 'ids_data': {}, + } + out = process_site_result( + ('body', 200, None), Mock(), logging.getLogger('test'), results_info, site, + ) + assert out['status'].query_time is None + + +@pytest.mark.slow +@pytest.mark.asyncio +async def test_query_time_populated_from_http_check(httpserver, local_test_db): + """check_site_for_username measures HTTP round-trip and populates query_time.""" + sites_dict = local_test_db.sites_dict + + # Delay the response on the test HTTP server to produce a measurable query_time. + DELAY = 0.25 + + def delayed_handler(request): + import time as _time + _time.sleep(DELAY) + from werkzeug.wrappers import Response + return Response('ok', status=200) + + httpserver.expect_request('/url', query_string='id=claimed').respond_with_handler(delayed_handler) + + result = await search('claimed', site_dict={'StatusCode': sites_dict['StatusCode']}, logger=Mock()) + status = result['StatusCode']['status'] + assert status.is_found() is True + assert isinstance(status.query_time, float) + assert status.query_time >= DELAY + # Upper bound: the measurement should not wildly exceed the server delay. + assert status.query_time < DELAY + 5.0