mirror/maigret
1
0
Fork 0
mirror of https://github.com/soxoj/maigret.git synced 2026-05-06 14:08:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improved "submit new site" mode, added tests, fixed top-500 sites (#1952)

Browse Source
This commit is contained in:
Soxoj
2024-12-10 18:02:43 +01:00
committed by GitHub
parent 51ab988e36
commit 81a817a39f
12 changed files with 691 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/conftest.py
+7
View File
@@ -79,6 +79,13 @@ def reports_autoclean():
remove_test_reports()
@pytest.fixture(scope='session')
def settings():
settings = Settings()
settings.load([SETTINGS_FILE])
return settings
@pytest.fixture(scope='session')
def argparser():
settings = Settings()
tests/db.json
+4 -4
View File
@@ -26,7 +26,7 @@
"alexaRank": 1,
"url": "https://play.google.com/store/apps/developer?id={username}",
"urlMain": "https://play.google.com/store",
"usernameClaimed": "OpenAI",
"usernameClaimed": "KONAMI",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"InvalidActive": {
@@ -36,7 +36,7 @@
"alexaRank": 1,
"url": "https://play.google.com/store/apps/dev?id={username}",
"urlMain": "https://play.google.com/store",
"usernameClaimed": "OpenAI",
"usernameClaimed": "KONAMI",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"ValidInactive": {
@@ -46,7 +46,7 @@
"alexaRank": 1,
"url": "https://play.google.com/store/apps/developer?id={username}",
"urlMain": "https://play.google.com/store",
"usernameClaimed": "OpenAI",
"usernameClaimed": "KONAMI",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"InvalidInactive": {
@@ -56,7 +56,7 @@
"alexaRank": 1,
"url": "https://play.google.com/store/apps/dev?id={username}",
"urlMain": "https://play.google.com/store",
"usernameClaimed": "OpenAI",
"usernameClaimed": "KONAMI",
"usernameUnclaimed": "noonewouldeverusethis7"
}
}
tests/test_activation.py
+1
View File
@@ -34,6 +34,7 @@ def test_vimeo_activation(default_db):
assert token1 != token2
@pytest.mark.slow
@pytest.mark.asyncio
async def test_import_aiohttp_cookies():
cookies_filename = 'cookies_test.txt'
tests/test_data.py
+2
View File
@@ -1,8 +1,10 @@
"""Maigret data test functions"""
import pytest
from maigret.utils import is_country_tag
@pytest.mark.slow
def test_tags_validity(default_db):
unknown_tags = set()
tests/test_executors.py
+1
View File
@@ -49,6 +49,7 @@ async def test_asyncio_progressbar_semaphore_executor():
assert executor.execution_time < 0.4
@pytest.mark.slow
@pytest.mark.asyncio
async def test_asyncio_progressbar_queue_executor():
tasks = [(func, [n], {}) for n in range(10)]
tests/test_maigret.py
+1
View File
@@ -84,6 +84,7 @@ def test_maigret_results(test_db):
assert results == RESULTS_EXAMPLE
@pytest.mark.slow
def test_extract_ids_from_url(default_db):
assert default_db.extract_ids_from_url('https://www.reddit.com/user/test') == {
'test': 'username'
tests/test_submit.py
+278
View File
@@ -0,0 +1,278 @@
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from maigret.submit import Submitter, MaigretSite, MaigretEngine
from aiohttp import ClientSession
from maigret.sites import MaigretDatabase
from maigret.settings import Settings
import logging
@pytest.mark.slow
@pytest.mark.asyncio
async def test_detect_known_engine(test_db, local_test_db):
# Use the database fixture instead of mocking
mock_db = test_db
mock_settings = MagicMock()
mock_logger = MagicMock()
mock_args = MagicMock()
mock_args.cookie_file = ""
mock_args.proxy = ""
# Mock the supposed usernames
mock_settings.supposed_usernames = ["adam"]
# Create the Submitter instance
submitter = Submitter(test_db, mock_settings, mock_logger, mock_args)
# Call the method with test URLs
url_exists = "https://devforum.zoom.us/u/adam"
url_mainpage = "https://devforum.zoom.us/"
# Mock extract_username_dialog to return "adam"
submitter.extract_username_dialog = MagicMock(return_value="adam")
sites, resp_text = await submitter.detect_known_engine(
url_exists, url_mainpage, session=None, follow_redirects=False, headers=None
)
# Assertions
assert len(sites) == 2
assert sites[0].name == "devforum.zoom.us"
assert sites[0].url_main == "https://devforum.zoom.us/"
assert sites[0].engine == "Discourse"
assert sites[0].username_claimed == "adam"
assert sites[0].username_unclaimed == "noonewouldeverusethis7"
assert resp_text != ""
await submitter.close()
# Create the Submitter instance without engines
submitter = Submitter(local_test_db, mock_settings, mock_logger, mock_args)
sites, resp_text = await submitter.detect_known_engine(
url_exists, url_mainpage, session=None, follow_redirects=False, headers=None
)
assert len(sites) == 0
await submitter.close()
@pytest.mark.slow
@pytest.mark.asyncio
async def test_check_features_manually_success(settings):
# Setup
db = MaigretDatabase()
logger = logging.getLogger("test_logger")
args = type(
'Args', (object,), {'proxy': None, 'cookie_file': None, 'verbose': False}
)()
submitter = Submitter(db, settings, logger, args)
username = "KONAMI"
url_exists = "https://play.google.com/store/apps/developer?id=KONAMI"
# Execute
presence_list, absence_list, status, random_username = (
await submitter.check_features_manually(
username=username,
url_exists=url_exists,
session=ClientSession(),
follow_redirects=False,
headers=None,
)
)
await submitter.close()
# Assert
assert status == "Found", "Expected status to be 'Found'"
assert isinstance(presence_list, list), "Presence list should be a list"
assert isinstance(absence_list, list), "Absence list should be a list"
assert isinstance(random_username, str), "Random username should be a string"
assert (
random_username != username
), "Random username should not be the same as the input username"
assert sorted(presence_list) == sorted(
[
' title=',
'og:title',
'display: none;',
'4;0',
'main-title',
]
)
assert sorted(absence_list) == sorted(
[
' body {',
' </style>',
'><title>Not Found</title>',
' <style nonce=',
' .rounded {',
]
)
@pytest.mark.slow
@pytest.mark.asyncio
async def test_check_features_manually_success(settings):
# Setup
db = MaigretDatabase()
logger = logging.getLogger("test_logger")
args = type(
'Args', (object,), {'proxy': None, 'cookie_file': None, 'verbose': False}
)()
submitter = Submitter(db, settings, logger, args)
username = "abel"
url_exists = "https://community.cloudflare.com/badges/1/basic?username=abel"
# Execute
presence_list, absence_list, status, random_username = (
await submitter.check_features_manually(
username=username,
url_exists=url_exists,
session=ClientSession(),
follow_redirects=False,
headers=None,
)
)
await submitter.close()
# Assert
assert status == "Cloudflare detected, skipping"
assert presence_list is None
assert absence_list is None
assert random_username != username
@pytest.mark.slow
@pytest.mark.asyncio
async def test_dialog_adds_site_positive(settings):
# Initialize necessary objects
db = MaigretDatabase()
logger = logging.getLogger("test_logger")
logger.setLevel(logging.INFO)
args = type(
'Args',
(object,),
{
'proxy': None,
'cookie_file': None,
'verbose': False,
'db_file': 'test_db.json',
'db': 'test_db.json',
},
)()
submitter = Submitter(db, settings, logger, args)
# Mock user inputs
user_inputs = [
'KONAMI', # Manually input username
'y', # Save the site in the Maigret DB
'GooglePlayStore', # Custom site name
'', # no custom tags
]
with patch('builtins.input', side_effect=user_inputs):
result = await submitter.dialog(
"https://play.google.com/store/apps/developer?id=KONAMI", None
)
await submitter.close()
assert result is True
assert len(db.sites) == 1
site = db.sites[0]
assert site.url_main == "https://play.google.com"
assert site.name == "GooglePlayStore"
assert site.tags == []
assert site.presense_strs != []
assert site.absence_strs != []
assert site.username_claimed == "KONAMI"
assert site.check_type == "message"
@pytest.mark.slow
@pytest.mark.asyncio
async def test_dialog_replace_site(settings, test_db):
# Initialize necessary objects
db = test_db
logger = logging.getLogger("test_logger")
logger.setLevel(logging.DEBUG)
args = type(
'Args',
(object,),
{
'proxy': None,
'cookie_file': None,
'verbose': False,
'db_file': 'test_db.json',
'db': 'test_db.json',
},
)()
assert len(db.sites) == 4
submitter = Submitter(db, settings, logger, args)
# Mock user inputs
user_inputs = [
'y', # Similar sites found, continue
'InvalidActive', # Choose site to replace
'', # Custom headers
'y', # Should we do redirects automatically?
'KONAMI', # Manually input username
'y', # Save the site in the Maigret DB
'', # Custom site name
'', # no custom tags
]
with patch('builtins.input', side_effect=user_inputs):
result = await submitter.dialog(
"https://play.google.com/store/apps/developer?id=KONAMI", None
)
await submitter.close()
assert result is True
assert len(db.sites) == 4
site = db.sites_dict["InvalidActive"]
assert site.name == "InvalidActive"
assert site.url_main == "https://play.google.com"
assert site.tags == ['global', 'us']
assert site.presense_strs != []
assert site.absence_strs != []
assert site.username_claimed == "KONAMI"
assert site.check_type == "message"
@pytest.mark.slow
@pytest.mark.asyncio
async def test_dialog_adds_site_negative(settings):
# Initialize necessary objects
db = MaigretDatabase()
logger = logging.getLogger("test_logger")
logger.setLevel(logging.INFO)
args = type(
'Args',
(object,),
{
'proxy': None,
'cookie_file': None,
'verbose': False,
'db_file': 'test_db.json',
'db': 'test_db.json',
},
)()
submitter = Submitter(db, settings, logger, args)
# Mock user inputs
user_inputs = [
'sokrat', # Manually input username
'y', # Save the site in the Maigret DB
]
with patch('builtins.input', side_effect=user_inputs):
result = await submitter.dialog("https://icq.im/sokrat", None)
await submitter.close()
assert result is False