mirror/maigret
1
0
Fork 0
mirror of https://github.com/soxoj/maigret.git synced 2026-05-13 18:05:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added several sites

Browse Source
This commit is contained in:
Soxoj
2021-06-01 00:26:37 +03:00
parent f7eae046a1
commit 1a2c7e944a
5 changed files with 288 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
maigret/checking.py
+7 -3
View File
@@ -738,7 +738,7 @@ def timeout_check(value):
async def site_self_check(
site: MaigretSite, logger, semaphore, db: MaigretDatabase, silent=False
site: MaigretSite, logger, semaphore, db: MaigretDatabase, silent=False, tor_proxy=None
):
changes = {
"disabled": False,
@@ -762,6 +762,7 @@ async def site_self_check(
forced=True,
no_progressbar=True,
retries=1,
tor_proxy=tor_proxy,
)
# don't disable entries with other ids types
@@ -771,6 +772,8 @@ async def site_self_check(
changes["disabled"] = True
continue
logger.debug(results_dict)
result = results_dict[site.name]["status"]
site_status = result.status
@@ -809,7 +812,8 @@ async def site_self_check(
async def self_check(
db: MaigretDatabase, site_data: dict, logger, silent=False, max_connections=10
db: MaigretDatabase, site_data: dict, logger, silent=False, max_connections=10,
tor_proxy=None
) -> bool:
sem = asyncio.Semaphore(max_connections)
tasks = []
@@ -821,7 +825,7 @@ async def self_check(
disabled_old_count = disabled_count(all_sites.values())
for _, site in all_sites.items():
check_coro = site_self_check(site, logger, sem, db, silent)
check_coro = site_self_check(site, logger, sem, db, silent, tor_proxy)
future = asyncio.ensure_future(check_coro)
tasks.append(future)
maigret/maigret.py
+2 -1
View File
@@ -529,7 +529,8 @@ async def main():
if args.self_check:
print('Maigret sites database self-checking...')
is_need_update = await self_check(
db, site_data, logger, max_connections=args.connections
db, site_data, logger, max_connections=args.connections,
tor_proxy=args.tor_proxy
)
if is_need_update:
if input('Do you want to save changes permanently? [Yn]\n').lower() in (
maigret/resources/data.json
+243 -6
View File
@@ -3438,6 +3438,7 @@
"usernameUnclaimed": "noonewouldeverusethis7"
},
"Destructoid": {
"disabled": true,
"tags": [
"us"
],
@@ -13317,6 +13318,7 @@
"ru"
],
"checkType": "response_url",
"regexCheck": "^[^-]+$",
"alexaRank": 1499,
"urlMain": "https://studfile.net",
"url": "https://studfile.net/users/{username}/",
@@ -13799,16 +13801,13 @@
},
"TheSimsResource": {
"tags": [
"de",
"gaming",
"it",
"us"
"gaming"
],
"checkType": "status_code",
"checkType": "response_url",
"alexaRank": 12278,
"urlMain": "https://www.thesimsresource.com/",
"url": "https://www.thesimsresource.com/members/{username}/",
"usernameClaimed": "adam",
"usernameClaimed": "DanSimsFantasy",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"TheStudentRoom": {
@@ -15822,6 +15821,32 @@
"usernameClaimed": "yandex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"YandexCollections API (by yandex_public_id)": {
"tags": [
"ru",
"sharing"
],
"type": "yandex_public_id",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36"
},
"errors": {
"action=\"/checkcaptcha\" onsubmit": "Captcha detected, use proxy/vpn"
},
"checkType": "message",
"presenseStrs": [
"public_id"
],
"absenceStrs": [
"cl-not-found-content__title"
],
"alexaRank": 50,
"urlMain": "https://yandex.ru/collections/",
"url": "https://yandex.ru/collections/api/users/{username}/",
"source": "Yandex",
"usernameClaimed": "hx0aur0arkyebkxztq8pr8b4dg",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"YandexMarket": {
"tags": [
"ru"
@@ -27944,6 +27969,218 @@
"tags": [
"video"
]
},
"boards.theforce.net": {
"urlMain": "https://boards.theforce.net",
"engine": "XenForo",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"aussiehomebrewer.com": {
"urlMain": "https://aussiehomebrewer.com",
"engine": "XenForo",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum-ukraina.net": {
"urlMain": "https://forum-ukraina.net",
"engine": "XenForo",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum-history.ru": {
"urlMain": "http://forum-history.ru",
"engine": "vBulletin",
"usernameClaimed": "red",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.vn.ua": {
"urlMain": "http://forum.vn.ua",
"engine": "vBulletin",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.bestflowers.ru": {
"urlMain": "https://forum.bestflowers.ru",
"engine": "XenForo",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.alconar.ru": {
"urlMain": "https://forum.alconar.ru",
"engine": "phpBB/Search",
"usernameClaimed": "admin",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.lancerx.ru": {
"urlMain": "https://forum.lancerx.ru",
"engine": "phpBB/Search",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"mfarmer.ru": {
"urlMain": "http://www.mfarmer.ru",
"engine": "vBulletin",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.league17.ru": {
"urlMain": "https://forum.league17.ru",
"engine": "phpBB/Search",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"krskforum.com": {
"urlMain": "https://krskforum.com",
"engine": "phpBB/Search",
"usernameClaimed": "admin",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.rarib.ag": {
"urlMain": "https://forum.rarib.ag",
"engine": "phpBB",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.oneclickchicks.com": {
"urlMain": "https://forum.oneclickchicks.com",
"engine": "vBulletin",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.trade-print.ru": {
"urlMain": "http://forum.trade-print.ru",
"engine": "vBulletin",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.setcombg.com": {
"urlMain": "https://forum.setcombg.com",
"engine": "vBulletin",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"vw-bus.ru": {
"urlMain": "https://vw-bus.ru",
"engine": "phpBB",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.ya1.ru": {
"urlMain": "https://forum.ya1.ru",
"engine": "vBulletin",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forum.wordreference.com": {
"urlMain": "https://forum.wordreference.com",
"engine": "XenForo",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"forums.zooclub.ru": {
"urlMain": "https://forums.zooclub.ru",
"engine": "vBulletin",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"homebrewtalk.com": {
"urlMain": "https://www.homebrewtalk.com",
"engine": "XenForo",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"navimba.com": {
"urlMain": "https://navimba.com",
"engine": "phpBB/Search",
"usernameClaimed": "admin",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"niva-club.net": {
"urlMain": "https://www.niva-club.net",
"engine": "phpBB/Search",
"usernameClaimed": "red",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"red-forum.com": {
"urlMain": "https://red-forum.com",
"engine": "XenForo",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"scaleforum.ru": {
"urlMain": "http://www.scaleforum.ru",
"engine": "vBulletin",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"sign-forum.ru": {
"urlMain": "https://sign-forum.ru",
"engine": "phpBB/Search",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"rec.poker": {
"urlMain": "https://rec.poker",
"engine": "Wordpress/Author",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"uforum.uz": {
"urlMain": "https://uforum.uz",
"engine": "vBulletin",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"DarkNet Trust": {
"absenceStrs": [
"notify notify-red mx-auto text-center"
],
"presenseStrs": [
"profiles"
],
"protocol": "tor",
"url": "http://dntrustmucd4mwec.onion/u/{username}",
"urlMain": "http://dntrustmucd4mwec.onion",
"usernameClaimed": "cheshirecat82",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"headers": {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5"
},
"tags": [
"tor"
]
},
"forum.freeton.org": {
"urlMain": "https://forum.freeton.org",
"engine": "Discourse",
"usernameClaimed": "maximmuzychenka",
"usernameUnclaimed": "noonewouldeverusethis7",
"alexaRank": 192123,
"tags": [
"finance",
"forum"
]
},
"lyricstranslate.com": {
"absenceStrs": [
"Page not found | Lyrics Translate"
],
"presenseStrs": [
"profileid"
],
"url": "https://lyricstranslate.com/sco/translator/{username}",
"urlMain": "https://lyricstranslate.com",
"usernameClaimed": "charming43",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"alexaRank": 3744,
"tags": [
"music"
]
}
},
"engines": {