From 656b9c19ea5ff09cb6e2292327ea354682152db8 Mon Sep 17 00:00:00 2001 From: Soxoj <31013580+soxoj@users.noreply.github.com> Date: Fri, 7 Jul 2023 21:24:20 +0200 Subject: [PATCH] Improved search through UnstoppableDomains (#1040) --- maigret/resources/data.json | 211 ++++-------------------------------- requirements.txt | 2 +- sites.md | 27 ++--- 3 files changed, 31 insertions(+), 209 deletions(-) diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 9bdff6a..ee026fe 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -31315,6 +31315,7 @@ "usernameUnclaimed": "noonewouldeverusethis7", "checkType": "message", "alexaRank": 362185, + "source": "Roblox", "tags": [ "gaming" ] @@ -34027,198 +34028,28 @@ "crypto" ] }, - "ud.me/nft": { - "absenceStrs": [ - "https://unstoppabledomains.com/search?searchTerm" - ], + "UnstoppableDomains": { "presenseStrs": [ - "profile", - "profileData", - "displayName", - "main", - "profileServiceUrl" + "reservedForUserId", + "\"registered\"", + "DomainProduct" ], - "url": "https://ud.me/{username}.nft", - "urlMain": "https://ud.me", - "usernameClaimed": "mlfed", - "usernameUnclaimed": "noonewouldeverusethis7", - "checkType": "message", - "tags": [ - "crypto" - ] - }, - "ud.me/crypto": { - "absenceStrs": [ - "https://unstoppabledomains.com/search?searchTerm" - ], - "presenseStrs": [ - "profile", - "profileData", - "displayName", - "main", - "profileServiceUrl" - ], - "url": "https://ud.me/{username}.crypto", - "urlMain": "https://ud.me", - "usernameClaimed": "unifiprotocol", - "usernameUnclaimed": "noonewouldeverusethis7", - "checkType": "message", - "tags": [ - "crypto" - ] - }, - "ud.me/x": { - "absenceStrs": [ - "https://unstoppabledomains.com/search?searchTerm" - ], - "presenseStrs": [ - "profile", - "profileData", - "displayName", - "main", - "profileServiceUrl" - ], - "url": "https://ud.me/{username}.x", - "urlMain": "https://ud.me", - "usernameClaimed": "mlfed", - "usernameUnclaimed": "noonewouldeverusethis7", - "checkType": "message", - "tags": [ - "crypto" - ] - }, - "ud.me/wallet": { - "absenceStrs": [ - "https://unstoppabledomains.com/search?searchTerm" - ], - "presenseStrs": [ - "profile", - "profileData", - "displayName", - "main", - "profileServiceUrl" - ], - "url": "https://ud.me/{username}.wallet", - "urlMain": "https://ud.me", - "usernameClaimed": "alex", - "usernameUnclaimed": "noonewouldeverusethis7", - "checkType": "message", - "tags": [ - "crypto" - ] - }, - "ud.me/polygon": { - "absenceStrs": [ - "https://unstoppabledomains.com/search?searchTerm" - ], - "presenseStrs": [ - "profile", - "profileData", - "displayName", - "main", - "profileServiceUrl" - ], - "url": "https://ud.me/{username}.polygon", - "urlMain": "https://ud.me", - "usernameClaimed": "alex", - "usernameUnclaimed": "noonewouldeverusethis7", - "checkType": "message", - "tags": [ - "crypto" - ] - }, - "ud.me/dao": { - "absenceStrs": [ - "https://unstoppabledomains.com/search?searchTerm" - ], - "presenseStrs": [ - "profile", - "profileData", - "displayName", - "main", - "profileServiceUrl" - ], - "url": "https://ud.me/{username}.dao", - "urlMain": "https://ud.me", - "usernameClaimed": "unifiprotocol", - "usernameUnclaimed": "noonewouldeverusethis7", - "checkType": "message", - "tags": [ - "crypto" - ] - }, - "ud.me/888": { - "absenceStrs": [ - "https://unstoppabledomains.com/search?searchTerm" - ], - "presenseStrs": [ - "profile", - "profileData", - "displayName", - "main", - "profileServiceUrl" - ], - "url": "https://ud.me/{username}.888", - "urlMain": "https://ud.me", - "usernameClaimed": "mlfed", - "usernameUnclaimed": "noonewouldeverusethis7", - "checkType": "message", - "tags": [ - "crypto" - ] - }, - "ud.me/zil": { - "absenceStrs": [ - "https://unstoppabledomains.com/search?searchTerm" - ], - "presenseStrs": [ - "profile", - "profileData", - "displayName", - "main", - "profileServiceUrl" - ], - "url": "https://ud.me/{username}.zil", - "urlMain": "https://ud.me", - "usernameClaimed": "everlast", - "usernameUnclaimed": "noonewouldeverusethis7", - "checkType": "message", - "tags": [ - "crypto" - ] - }, - "ud.me/blockchain": { - "absenceStrs": [ - "https://unstoppabledomains.com/search?searchTerm" - ], - "presenseStrs": [ - "profile", - "profileData", - "displayName", - "main", - "profileServiceUrl" - ], - "url": "https://ud.me/{username}.blockchain", - "urlMain": "https://ud.me", - "usernameClaimed": "unifiprotocol", - "usernameUnclaimed": "noonewouldeverusethis7", - "checkType": "message", - "tags": [ - "crypto" - ] - }, - "ud.me/bitcoin": { - "absenceStrs": [ - "https://unstoppabledomains.com/search?searchTerm" - ], - "presenseStrs": [ - "profile", - "profileData", - "displayName", - "main", - "profileServiceUrl" - ], - "url": "https://ud.me/{username}.bitcoin", + "headers": { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0", + "Accept": "*/*", + "Accept-Language": "en-US,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br", + "Referer": "https://unstoppabledomains.com/", + "Connection": "keep-alive", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", + "Pragma": "no-cache", + "Cache-Control": "no-cache", + "TE": "trailers" + }, + "urlProbe": "https://unstoppabledomains.com/api/domain/search?q={username}", + "url": "https://ud.me/{username}", "urlMain": "https://ud.me", "usernameClaimed": "mlfed", "usernameUnclaimed": "noonewouldeverusethis7", diff --git a/requirements.txt b/requirements.txt index 0b8b994..dfabbdd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ python-bidi==0.4.2 requests==2.31.0 requests-futures==1.0.0 six==1.16.0 -socid-extractor>=0.0.21 +socid-extractor>=0.0.24 soupsieve==2.3.2.post1 stem==1.8.1 torrequest==0.1.0 diff --git a/sites.md b/sites.md index 6f02e7f..273fd9e 100644 --- a/sites.md +++ b/sites.md @@ -1,5 +1,5 @@ -## List of supported sites (search methods): total 3101 +## List of supported sites (search methods): total 3092 Rank data fetched from Alexa by domains. @@ -3079,16 +3079,7 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=) [bugbounty ()]()*: top 100M, hacking* 1. ![](https://www.google.com/s2/favicons?domain=) [universocraft ()]()*: top 100M, gaming* 1. ![](https://www.google.com/s2/favicons?domain=https://fragment.com) [fragment.com (https://fragment.com)](https://fragment.com)*: top 100M, crypto* -1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/nft (https://ud.me)](https://ud.me)*: top 100M, crypto* -1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/crypto (https://ud.me)](https://ud.me)*: top 100M, crypto* -1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/x (https://ud.me)](https://ud.me)*: top 100M, crypto* -1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/wallet (https://ud.me)](https://ud.me)*: top 100M, crypto* -1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/polygon (https://ud.me)](https://ud.me)*: top 100M, crypto* -1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/dao (https://ud.me)](https://ud.me)*: top 100M, crypto* -1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/888 (https://ud.me)](https://ud.me)*: top 100M, crypto* -1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/zil (https://ud.me)](https://ud.me)*: top 100M, crypto* -1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/blockchain (https://ud.me)](https://ud.me)*: top 100M, crypto* -1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/bitcoin (https://ud.me)](https://ud.me)*: top 100M, crypto* +1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [UnstoppableDomains (https://ud.me)](https://ud.me)*: top 100M, crypto* 1. ![](https://www.google.com/s2/favicons?domain=https://api.edns.domains) [edns.domains/meta (https://api.edns.domains)](https://api.edns.domains)*: top 100M, crypto* 1. ![](https://www.google.com/s2/favicons?domain=https://api.edns.domains) [edns.domains/music (https://api.edns.domains)](https://api.edns.domains)*: top 100M, crypto* 1. ![](https://www.google.com/s2/favicons?domain=https://api.edns.domains) [edns.domains/ass (https://api.edns.domains)](https://api.edns.domains)*: top 100M, crypto* @@ -3105,20 +3096,20 @@ Rank data fetched from Alexa by domains. 1. ![](https://www.google.com/s2/favicons?domain=https://peername.com/) [peername.com/emc (https://peername.com/)](https://peername.com/)*: top 100M, crypto* 1. ![](https://www.google.com/s2/favicons?domain=https://peername.com/) [peername.com/tor (https://peername.com/)](https://peername.com/)*: top 100M, crypto* -The list was updated at (2023-05-13 10:19:52.408695 UTC) +The list was updated at (2023-07-07 19:15:34.376816 UTC) ## Statistics -Enabled/total sites: 2845/3101 = 91.74% +Enabled/total sites: 2836/3092 = 91.72% -Incomplete message checks: 473/2845 = 16.63% (false positive risks) +Incomplete message checks: 474/2836 = 16.71% (false positive risks) -Status code checks: 726/2845 = 25.52% (false positive risks) +Status code checks: 726/2836 = 25.6% (false positive risks) -False positive risk (total): 42.15% +False positive risk (total): 42.31% Top 20 profile URLs: - (796) `{urlMain}/index/8-0-{username} (uCoz)` -- (294) `/{username}` +- (295) `/{username}` - (221) `{urlMain}{urlSubpath}/members/?username={username} (XenForo)` - (160) `/user/{username}` - (133) `{urlMain}{urlSubpath}/member.php?username={username} (vBulletin)` @@ -3157,5 +3148,5 @@ Top 20 tags: - (9) `movies` - (8) `hobby` - (8) `hacking` +- (8) `crypto` - (7) `sport` -- (7) `crypto`