mirror/maigret
1
0
Fork 0
mirror of https://github.com/soxoj/maigret.git synced 2026-05-07 06:24:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #39 from soxoj/import-404-specific

Browse Source 
Default engines for specific cases, sites list updated
This commit is contained in:
soxoj
2021-01-21 22:20:38 +03:00
committed by GitHub
parent 4f3120df82 50f1f6d915
commit c2defb20de
4 changed files with 3156 additions and 2807 deletions
Download Patch File Download Diff File Expand all files Collapse all files
maigret/resources/data.json
+1887 -1584
View File
File diff suppressed because it is too large Load Diff
maigret/sites.py
+1
View File
@@ -117,6 +117,7 @@ class MaigretSite:
# remove list items # remove list items
if isinstance(engine_data[k], list) and is_exists: if isinstance(engine_data[k], list) and is_exists:
for f in engine_data[k]: for f in engine_data[k]:
if f in self_copy.__dict__[field]:
self_copy.__dict__[field].remove(f) self_copy.__dict__[field].remove(f)
continue continue
if is_exists: if is_exists:
sites.md
+1241 -1199
View File
File diff suppressed because it is too large Load Diff
utils/update_site_data.py
+3
View File
@@ -24,7 +24,10 @@ RANKS.update({
'50000000': '10M', '50000000': '10M',
}) })
SEMAPHORE = threading.Semaphore(10)
def get_rank(domain_to_query, site, print_errors=True): def get_rank(domain_to_query, site, print_errors=True):
with SEMAPHORE:
#Retrieve ranking data via alexa API #Retrieve ranking data via alexa API
url = f"http://data.alexa.com/data?cli=10&url={domain_to_query}" url = f"http://data.alexa.com/data?cli=10&url={domain_to_query}"
xml_data = requests.get(url).text xml_data = requests.get(url).text