mirror/maigret
1
0
Fork 0
mirror of https://github.com/soxoj/maigret.git synced 2026-05-06 22:19:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sites list updated

Browse Source
This commit is contained in:
Soxoj
2021-01-21 22:18:05 +03:00
parent cb12b8cb06
commit 50f1f6d915
3 changed files with 3096 additions and 2806 deletions
Download Patch File Download Diff File Expand all files Collapse all files
maigret/resources/data.json
+1829 -1584
View File
File diff suppressed because it is too large Load Diff
sites.md
+1241 -1199
View File
File diff suppressed because it is too large Load Diff
utils/update_site_data.py
+26 -23
View File
@@ -24,32 +24,35 @@ RANKS.update({
'50000000': '10M', '50000000': '10M',
}) })
SEMAPHORE = threading.Semaphore(10)
def get_rank(domain_to_query, site, print_errors=True): def get_rank(domain_to_query, site, print_errors=True):
#Retrieve ranking data via alexa API with SEMAPHORE:
url = f"http://data.alexa.com/data?cli=10&url={domain_to_query}" #Retrieve ranking data via alexa API
xml_data = requests.get(url).text url = f"http://data.alexa.com/data?cli=10&url={domain_to_query}"
root = ET.fromstring(xml_data) xml_data = requests.get(url).text
root = ET.fromstring(xml_data)
try: try:
#Get ranking for this site. #Get ranking for this site.
site.alexa_rank = int(root.find('.//REACH').attrib['RANK']) site.alexa_rank = int(root.find('.//REACH').attrib['RANK'])
country = root.find('.//COUNTRY') country = root.find('.//COUNTRY')
if not country is None and country.attrib: if not country is None and country.attrib:
country_code = country.attrib['CODE'] country_code = country.attrib['CODE']
tags = set(site.tags) tags = set(site.tags)
if country_code: if country_code:
tags.add(country_code.lower()) tags.add(country_code.lower())
site.tags = sorted(list(tags)) site.tags = sorted(list(tags))
if site.type != 'username': if site.type != 'username':
site.disabled = False site.disabled = False
except Exception as e: except Exception as e:
if print_errors: if print_errors:
logging.error(e) logging.error(e)
# We did not find the rank for some reason. # We did not find the rank for some reason.
print(f"Error retrieving rank information for '{domain_to_query}'") print(f"Error retrieving rank information for '{domain_to_query}'")
print(f" Returned XML is |{xml_data}|") print(f" Returned XML is |{xml_data}|")
return return
def get_step_rank(rank): def get_step_rank(rank):