Merge pull request #39 from soxoj/import-404-specific

Default engines for specific cases, sites list updated

maigret/sites.py

@@ -117,6 +117,7 @@ class MaigretSite:
             # remove list items
             if isinstance(engine_data[k], list) and is_exists:
                 for f in engine_data[k]:
+                    if f in self_copy.__dict__[field]:
                         self_copy.__dict__[field].remove(f)
                 continue
             if is_exists:

utils/update_site_data.py

@@ -24,7 +24,10 @@ RANKS.update({
     '50000000': '10M',
 })
 
+SEMAPHORE = threading.Semaphore(10)
+
 def get_rank(domain_to_query, site, print_errors=True):
+    with SEMAPHORE:
         #Retrieve ranking data via alexa API
         url = f"http://data.alexa.com/data?cli=10&url={domain_to_query}"
         xml_data = requests.get(url).text