Merge pull request #189 from soxoj/additional-speed-up

Delayed import of requests module, speed check command, reqs updated

Makefile

@@ -18,6 +18,12 @@ lint:
 	@echo 'mypy'
 	mypy ${LINT_FILES}
 
+speed:
+	time python3 ./maigret.py --version
+	python3 -c "import timeit; t = timeit.Timer('import maigret'); print(t.timeit(number = 1000000))"
+	python3 -X importtime -c "import maigret" 2> maigret-import.log
+	python3 -m tuna maigret-import.log
+
 format:
 	@echo 'black'
 	black --skip-string-normalization ${LINT_FILES}

maigret/activation.py

@@ -1,7 +1,6 @@
 from http.cookiejar import MozillaCookieJar
 from http.cookies import Morsel
 
-import requests
 from aiohttp import CookieJar
 
 
@@ -10,6 +9,8 @@ class ParsingActivator:
     def twitter(site, logger, cookies={}):
         headers = dict(site.headers)
         del headers["x-guest-token"]
+        import requests
+
         r = requests.post(site.activation["url"], headers=headers)
         logger.info(r)
         j = r.json()
@@ -21,6 +22,8 @@ class ParsingActivator:
         headers = dict(site.headers)
         if "Authorization" in headers:
             del headers["Authorization"]
+        import requests
+
         r = requests.get(site.activation["url"], headers=headers)
         jwt_token = r.json()["jwt"]
         site.headers["Authorization"] = "jwt " + jwt_token
@@ -30,6 +33,8 @@ class ParsingActivator:
         headers = dict(site.headers)
         if "Authorization" in headers:
             del headers["Authorization"]
+        import requests
+
         r = requests.get(site.activation["url"])
         bearer_token = r.json()["accessToken"]
         site.headers["authorization"] = f"Bearer {bearer_token}"

maigret/checking.py

@@ -63,9 +63,7 @@ class SimpleAiohttpChecker(CheckerBase):
         from aiohttp_socks import ProxyConnector
 
         # make http client session
-        connector = (
-            ProxyConnector.from_url(proxy) if proxy else TCPConnector(ssl=False)
-        )
+        connector = ProxyConnector.from_url(proxy) if proxy else TCPConnector(ssl=False)
         connector.verify_ssl = False
         self.session = ClientSession(
             connector=connector, trust_env=True, cookie_jar=cookie_jar

maigret/sites.py

@@ -5,8 +5,6 @@ import json
 import sys
 from typing import Optional, List, Dict, Any, Tuple
 
-import requests
-
 from .utils import CaseConverter, URLMatcher, is_country_tag
 
 
@@ -359,6 +357,8 @@ class MaigretDatabase:
         if not is_url_valid:
             raise FileNotFoundError(f"Invalid data file URL '{url}'.")
 
+        import requests
+
         try:
             response = requests.get(url=url)
         except Exception as error:

requirements.txt

@@ -1,15 +1,12 @@
 aiodns==3.0.0
-aiohttp==3.7.4
+aiohttp==3.7.4.post0
 aiohttp-socks==0.5.5
 arabic-reshaper==2.1.1
 async-timeout==3.0.1
 attrs==20.3.0
-beautifulsoup4==4.9.3
-bs4==0.0.1
 certifi==2020.12.5
 chardet==3.0.4
 colorama==0.4.4
-python-dateutil==2.8.1
 future==0.18.2
 future-annotations==1.0.0
 html5lib==1.1
@@ -24,7 +21,7 @@ PyPDF2==1.26.0
 PySocks==1.7.1
 python-bidi==0.4.2
 python-socks==1.1.2
-requests>=2.24.0
+requests==2.26.0
 requests-futures==1.0.0
 six==1.15.0
 socid-extractor>=0.0.21