diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 295160b..382283f 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -2,9 +2,9 @@ name: Linting and testing
on:
push:
- branches: [ main ]
pull_request:
branches: [ main ]
+ types: [opened, synchronize, reopened]
jobs:
build:
diff --git a/.github/workflows/update-site-data.yml b/.github/workflows/update-site-data.yml
index d4689da..a7b3d0e 100644
--- a/.github/workflows/update-site-data.yml
+++ b/.github/workflows/update-site-data.yml
@@ -1,18 +1,34 @@
-name: Update sites rating
+name: Update sites rating and statistics
on:
- push:
- branches: [ main ]
pull_request:
branches: [ main ]
+ types: [opened, synchronize]
jobs:
build:
runs-on: ubuntu-latest
steps:
- - name: checkout repo
- uses: actions/checkout@v2
+ - name: Checkout repository
+ uses: actions/checkout@v2.3.2
+ with:
+ ref: ${{ github.event.pull_request.head.sha }}
+ fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.
+
- name: build application
run: |
pip3 install .
python3 ./utils/update_site_data.py --empty-only
+
+ - name: Commit and push changes
+ run: |
+ git config --global user.name "Maigret autoupdate"
+ git config --global user.email "soxoj@protonmail.com"
+ echo `git name-rev ${{ github.event.pull_request.head.sha }} --name-only`
+ export BRANCH=`git name-rev ${{ github.event.pull_request.head.sha }} --name-only | sed 's/remotes\/origin\///'`
+ echo $BRANCH
+ git remote -v
+ git checkout $BRANCH
+ git add sites.md
+ git commit -m "Updated site list and statistics"
+ git push origin $BRANCH
\ No newline at end of file
diff --git a/docs/source/development.rst b/docs/source/development.rst
index 95ae37a..c17ed63 100644
--- a/docs/source/development.rst
+++ b/docs/source/development.rst
@@ -3,6 +3,37 @@
Development
==============
+Testing
+-------
+
+It is recommended use Python 3.7/3.8 for test due to some conflicts in 3.9.
+
+Install test requirements:
+
+.. code-block:: console
+
+ pip install -r test-requirements.txt
+
+
+Use the following commands to check Maigret:
+
+.. code-block:: console
+
+ # run linter and typing checks
+ # order of checks%
+ # - critical syntax errors or undefined names
+ # - flake checks
+ # - mypy checks
+ make lint
+
+ # run testing with coverage html report
+ # current test coverage is 60%
+ make text
+
+ # open html report
+ open htmlcov/index.html
+
+
How to publish new version of Maigret
-------------------------------------
diff --git a/maigret/checking.py b/maigret/checking.py
index a196cfc..b281287 100644
--- a/maigret/checking.py
+++ b/maigret/checking.py
@@ -132,7 +132,7 @@ class SimpleAiohttpChecker(CheckerBase):
error = CheckError("Unexpected", str(e))
if error == "Invalid proxy response":
- self.logger.debug(e, exc_info=True)
+ self.logger.debug(error, exc_info=True)
return str(html_text), status_code, error
diff --git a/maigret/maigret.py b/maigret/maigret.py
index 224b33c..17700d4 100755
--- a/maigret/maigret.py
+++ b/maigret/maigret.py
@@ -48,7 +48,7 @@ def notify_about_errors(search_results: QueryResultWrapper, query_notify):
for e in errs:
if not errors.is_important(e):
continue
- text = f'Too many errors of type "{e["err"]}" ({e["perc"]}%)'
+ text = f'Too many errors of type "{e["err"]}" ({round(e["perc"],2)}%)'
solution = errors.solution_of(e['err'])
if solution:
text = '. '.join([text, solution.capitalize()])
diff --git a/maigret/resources/data.json b/maigret/resources/data.json
index 8832762..e9e8392 100644
--- a/maigret/resources/data.json
+++ b/maigret/resources/data.json
@@ -1507,6 +1507,7 @@
"forum",
"ru"
],
+ "disabled": true,
"checkType": "message",
"absenceStrs": [
"\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d"
@@ -1947,6 +1948,7 @@
"pk",
"us"
],
+ "disabled": true,
"checkType": "status_code",
"alexaRank": 522588,
"urlMain": "https://bethepro.com",
@@ -1961,7 +1963,7 @@
],
"checkType": "message",
"absenceStrs": [
- "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f \u0441 \u0442\u0430\u043a\u0438\u043c \u0438\u043c\u0435\u043d\u0435\u043c \u043d\u0435 \u0441\u0443\u0449\u0435\u0441\u0442\u0432\u0443\u0435\u0442!"
+ "\u041e\u0448\u0438\u0431\u043a\u0430 / \u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f \u0441 \u0442\u0430\u043a\u0438\u043c \u0438\u043c\u0435\u043d\u0435\u043c \u043d\u0435 \u0441\u0443\u0449\u0435\u0441\u0442\u0432\u0443\u0435\u0442"
],
"alexaRank": 2303903,
"urlMain": "https://bgforum.ru",
@@ -3823,6 +3825,7 @@
"tags": [
"ru"
],
+ "disabled": true,
"checkType": "status_code",
"urlMain": "https://dinsk.su",
"url": "https://dinsk.su/user/{username}",
@@ -4922,6 +4925,7 @@
"tags": [
"ru"
],
+ "disabled": true,
"checkType": "status_code",
"alexaRank": 1225740,
"urlMain": "https://favera.ru",
@@ -5554,11 +5558,14 @@
],
"checkType": "message",
"presenceStrs": [
- "userStatsTitle"
+ "Foursquare "
],
"alexaRank": 3413,
- "urlMain": "https://ru.foursquare.com/",
- "url": "https://ru.foursquare.com/{username}",
+ "urlMain": "https://foursquare.com/",
+ "url": "https://foursquare.com/{username}",
"usernameClaimed": "adam",
"usernameUnclaimed": "noonewouldeverusethis7"
},
@@ -6685,7 +6692,10 @@
],
"checkType": "message",
"absenceStrs": [
- "Page not found."
+ "Page not found"
+ ],
+ "presenseStrs": [
+ "title=\"Gumroad\""
],
"alexaRank": 4728,
"urlMain": "https://www.gumroad.com/",
@@ -8255,6 +8265,7 @@
"tags": [
"coding"
],
+ "disabled": true,
"checkType": "status_code",
"alexaRank": 1859,
"urlMain": "https://leetcode.com/",
@@ -9034,6 +9045,7 @@
"tags": [
"forum"
],
+ "disabled": true,
"checkType": "message",
"absenceStrs": [
"The specified member cannot be found"
@@ -9351,7 +9363,10 @@
],
"checkType": "message",
"absenceStrs": [
- "\u0417\u0434\u0435\u0441\u044c \u043f\u043e\u043a\u0430 \u043d\u0438\u0447\u0435\u0433\u043e \u043d\u0435\u0442"
+ "\u041f\u043e \u0412\u0430\u0448\u0435\u043c\u0443 \u0437\u0430\u043f\u0440\u043e\u0441\u0443 \u043d\u0438\u0447\u0435\u0433\u043e \u043d\u0435 \u043d\u0430\u0439\u0434\u0435\u043d\u043e"
+ ],
+ "presenseStrs": [
+ "\u041b\u044e\u0434\u0438"
],
"alexaRank": 6409,
"urlMain": "https://mirtesen.ru",
@@ -9567,12 +9582,9 @@
},
"Movescount": {
"tags": [
- "es",
- "in",
- "pk",
- "ru",
- "us"
+ "maps"
],
+ "disabled": true,
"checkType": "message",
"absenceStrs": [
"error=4&"
@@ -10206,12 +10218,15 @@
},
"Netvibes": {
"tags": [
- "de",
- "fr",
- "jp"
+ "business",
+ "fr"
],
+ "disabled": true,
"checkType": "status_code",
"alexaRank": 5730,
+ "headers": {
+ "User-Agent": "curl/7.64.1"
+ },
"urlMain": "https://www.netvibes.com",
"url": "https://www.netvibes.com/{username}#General",
"usernameClaimed": "blue",
@@ -10657,10 +10672,7 @@
"tags": [
"ru"
],
- "checkType": "message",
- "absenceStrs": [
- "404 - Not Found"
- ],
+ "checkType": "status_code",
"alexaRank": 25200,
"urlMain": "https://overclockers.ru",
"url": "https://overclockers.ru/cpubase/user/{username}",
@@ -11205,7 +11217,11 @@
"checkType": "message",
"absenceStrs": [
"Hmm, it seems that you've come across an invalid username",
- "404 Not Found"
+ "404 Not Found",
+ "Member Not Found"
+ ],
+ "presenseStrs": [
+ "profile on Planet Minecraft to see their public Minecraft community activity"
],
"alexaRank": 9050,
"urlMain": "https://www.planetminecraft.com",
@@ -13342,7 +13358,13 @@
"tags": [
"music"
],
- "checkType": "status_code",
+ "checkType": "message",
+ "presenseStrs": [
+ "Profile: "
+ ],
+ "absenceStrs": [
+ "Smule | Page Not Found (404)"
+ ],
"alexaRank": 11742,
"urlMain": "https://www.smule.com/",
"url": "https://www.smule.com/{username}",
@@ -13366,16 +13388,19 @@
},
"Snooth": {
"tags": [
- "in"
+ "news"
],
"checkType": "message",
"absenceStrs": [
- "Profiles on Snooth"
+ "Page not found"
+ ],
+ "presenseStrs": [
+ "content=\"https://www.snooth.com/author/"
],
"alexaRank": 4088489,
"urlMain": "https://www.snooth.com/",
- "url": "https://www.snooth.com/profiles/{username}/",
- "usernameClaimed": "GregT",
+ "url": "https://www.snooth.com/author/{username}/",
+ "usernameClaimed": "joshua",
"usernameUnclaimed": "noonewouldeverusethis7"
},
"SocialLibremOne": {
@@ -13608,7 +13633,7 @@
"us"
],
"headers": {
- "authorization": "Bearer BQBLyPSs0y1FzkCHjMKD70dV9w93fNraURmQAzR9xTuDyzbQWBAVnGecE1lELz54ZjRnJI8jEMlZ4tuhC8s"
+ "authorization": "Bearer BQBFTijjpshGAhX7n9-sO46wb8zJIkhu6TT3Ss7b-0V1dw_jXZhcff1agUpqRgbhznOG8pSIRlHtJAtd2TU"
},
"errors": {
"Spotify is currently not available in your country.": "Access denied in your country, use proxy/vpn"
@@ -15464,7 +15489,7 @@
"video"
],
"headers": {
- "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2NDQzOTg0MDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.RbupVlEMlatBOdlII3XdUAQkjuCRa1BAI_7ssJx9B3o"
+ "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2NDU4Nzk3NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.T8E8Vrx0sO-9WP4RdZGNQZw2EB1hYTIXbIguXIZbfNQ"
},
"activation": {
"url": "https://vimeo.com/_rv/viewer",
@@ -16013,7 +16038,8 @@
],
"checkType": "message",
"absenceStrs": [
- "is not registered"
+ "is not registered",
+ "Wikipedia does not have a"
],
"alexaRank": 12,
"urlMain": "https://www.wikipedia.org/",
@@ -17069,12 +17095,7 @@
"br",
"us"
],
- "checkType": "message",
- "presenseStrs": "common.SummonerHistory.Favorite",
- "absenceStrs": [
- "SummonerNotFoundLayout"
- ],
- "alexaRank": 331,
+ "engine": "op.gg",
"urlMain": "https://br.op.gg/",
"url": "https://br.op.gg/summoner/userName={username}",
"usernameClaimed": "adam",
@@ -17615,12 +17636,7 @@
"gaming",
"us"
],
- "checkType": "message",
- "presenseStrs": "common.SummonerHistory.Favorite",
- "absenceStrs": [
- "SummonerNotFoundLayout"
- ],
- "alexaRank": 331,
+ "engine": "op.gg",
"urlMain": "https://eune.op.gg/",
"url": "https://eune.op.gg/summoner/userName={username}",
"usernameClaimed": "adam",
@@ -17631,12 +17647,7 @@
"gaming",
"us"
],
- "checkType": "message",
- "presenseStrs": "common.SummonerHistory.Favorite",
- "absenceStrs": [
- "SummonerNotFoundLayout"
- ],
- "alexaRank": 331,
+ "engine": "op.gg",
"urlMain": "https://euw.op.gg/",
"url": "https://euw.op.gg/summoner/userName={username}",
"usernameClaimed": "blue",
@@ -18575,12 +18586,7 @@
"tags": [
"us"
],
- "checkType": "message",
- "presenseStrs": "common.SummonerHistory.Favorite",
- "absenceStrs": [
- "SummonerNotFoundLayout"
- ],
- "alexaRank": 331,
+ "engine": "op.gg",
"urlMain": "https://lan.op.gg/",
"url": "https://lan.op.gg/summoner/userName={username}",
"usernameClaimed": "adam",
@@ -18591,12 +18597,7 @@
"gaming",
"us"
],
- "checkType": "message",
- "presenseStrs": "common.SummonerHistory.Favorite",
- "absenceStrs": [
- "SummonerNotFoundLayout"
- ],
- "alexaRank": 331,
+ "engine": "op.gg",
"urlMain": "https://las.op.gg/",
"url": "https://las.op.gg/summoner/userName={username}",
"usernameClaimed": "adam",
@@ -18641,6 +18642,7 @@
"tags": [
"ru"
],
+ "disabled": true,
"checkType": "status_code",
"alexaRank": 6054365,
"urlMain": "http://linuxmint.info",
@@ -18980,12 +18982,7 @@
"tags": [
"gaming"
],
- "checkType": "message",
- "presenseStrs": "common.SummonerHistory.Favorite",
- "absenceStrs": [
- "SummonerNotFoundLayout"
- ],
- "alexaRank": 331,
+ "engine": "op.gg",
"urlMain": "https://na.op.gg/",
"url": "https://na.op.gg/summoner/userName={username}",
"usernameClaimed": "adam",
@@ -19165,12 +19162,7 @@
"gaming",
"us"
],
- "checkType": "message",
- "presenseStrs": "common.SummonerHistory.Favorite",
- "absenceStrs": [
- "SummonerNotFoundLayout"
- ],
- "alexaRank": 331,
+ "engine": "op.gg",
"urlMain": "https://oce.op.gg/",
"url": "https://oce.op.gg/summoner/userName={username}",
"usernameClaimed": "adam",
@@ -19637,12 +19629,7 @@
"ru",
"us"
],
- "checkType": "message",
- "presenseStrs": "common.SummonerHistory.Favorite",
- "absenceStrs": [
- "SummonerNotFoundLayout"
- ],
- "alexaRank": 331,
+ "engine": "op.gg",
"urlMain": "https://ru.op.gg/",
"url": "https://ru.op.gg/summoner/userName={username}",
"usernameClaimed": "adam",
@@ -20189,12 +20176,7 @@
"tr",
"us"
],
- "checkType": "message",
- "presenseStrs": "common.SummonerHistory.Favorite",
- "absenceStrs": [
- "SummonerNotFoundLayout"
- ],
- "alexaRank": 331,
+ "engine": "op.gg",
"urlMain": "https://tr.op.gg/",
"url": "https://tr.op.gg/summoner/userName={username}",
"usernameClaimed": "adam",
@@ -25003,23 +24985,35 @@
},
"thoughts.com": {
"tags": [
- "in"
+ "blog"
+ ],
+ "checkType": "message",
+ "absenceStrs": [
+ "<title>Start a Blog"
+ ],
+ "presenseStrs": [
+ "– Thoughts.com"
],
- "engine": "engine404get",
"urlMain": "http://thoughts.com",
- "url": "http://thoughts.com/profile/{username}",
+ "url": "http://thoughts.com/members/{username}",
"usernameUnclaimed": "noonewouldeverusethis7",
- "usernameClaimed": "red"
+ "usernameClaimed": "alicia12"
},
"hackernoon.com": {
"tags": [
- "in",
- "us"
+ "us",
+ "news"
+ ],
+ "checkType": "message",
+ "absenceStrs": [
+ "HackerNoon"
+ ],
+ "presenseStrs": [
+ " | HackerNoon"
],
- "engine": "engine404message",
"urlMain": "https://hackernoon.com",
"url": "https://hackernoon.com/u/{username}",
- "usernameUnclaimed": "noonewouldeverusethis7",
+ "usernameUnclaimed": "noonewouldeverusethis71",
"usernameClaimed": "god"
},
"Intigriti": {
@@ -25352,7 +25346,7 @@
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7"
},
- "hashnode.com": {
+ "hashnode": {
"tags": [
"in"
],
@@ -25365,7 +25359,7 @@
" name="
],
"absenceStrs": [
- ">noonewouldeverusethis7 » \u0420\u0435\u0441\u0443\u0440\u0441\u044b \u0434\u043b\u044f \u0424\u043e\u0442\u043e\u0448\u043e\u043f\u0430"
+ "