Merge pull request #385 from soxoj/v0.4.2

Bump to 0.4.2

.github/workflows/python-package.yml

@@ -1,13 +1,10 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+name: Linting and testing
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
-name: Python package
 
 on:
   push:
-    branches: [ main ]
   pull_request:
     branches: [ main ]
+    types: [opened, synchronize, reopened]
 
 jobs:
   build:

.github/workflows/python-publish.yml

@@ -1,6 +1,3 @@
-# This workflow will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
 name: Upload Python Package
 
 on:

.github/workflows/update-site-data.yml

@@ -0,0 +1,34 @@
+name: Update sites rating and statistics
+
+on:
+  pull_request:
+    branches: [ main ]
+    types: [opened, synchronize]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2.3.2
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository.
+
+    - name: build application
+      run: |
+        pip3 install .
+        python3 ./utils/update_site_data.py --empty-only
+
+    - name: Commit and push changes
+      run: |
+        git config --global user.name "Maigret autoupdate"
+        git config --global user.email "soxoj@protonmail.com"
+        echo `git name-rev ${{ github.event.pull_request.head.sha }} --name-only`
+        export BRANCH=`git name-rev ${{ github.event.pull_request.head.sha }} --name-only | sed 's/remotes\/origin\///'`
+        echo $BRANCH
+        git remote -v
+        git checkout $BRANCH
+        git add sites.md
+        git commit -m "Updated site list and statistics"
+        git push origin $BRANCH

CHANGELOG.md

@@ -2,6 +2,43 @@
 
 ## [Unreleased]
 
+## [0.4.2] - 2022-03-07
+* [ImgBot] Optimize images by @imgbot in https://github.com/soxoj/maigret/pull/319
+* Bump pytest-asyncio from 0.17.0 to 0.17.1 by @dependabot in https://github.com/soxoj/maigret/pull/321
+* Bump pytest-asyncio from 0.17.1 to 0.17.2 by @dependabot in https://github.com/soxoj/maigret/pull/323
+* Disabled Ruboard by @soxoj in https://github.com/soxoj/maigret/pull/327
+* Disable kinooh, sites list update workflow added by @soxoj in https://github.com/soxoj/maigret/pull/329
+* Bump multidict from 5.2.0 to 6.0.1 by @dependabot in https://github.com/soxoj/maigret/pull/332
+* Bump multidict from 6.0.1 to 6.0.2 by @dependabot in https://github.com/soxoj/maigret/pull/333
+* Bump pytest-httpserver from 1.0.3 to 1.0.4 by @dependabot in https://github.com/soxoj/maigret/pull/334
+* Bump pytest from 6.2.5 to 7.0.0 by @dependabot in https://github.com/soxoj/maigret/pull/339
+* Bump pytest-asyncio from 0.17.2 to 0.18.0 by @dependabot in https://github.com/soxoj/maigret/pull/340
+* Bump pytest-asyncio from 0.18.0 to 0.18.1 by @dependabot in https://github.com/soxoj/maigret/pull/343
+* Bump pytest from 7.0.0 to 7.0.1 by @dependabot in https://github.com/soxoj/maigret/pull/345
+* Bump typing-extensions from 4.0.1 to 4.1.1 by @dependabot in https://github.com/soxoj/maigret/pull/346
+* Bump lxml from 4.7.1 to 4.8.0 by @dependabot in https://github.com/soxoj/maigret/pull/350
+* Pin reportlab version by @cyb3rk0tik in https://github.com/soxoj/maigret/pull/351
+* Fix reportlab not only for testing by @cyb3rk0tik in https://github.com/soxoj/maigret/pull/352
+* Added some scripts by @soxoj in https://github.com/soxoj/maigret/pull/355
+* Added package publishing instruction by @soxoj in https://github.com/soxoj/maigret/pull/356
+* Added DB statistics autoupdate and write to sites.md by @soxoj in https://github.com/soxoj/maigret/pull/357
+* CI autoupdate by @soxoj in https://github.com/soxoj/maigret/pull/359
+* Op.gg fixes by @soxoj in https://github.com/soxoj/maigret/pull/363
+* Wikipedia fix by @soxoj in https://github.com/soxoj/maigret/pull/365
+* Disabled Netvibes and LeetCode by @soxoj in https://github.com/soxoj/maigret/pull/366
+* Fixed several false positives, improved statistics info by @soxoj in https://github.com/soxoj/maigret/pull/368
+* Fix false positives  by @soxoj in https://github.com/soxoj/maigret/pull/370
+* Fixed the rest of false positives for now by @soxoj in https://github.com/soxoj/maigret/pull/371
+* Fix false positive and CI by @soxoj in https://github.com/soxoj/maigret/pull/372
+* Added new sites to data.json by @kustermariocoding in https://github.com/soxoj/maigret/pull/375
+* Fixed issue with str alexaRank by @soxoj in https://github.com/soxoj/maigret/pull/382
+* Bump tqdm from 4.62.3 to 4.63.0 by @dependabot in https://github.com/soxoj/maigret/pull/374
+* Bump pytest-asyncio from 0.18.1 to 0.18.2 by @dependabot in https://github.com/soxoj/maigret/pull/380
+* @imgbot made their first contribution in https://github.com/soxoj/maigret/pull/319
+* @kustermariocoding made their first contribution in https://github.com/soxoj/maigret/pull/375
+
+**Full Changelog**: https://github.com/soxoj/maigret/compare/v0.4.1...v0.4.2
+
 ## [0.4.1] - 2022-01-15
 * Added dozen of sites, improved submit mode by @soxoj in https://github.com/soxoj/maigret/pull/288
 * Bump requests from 2.26.0 to 2.27.0 by @dependabot in https://github.com/soxoj/maigret/pull/292

README.md

@@ -101,7 +101,7 @@ maigret user --tags photo,dating
 maigret user1 user2 user3 -a
 ```
 
-Use `maigret --help` to get full options description. Also options are documented in [the Maigret Wiki](https://github.com/soxoj/maigret/wiki/Command-line-options).
+Use `maigret --help` to get full options description. Also options [are documented](https://maigret.readthedocs.io/en/latest/command-line-options.html).
 
 
 ## Demo with page parsing and recursive username search

docs/source/conf.py

@@ -6,8 +6,8 @@ project = 'Maigret'
 copyright = '2021, soxoj'
 author = 'soxoj'
 
-release = '0.4.1'
+release = '0.4.2'
-version = '0.4.1'
+version = '0.4.2'
 
 # -- General configuration
 

docs/source/development.rst

@@ -0,0 +1,101 @@
+.. _development:
+
+Development
+==============
+
+Testing
+-------
+
+It is recommended use Python 3.7/3.8 for test due to some conflicts in 3.9.
+
+Install test requirements:
+
+.. code-block:: console
+
+  pip install -r test-requirements.txt
+
+
+Use the following commands to check Maigret:
+
+.. code-block:: console
+
+  # run linter and typing checks
+  # order of checks%
+  # - critical syntax errors or undefined names
+  # - flake checks
+  # - mypy checks
+  make lint
+
+  # run testing with coverage html report
+  # current test coverage is 60%
+  make text
+
+  # open html report
+  open htmlcov/index.html
+
+
+How to publish new version of Maigret
+-------------------------------------
+
+**Collaborats rights are requires, write Soxoj to get them**.
+
+For new version publishing you must create a new branch in repository
+with a bumped version number and actual changelog first. After it you
+must create a release, and GitHub action automatically create a new 
+PyPi package. 
+
+- New branch example: https://github.com/soxoj/maigret/commit/e520418f6a25d7edacde2d73b41a8ae7c80ddf39
+- Release example: https://github.com/soxoj/maigret/releases/tag/v0.4.1
+
+1. Make a new branch locally with a new version name. Check the current version number here: https://pypi.org/project/maigret/.
+**Increase only patch version (third number)** if there are no breaking changes.
+
+.. code-block:: console
+
+  git checkout -b 0.4.0
+
+2. Update Maigret version in three files manually:
+
+- setup.py
+- maigret/__version__.py 
+- docs/source/conf.py 
+
+3. Create a new empty text section in the beginning of the file `CHANGELOG.md` with a current date:
+
+.. code-block:: console
+
+  ## [0.4.0] - 2022-01-03
+
+4. Get auto-generate release notes:
+
+- Open https://github.com/soxoj/maigret/releases/new
+- Click `Choose a tag`, enter `test`
+- Click `Create new tag`
+- Press `+ Auto-generate release notes`
+- Copy all the text from description text field below
+- Paste it to empty text section in `CHANGELOG.txt`
+- Remove redundant lines `## What's Changed` and `## New Contributors` section if it exists
+- *Close the new release page*
+
+5. Commit all the changes, push, make pull request
+
+.. code-block:: console
+
+  git add ...
+  git commit -m 'Bump to 0.4.0'
+  git push origin head
+
+
+6. Merge pull request
+
+7. Create new release
+
+- Open https://github.com/soxoj/maigret/releases/new again
+- Click `Choose a tag`
+- Enter actual version in format `v0.4.0`
+- Also enter actual version in the field `Release title` 
+- Click `Create new tag`
+- Press `+ Auto-generate release notes`
+- **Press "Publish release" button**
+
+8. That's all, now you can simply wait push to PyPi. You can monitor it in Action page: https://github.com/soxoj/maigret/actions/workflows/python-publish.yml

docs/source/index.rst

@@ -28,3 +28,4 @@ You may be interested in:
    tags
    usage-examples
    settings
+   development

maigret/__version__.py

@@ -1,3 +1,3 @@
 """Maigret version file"""
 
-__version__ = '0.4.1'
+__version__ = '0.4.2'

maigret/checking.py

@@ -132,7 +132,7 @@ class SimpleAiohttpChecker(CheckerBase):
                 error = CheckError("Unexpected", str(e))
 
         if error == "Invalid proxy response":
-            self.logger.debug(e, exc_info=True)
+            self.logger.debug(error, exc_info=True)
 
         return str(html_text), status_code, error
 

maigret/maigret.py

@@ -48,7 +48,7 @@ def notify_about_errors(search_results: QueryResultWrapper, query_notify):
     for e in errs:
         if not errors.is_important(e):
             continue
-        text = f'Too many errors of type "{e["err"]}" ({e["perc"]}%)'
+        text = f'Too many errors of type "{e["err"]}" ({round(e["perc"],2)}%)'
         solution = errors.solution_of(e['err'])
         if solution:
             text = '. '.join([text, solution.capitalize()])
@@ -566,7 +566,7 @@ async def main():
 
     # Database statistics
     if args.stats:
-        print(db.get_db_stats(db.sites_dict))
+        print(db.get_db_stats())
 
     report_dir = path.join(os.getcwd(), args.folderoutput)
 

maigret/sites.py

@@ -419,9 +419,8 @@ class MaigretDatabase:
             results[_id] = _type
         return results
 
-    def get_db_stats(self, sites_dict):
+    def get_db_stats(self, is_markdown=False):
-        if not sites_dict:
+        sites_dict = self.sites_dict
-            sites_dict = self.sites_dict()
 
         urls = {}
         tags = {}
@@ -429,6 +428,9 @@ class MaigretDatabase:
         disabled_count = 0
         total_count = len(sites_dict)
 
+        message_checks = 0
+        message_checks_one_factor = 0
+
         for _, site in sites_dict.items():
             if site.disabled:
                 disabled_count += 1
@@ -436,24 +438,37 @@ class MaigretDatabase:
             url_type = site.get_url_template()
             urls[url_type] = urls.get(url_type, 0) + 1
 
+            if site.check_type == 'message' and not site.disabled:
+                message_checks += 1
+                if site.absence_strs and site.presense_strs:
+                    continue
+                message_checks_one_factor += 1
+
             if not site.tags:
                 tags["NO_TAGS"] = tags.get("NO_TAGS", 0) + 1
 
             for tag in filter(lambda x: not is_country_tag(x), site.tags):
                 tags[tag] = tags.get(tag, 0) + 1
 
-        output += f"Enabled/total sites: {total_count - disabled_count}/{total_count}\n"
+        enabled_perc = round(100*(total_count-disabled_count)/total_count, 2)
-        output += "Top profile URLs:\n"
+        output += f"Enabled/total sites: {total_count - disabled_count}/{total_count} = {enabled_perc}%\n\n"
-        for url, count in sorted(urls.items(), key=lambda x: x[1], reverse=True)[:20]:
+
+        checks_perc = round(100*message_checks_one_factor/message_checks, 2)
+        output += f"Incomplete checks: {message_checks_one_factor}/{message_checks} = {checks_perc}% (false positive risks)\n\n"
+
+        top_urls_count = 20
+        output += f"Top {top_urls_count} profile URLs:\n"
+        for url, count in sorted(urls.items(), key=lambda x: x[1], reverse=True)[:top_urls_count]:
             if count == 1:
                 break
-            output += f"{count}\t{url}\n"
+            output += f"- ({count})\t`{url}`\n" if is_markdown else f"{count}\t{url}\n"
 
-        output += "Top tags:\n"
+        top_tags_count = 20
-        for tag, count in sorted(tags.items(), key=lambda x: x[1], reverse=True)[:200]:
+        output += f"\nTop {top_tags_count} tags:\n"
+        for tag, count in sorted(tags.items(), key=lambda x: x[1], reverse=True)[:top_tags_count]:
             mark = ""
             if tag not in self._tags:
                 mark = " (non-standard)"
-            output += f"{count}\t{tag}{mark}\n"
+            output += f"- ({count})\t`{tag}`{mark}\n" if is_markdown else f"{count}\t{tag}{mark}\n"
 
         return output

requirements.txt

@@ -12,10 +12,11 @@ future-annotations==1.0.0
 html5lib==1.1
 idna==3.3
 Jinja2==3.0.3
-lxml==4.7.1
+lxml==4.8.0
 MarkupSafe==2.0.1
 mock==4.0.3
-multidict==5.2.0
+multidict==5.2.0;python_version<"3.7"
+multidict==6.0.2;python_version>="3.7"
 pycountry==22.1.10
 PyPDF2==1.26.0
 PySocks==1.7.1
@@ -27,11 +28,12 @@ socid-extractor>=0.0.21
 soupsieve==2.3.1
 stem==1.8.0
 torrequest==0.1.0
-tqdm==4.62.3
+tqdm==4.63.0
-typing-extensions==4.0.1
+typing-extensions==4.1.1
 webencodings==0.5.1
 xhtml2pdf==0.2.5
 XMind==1.2.0
 yarl==1.7.2
 networkx==2.5.1
 pyvis==0.1.9
+reportlab==3.6.6

setup.py

@@ -11,7 +11,7 @@ with open('requirements.txt') as rf:
     requires = rf.read().splitlines()
 
 setup(name='maigret',
-      version='0.4.1',
+      version='0.4.2',
       description='Collect a dossier on a person by username from a huge number of sites',
       long_description=long_description,
       long_description_content_type="text/markdown",

sites.md

@@ -1,5 +1,5 @@
 
-## List of supported sites (search methods): total 2593
+## List of supported sites (search methods): total 2638
 
 Rank data fetched from Alexa by domains.
 
@@ -20,7 +20,8 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://naver.com) [Naver (https://naver.com)](https://naver.com)*: top 50, kr*
 1. ![](https://www.google.com/s2/favicons?domain=https://developer.apple.com/forums) [AppleDeveloper (https://developer.apple.com/forums)](https://developer.apple.com/forums)*: top 50, forum, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://discussions.apple.com/) [AppleDiscussions (https://discussions.apple.com/)](https://discussions.apple.com/)*: top 50, us*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.twitter.com/) [Twitter (https://www.twitter.com/)](https://www.twitter.com/)*: top 50, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.nitter.net/) [Nitter (https://www.nitter.net/)](https://www.nitter.net/)*: top 50, messaging*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.twitter.com/) [Twitter (https://www.twitter.com/)](https://www.twitter.com/)*: top 50, messaging*
 1. ![](https://www.google.com/s2/favicons?domain=https://allods.mail.ru) [Allods (https://allods.mail.ru)](https://allods.mail.ru)*: top 50, forum, gaming, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://aa.mail.ru) [ArcheAge (https://aa.mail.ru)](https://aa.mail.ru)*: top 50, forum, gaming, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://cfire.mail.ru) [Crossfire (https://cfire.mail.ru)](https://cfire.mail.ru)*: top 50, gaming, ru*
@@ -51,6 +52,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://wordpress.com) [WordPress (https://wordpress.com)](https://wordpress.com)*: top 100, blog*
 1. ![](https://www.google.com/s2/favicons?domain=https://ok.ru/) [OK (https://ok.ru/)](https://ok.ru/)*: top 100, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.tradingview.com/) [TradingView (https://www.tradingview.com/)](https://www.tradingview.com/)*: top 100, trading, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.aparat.com) [Aparat (https://www.aparat.com)](https://www.aparat.com)*: top 100, ir, video*
 1. ![](https://www.google.com/s2/favicons?domain=https://chaturbate.com) [ChaturBate (https://chaturbate.com)](https://chaturbate.com)*: top 100, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://medium.com/) [Medium (https://medium.com/)](https://medium.com/)*: top 100, blog, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.livejasmin.com/) [Livejasmin (https://www.livejasmin.com/)](https://www.livejasmin.com/)*: top 100, us, webcam*
@@ -195,7 +197,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://www.polygon.com/) [Polygon (https://www.polygon.com/)](https://www.polygon.com/)*: top 5K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://otzovik.com/) [Otzovik (https://otzovik.com/)](https://otzovik.com/)*: top 5K, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.liveinternet.ru) [LiveInternet (https://www.liveinternet.ru)](https://www.liveinternet.ru)*: top 5K, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://leetcode.com/) [LeetCode (https://leetcode.com/)](https://leetcode.com/)*: top 5K, coding*
+1. ![](https://www.google.com/s2/favicons?domain=https://leetcode.com/) [LeetCode (https://leetcode.com/)](https://leetcode.com/)*: top 5K, coding*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://www.kaggle.com/) [Kaggle (https://www.kaggle.com/)](https://www.kaggle.com/)*: top 5K, tech*
 1. ![](https://www.google.com/s2/favicons?domain=https://codepen.io/) [Codepen (https://codepen.io/)](https://codepen.io/)*: top 5K, coding, in*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.rajce.idnes.cz/) [Rajce.net (https://www.rajce.idnes.cz/)](https://www.rajce.idnes.cz/)*: top 5K, cz*
@@ -247,7 +249,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.xda-developers.com) [XDA (https://forum.xda-developers.com)](https://forum.xda-developers.com)*: top 5K, apps, forum*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://i.thechive.com/) [Thechive (https://i.thechive.com/)](https://i.thechive.com/)*: top 5K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://999.md) [999.md (https://999.md)](https://999.md)*: top 5K, freelance, md, shopping*
-1. ![](https://www.google.com/s2/favicons?domain=https://ru.foursquare.com/) [Foursquare (https://ru.foursquare.com/)](https://ru.foursquare.com/)*: top 5K, geosocial, in*
+1. ![](https://www.google.com/s2/favicons?domain=https://foursquare.com/) [Foursquare (https://foursquare.com/)](https://foursquare.com/)*: top 5K, geosocial, in*
 1. ![](https://www.google.com/s2/favicons?domain=https://4pda.ru/) [4pda (https://4pda.ru/)](https://4pda.ru/)*: top 5K, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.weforum.org) [Weforum (https://www.weforum.org)](https://www.weforum.org)*: top 5K, forum, us*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.techspot.com/community/) [techspot.com (http://www.techspot.com/community/)](http://www.techspot.com/community/)*: top 5K, forum, us*
@@ -298,7 +300,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://forums.digitalspy.com/) [Digitalspy (https://forums.digitalspy.com/)](https://forums.digitalspy.com/)*: top 10K, forum, gb, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.bibsonomy.org) [Bibsonomy (https://www.bibsonomy.org)](https://www.bibsonomy.org)*: top 10K, in*
 1. ![](https://www.google.com/s2/favicons?domain=https://slashdot.org) [Slashdot (https://slashdot.org)](https://slashdot.org)*: top 10K, news*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.netvibes.com) [Netvibes (https://www.netvibes.com)](https://www.netvibes.com)*: top 10K, de, fr, jp*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.netvibes.com) [Netvibes (https://www.netvibes.com)](https://www.netvibes.com)*: top 10K, business, fr*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://opensource.com/) [opensource (https://opensource.com/)](https://opensource.com/)*: top 10K, in, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://discuss.elastic.co/) [Discuss.Elastic.co (https://discuss.elastic.co/)](https://discuss.elastic.co/)*: top 10K, forum, tech, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.baby.ru/) [Baby.ru (https://www.baby.ru/)](https://www.baby.ru/)*: top 10K, ru*
@@ -425,7 +427,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://allmylinks.com/) [allmylinks (https://allmylinks.com/)](https://allmylinks.com/)*: top 100K, links*
 1. ![](https://www.google.com/s2/favicons?domain=https://ello.co/) [Ello (https://ello.co/)](https://ello.co/)*: top 100K, in*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.metal-archives.com) [Metal-archives (https://www.metal-archives.com)](https://www.metal-archives.com)*: top 100K, de, music, pl, us*
-1. ![](https://www.google.com/s2/favicons?domain=https://mel.fm) [mel.fm (https://mel.fm)](https://mel.fm)*: top 100K, *
+1. ![](https://www.google.com/s2/favicons?domain=https://mel.fm) [mel.fm (https://mel.fm)](https://mel.fm)*: top 100K, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.influenster.com/) [Influenster (https://www.influenster.com/)](https://www.influenster.com/)*: top 100K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.neoseeker.com) [Neoseeker (https://www.neoseeker.com)](https://www.neoseeker.com)*: top 100K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://community.infosecinstitute.com) [InfosecInstitute (https://community.infosecinstitute.com)](https://community.infosecinstitute.com)*: top 100K, us*
@@ -646,6 +648,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://www.bookcrossing.com/) [Bookcrossing (https://www.bookcrossing.com/)](https://www.bookcrossing.com/)*: top 100K, in*
 1. ![](https://www.google.com/s2/favicons?domain=https://dota2.ru/) [Dota2 (https://dota2.ru/)](https://dota2.ru/)*: top 100K, gaming, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.questionablequesting.com) [QuestionableQuesting (https://forum.questionablequesting.com)](https://forum.questionablequesting.com)*: top 100K, forum, gb, jp, us*, search is disabled
+1. ![](https://www.google.com/s2/favicons?domain=) [Appian ()]()*: top 100K*
 1. ![](https://www.google.com/s2/favicons?domain=https://dissenter.com/) [Dissenter (https://dissenter.com/)](https://dissenter.com/)*: top 100K, us*
 1. ![](https://www.google.com/s2/favicons?domain=insanejournal.com) [Insanejournal (insanejournal.com)](insanejournal.com)*: top 100K, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://shazoo.ru) [Shazoo (https://shazoo.ru)](https://shazoo.ru)*: top 100K, ru*
@@ -810,7 +813,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://bbs.huami.com) [bbs.huami.com (https://bbs.huami.com)](https://bbs.huami.com)*: top 10M, cn, in, ir, ru, us*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://amazfitwatchfaces.com) [AmazfitWatchFaces (https://amazfitwatchfaces.com)](https://amazfitwatchfaces.com)*: top 10M, ae, es, forum, gr, id, ir, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://moikrug.ru/) [MoiKrug (https://moikrug.ru/)](https://moikrug.ru/)*: top 10M, career, us*
-1. ![](https://www.google.com/s2/favicons?domain=http://www.movescount.com) [Movescount (http://www.movescount.com)](http://www.movescount.com)*: top 10M, es, in, pk, ru, us*
+1. ![](https://www.google.com/s2/favicons?domain=http://www.movescount.com) [Movescount (http://www.movescount.com)](http://www.movescount.com)*: top 10M, maps*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://tamtam.chat/) [TamTam (https://tamtam.chat/)](https://tamtam.chat/)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.velomania.ru/) [Velomania (https://forum.velomania.ru/)](https://forum.velomania.ru/)*: top 10M, forum, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.itvdn.com) [ITVDN Forum (https://forum.itvdn.com)](https://forum.itvdn.com)*: top 10M, forum, ru, ua*
@@ -835,7 +838,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://ethresear.ch) [Ethresear (https://ethresear.ch)](https://ethresear.ch)*: top 10M, ch, cr, forum, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://sysadmins.ru) [Sysadmins (https://sysadmins.ru)](https://sysadmins.ru)*: top 10M, forum, ru, tech*
 1. ![](https://www.google.com/s2/favicons?domain=https://radioskot.ru) [radioskot (https://radioskot.ru)](https://radioskot.ru)*: top 10M, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://forum.ruboard.ru) [ruboard (https://forum.ruboard.ru)](https://forum.ruboard.ru)*: top 10M, forum, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://forum.ruboard.ru) [ruboard (https://forum.ruboard.ru)](https://forum.ruboard.ru)*: top 10M, forum, ru*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://carmasters.org) [Carmasters (https://carmasters.org)](https://carmasters.org)*: top 10M, fi, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://fclmnews.ru) [Fclmnews (https://fclmnews.ru)](https://fclmnews.ru)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.ustream.tv) [Ustream (http://www.ustream.tv)](http://www.ustream.tv)*: top 10M, eg, us*
@@ -1026,7 +1029,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://ethereum-magicians.org) [Ethereum-magicians (https://ethereum-magicians.org)](https://ethereum-magicians.org)*: top 10M, cr, forum*
 1. ![](https://www.google.com/s2/favicons?domain=http://bbs.evony.com) [bbs.evony.com (http://bbs.evony.com)](http://bbs.evony.com)*: top 10M, forum, in, pk, tr, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.animeforum.com) [Animeforum (https://www.animeforum.com)](https://www.animeforum.com)*: top 10M, forum, pk, us, vn*
-1. ![](https://www.google.com/s2/favicons?domain=https://kinooh.ru) [Kinooh (https://kinooh.ru)](https://kinooh.ru)*: top 10M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://kinooh.ru) [Kinooh (https://kinooh.ru)](https://kinooh.ru)*: top 10M, ru*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://www.clubsnap.com/) [clubsnap.com (https://www.clubsnap.com/)](https://www.clubsnap.com/)*: top 10M, forum, in, sg, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://southklad.ru) [Southklad (https://southklad.ru)](https://southklad.ru)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.theturboforums.com/forums/) [theturboforums.com (https://www.theturboforums.com/forums/)](https://www.theturboforums.com/forums/)*: top 10M, forum, us*
@@ -1054,7 +1057,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.swiftbook.ru) [swiftbook (https://forum.swiftbook.ru)](https://forum.swiftbook.ru)*: top 10M, forum, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://ladies.zp.ua) [Ladies (http://ladies.zp.ua)](http://ladies.zp.ua)*: top 10M, forum, ua*
 1. ![](https://www.google.com/s2/favicons?domain=https://archlinux.org.ru) [Archlinux (https://archlinux.org.ru)](https://archlinux.org.ru)*: top 10M, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://bethepro.com) [Bethepro (https://bethepro.com)](https://bethepro.com)*: top 10M, pk, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://bethepro.com) [Bethepro (https://bethepro.com)](https://bethepro.com)*: top 10M, pk, us*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.mapillary.com) [Mapillary Forum (https://forum.mapillary.com)](https://forum.mapillary.com)*: top 10M, forum, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.firearmstalk.com) [Firearmstalk (https://www.firearmstalk.com)](https://www.firearmstalk.com)*: top 10M, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.houserepairtalk.com) [Houserepairtalk (https://www.houserepairtalk.com)](https://www.houserepairtalk.com)*: top 10M, forum, in, us*
@@ -1168,7 +1171,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=http://www.cigarpass.com/forum) [cigarpass.com (http://www.cigarpass.com/forum)](http://www.cigarpass.com/forum)*: top 10M, forum, ir*
 1. ![](https://www.google.com/s2/favicons?domain=https://letschatlove.com) [Letschatlove (https://letschatlove.com)](https://letschatlove.com)*: top 10M, in*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.connosr.com/) [Connosr (https://www.connosr.com/)](https://www.connosr.com/)*: top 10M, gb*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.bayoushooter.com) [Bayoushooter (https://www.bayoushooter.com)](https://www.bayoushooter.com)*: top 10M, forum, pk, us*, search is disabled
+1. ![](https://www.google.com/s2/favicons?domain=https://www.bayoushooter.com) [Bayoushooter (https://www.bayoushooter.com)](https://www.bayoushooter.com)*: top 10M, forum, pk, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://nygunforum.com) [Nygunforum (https://nygunforum.com)](https://nygunforum.com)*: top 10M, forum, us*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.astra-club.ru) [Astra-club (http://www.astra-club.ru)](http://www.astra-club.ru)*: top 10M, ru, ua*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=http://phrack.org) [Phrack (http://phrack.org)](http://phrack.org)*: top 10M*
@@ -1197,7 +1200,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://mastodon.xyz/) [mastodon.xyz (https://mastodon.xyz/)](https://mastodon.xyz/)*: top 10M, th*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.gays.com) [Gays (https://www.gays.com)](https://www.gays.com)*: top 10M, in*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=http://transit-club.com) [transit-club.com (http://transit-club.com)](http://transit-club.com)*: top 10M, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://favera.ru) [Favera (https://favera.ru)](https://favera.ru)*: top 10M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://favera.ru) [Favera (https://favera.ru)](https://favera.ru)*: top 10M, ru*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://soylentnews.org) [soylentnews (https://soylentnews.org)](https://soylentnews.org)*: top 10M, us*
 1. ![](https://www.google.com/s2/favicons?domain=http://chan4chan.com/) [Chan4chan (http://chan4chan.com/)](http://chan4chan.com/)*: top 10M, hu*
 1. ![](https://www.google.com/s2/favicons?domain=http://the-mainboard.com/index.php) [the-mainboard.com (http://the-mainboard.com/index.php)](http://the-mainboard.com/index.php)*: top 10M, forum, us*
@@ -1231,6 +1234,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://devtribe.ru) [Devtribe (https://devtribe.ru)](https://devtribe.ru)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://forest.ru/) [Forest (https://forest.ru/)](https://forest.ru/)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://shotbow.net) [Shotbow (https://shotbow.net)](https://shotbow.net)*: top 10M, ca, forum, us*, search is disabled
+1. ![](https://www.google.com/s2/favicons?domain=https://101010.pl/) [101010.pl (https://101010.pl/)](https://101010.pl/)*: top 10M*
 1. ![](https://www.google.com/s2/favicons?domain=https://otechie.com) [Otechie (https://otechie.com)](https://otechie.com)*: top 10M, finance, us*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.volgogradru.com) [Volgogradru (http://www.volgogradru.com)](http://www.volgogradru.com)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://pilguy.com) [Pilguy (https://pilguy.com)](https://pilguy.com)*: top 10M, forum, ru*, search is disabled
@@ -1388,7 +1392,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://uaksu.forum24.ru/) [Uaksu (https://uaksu.forum24.ru/)](https://uaksu.forum24.ru/)*: top 10M, forum, ru, ua*
 1. ![](https://www.google.com/s2/favicons?domain=http://0-3.ru) [0-3.RU (http://0-3.ru)](http://0-3.ru)*: top 10M, forum, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.forumsi.org) [Forumsi (http://www.forumsi.org)](http://www.forumsi.org)*: top 10M, forum, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://www.snooth.com/) [Snooth (https://www.snooth.com/)](https://www.snooth.com/)*: top 10M, in*
+1. ![](https://www.google.com/s2/favicons?domain=https://www.snooth.com/) [Snooth (https://www.snooth.com/)](https://www.snooth.com/)*: top 10M, news*
 1. ![](https://www.google.com/s2/favicons?domain=http://soft-deniz.ucoz.ru) [soft-deniz.ucoz.ru (http://soft-deniz.ucoz.ru)](http://soft-deniz.ucoz.ru)*: top 10M*
 1. ![](https://www.google.com/s2/favicons?domain=http://oih.at.ua) [oih.at.ua (http://oih.at.ua)](http://oih.at.ua)*: top 10M, ua*
 1. ![](https://www.google.com/s2/favicons?domain=http://gorodanapa.ru/) [Gorodanapa (http://gorodanapa.ru/)](http://gorodanapa.ru/)*: top 10M, ru*, search is disabled
@@ -1422,7 +1426,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=http://www.mactalk.com.au/) [Mactalk (http://www.mactalk.com.au/)](http://www.mactalk.com.au/)*: top 10M, au, in, pk*
 1. ![](https://www.google.com/s2/favicons?domain=http://animal-hope.ru) [animal-hope.ru (http://animal-hope.ru)](http://animal-hope.ru)*: top 10M*
 1. ![](https://www.google.com/s2/favicons?domain=https://sexforum.win) [Sexwin (https://sexforum.win)](https://sexforum.win)*: top 10M, forum, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://ttonlineviewer.com) [TikTok Online Viewer (https://ttonlineviewer.com)](https://ttonlineviewer.com)*: top 10M, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://ttonlineviewer.com) [TikTok Online Viewer (https://ttonlineviewer.com)](https://ttonlineviewer.com)*: top 10M, us*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=http://tavr-obrazovanie.ru) [tavr-obrazovanie.ru (http://tavr-obrazovanie.ru)](http://tavr-obrazovanie.ru)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://studentur.com.ua) [studentur.com.ua (http://studentur.com.ua)](http://studentur.com.ua)*: top 10M, ua*
 1. ![](https://www.google.com/s2/favicons?domain=https://offline.by) [Offline.by (https://offline.by)](https://offline.by)*: top 10M, forum, ru*
@@ -1440,7 +1444,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=http://rodobozhie.ru) [rodobozhie.ru (http://rodobozhie.ru)](http://rodobozhie.ru)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://satwarez.ru) [satwarez.ru (http://satwarez.ru)](http://satwarez.ru)*: top 10M*
 1. ![](https://www.google.com/s2/favicons?domain=http://nf-club.ru) [nf-club.ru (http://nf-club.ru)](http://nf-club.ru)*: top 10M*
-1. ![](https://www.google.com/s2/favicons?domain=http://linuxmint.info) [linuxmint.info (http://linuxmint.info)](http://linuxmint.info)*: top 10M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=http://linuxmint.info) [linuxmint.info (http://linuxmint.info)](http://linuxmint.info)*: top 10M, ru*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=http://kiabongo.info) [kiabongo.info (http://kiabongo.info)](http://kiabongo.info)*: top 10M*
 1. ![](https://www.google.com/s2/favicons?domain=http://koshtoris.at.ua) [koshtoris.at.ua (http://koshtoris.at.ua)](http://koshtoris.at.ua)*: top 10M*
 1. ![](https://www.google.com/s2/favicons?domain=http://xn--90anbhklk.xn--p1ai) [xn--90anbhklk.xn--p1ai (http://xn--90anbhklk.xn--p1ai)](http://xn--90anbhklk.xn--p1ai)*: top 10M*
@@ -1467,7 +1471,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=http://wolga24.at.ua) [wolga24.at.ua (http://wolga24.at.ua)](http://wolga24.at.ua)*: top 10M*
 1. ![](https://www.google.com/s2/favicons?domain=http://millerovo161.ru) [millerovo161.ru (http://millerovo161.ru)](http://millerovo161.ru)*: top 10M*
 1. ![](https://www.google.com/s2/favicons?domain=http://videomuzon.ucoz.ru) [videomuzon.ucoz.ru (http://videomuzon.ucoz.ru)](http://videomuzon.ucoz.ru)*: top 10M*
-1. ![](https://www.google.com/s2/favicons?domain=https://community.autolenta.ru) [Autolenta (https://community.autolenta.ru)](https://community.autolenta.ru)*: top 10M, auto, forum, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://community.autolenta.ru) [Autolenta (https://community.autolenta.ru)](https://community.autolenta.ru)*: top 10M, auto, forum, ru*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=http://amax-sb.ru) [amax-sb.ru (http://amax-sb.ru)](http://amax-sb.ru)*: top 10M*
 1. ![](https://www.google.com/s2/favicons?domain=http://angelgothics.ru) [Angelgothics (http://angelgothics.ru)](http://angelgothics.ru)*: top 10M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://help-baby.org) [help-baby.org (http://help-baby.org)](http://help-baby.org)*: top 10M*
@@ -1545,23 +1549,52 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=http://74507.ucoz.ru) [74507.ucoz.ru (http://74507.ucoz.ru)](http://74507.ucoz.ru)*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://apteka.ee) [AMUR (https://apteka.ee)](https://apteka.ee)*: top 100M, dating, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://alabay.forum24.ru) [Alabay (https://alabay.forum24.ru)](https://alabay.forum24.ru)*: top 100M, forum, ru*
+1. ![](https://www.google.com/s2/favicons?domain=) [Anilist ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://antipunk.com/) [Antipunk (https://antipunk.com/)](https://antipunk.com/)*: top 100M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=) [Arduino ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Artistsnclients ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=http://artpersona.org/) [Artpersona (http://artpersona.org/)](http://artpersona.org/)*: top 100M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=) [AvidCommunity ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://avtoforum.org) [Avtoforum (https://avtoforum.org)](https://avtoforum.org)*: top 100M, forum, ru*
+1. ![](https://www.google.com/s2/favicons?domain=) [Bandlab ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Bentbox ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Biggerpockets ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Bitwarden ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Blogi.pl ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Bugcrowd ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Caringbridge ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Castingcallclub ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Chomikuj.pl ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://3examplesite.ru) [CPlusPlus (https://3examplesite.ru)](https://3examplesite.ru)*: top 100M, ru*, search is disabled
+1. ![](https://www.google.com/s2/favicons?domain=) [Crowdin ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.casial.net) [Casial (http://www.casial.net)](http://www.casial.net)*: top 100M, de*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.casino-affiliate-forum.com) [Casino-affiliate-forum (https://www.casino-affiliate-forum.com)](https://www.casino-affiliate-forum.com)*: top 100M, de, forum*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.chess-russia.ru) [Chess-russia (http://www.chess-russia.ru)](http://www.chess-russia.ru)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://club-comedy.clan.su) [Club-comedy.clan.su (https://club-comedy.clan.su)](https://club-comedy.clan.su)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.cssomsk.ru) [Cssomsk (http://www.cssomsk.ru)](http://www.cssomsk.ru)*: top 100M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=) [Cults3d ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://cyberclock.cc) [Cyberclock (https://cyberclock.cc)](https://cyberclock.cc)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.cydak.ru) [Cydak (http://www.cydak.ru)](http://www.cydak.ru)*: top 100M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=) [Cytoid.io ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Demotywatory ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Dojoverse ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.designspiration.net/) [Designspiration (https://www.designspiration.net/)](https://www.designspiration.net/)*: top 100M*
-1. ![](https://www.google.com/s2/favicons?domain=https://dinsk.su) [Dinsk (https://dinsk.su)](https://dinsk.su)*: top 100M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=https://dinsk.su) [Dinsk (https://dinsk.su)](https://dinsk.su)*: top 100M, ru*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.djangoproject.co) [Djangoproject.co (https://forum.djangoproject.co)](https://forum.djangoproject.co)*: top 100M, coding, forum*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.dublikat.shop) [Dublikat (https://www.dublikat.shop)](https://www.dublikat.shop)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://eightbit.me/) [Eightbit (http://eightbit.me/)](http://eightbit.me/)*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://enot-poloskun.ru/) [Enot-poloskun (https://enot-poloskun.ru/)](https://enot-poloskun.ru/)*: top 100M, ru*
+1. ![](https://www.google.com/s2/favicons?domain=) [Fabswingers ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Faktopedia ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Fancentro ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Fansly ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Fedi.lewactwo.pl ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Forumprawne.org ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Fosstodon ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Fotka ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Friendfinder ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Friendfinder-x ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Furaffinity ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.fegatch.com/) [Fegatch (http://www.fegatch.com/)](http://www.fegatch.com/)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.filmo.gs/) [Filmogs (https://www.filmo.gs/)](https://www.filmo.gs/)*: top 100M, movies*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://forum.jambox.ru) [Forum.jambox.ru (https://forum.jambox.ru)](https://forum.jambox.ru)*: top 100M, forum, ru*
@@ -1572,19 +1605,31 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://gam1ng.com.br) [Gam1ng (https://gam1ng.com.br)](https://gam1ng.com.br)*: top 100M, br, webcam*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://www.garagepunk.com) [GaragePunk (https://www.garagepunk.com)](https://www.garagepunk.com)*: top 100M, us*
 1. ![](https://www.google.com/s2/favicons?domain=http://www.gliger.ru) [Gliger (http://www.gliger.ru)](http://www.gliger.ru)*: top 100M, ru*, search is disabled
+1. ![](https://www.google.com/s2/favicons?domain=) [Gnome-vcs ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://forums.golf-monthly.co.uk/) [GolfMonthly (https://forums.golf-monthly.co.uk/)](https://forums.golf-monthly.co.uk/)*: top 100M, forum, gb, us*
 1. ![](https://www.google.com/s2/favicons?domain=http://gothic.su) [Gothic (http://gothic.su)](http://gothic.su)*: top 100M, forum, ru*
+1. ![](https://www.google.com/s2/favicons?domain=) [Gradle ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.gunandgame.co) [Gunandgame (https://www.gunandgame.co)](https://www.gunandgame.co)*: top 100M, us*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=https://hackeralexaRank.com/) [HackeralexaRank (https://hackeralexaRank.com/)](https://hackeralexaRank.com/)*: top 100M*, search is disabled
+1. ![](https://www.google.com/s2/favicons?domain=) [Hackerrank ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Hexrpg ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Inkbunny ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Ipolska.pl ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.ingunowners.com) [Ingunowners (https://www.ingunowners.com)](https://www.ingunowners.com)*: top 100M, forum, us*
+1. ![](https://www.google.com/s2/favicons?domain=) [Jbzd ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Jeja.pl ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Jellyfin Weblate ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=http://jer.forum24.ru) [Jer.forum24.ru (http://jer.forum24.ru)](http://jer.forum24.ru)*: top 100M, forum, ru*
+1. ![](https://www.google.com/s2/favicons?domain=) [Joemonster ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Karab.in ()]()*: top 100M*
+1. ![](https://www.google.com/s2/favicons?domain=) [Kotburger ()]()*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://lib.reviews) [LibReviews (https://lib.reviews)](https://lib.reviews)*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.liebe69.de) [Liebe69 (https://www.liebe69.de)](https://www.liebe69.de)*: top 100M, de*
 1. ![](https://www.google.com/s2/favicons?domain=https://lovemakeup.ru) [Lovemakeup (https://lovemakeup.ru)](https://lovemakeup.ru)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://macqa.ru) [Macqa (https://macqa.ru)](https://macqa.ru)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://magiimir.com) [Magiimir (https://magiimir.com)](https://magiimir.com)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://mamochki.by/) [Mamochki (https://mamochki.by/)](https://mamochki.by/)*: top 100M, by, ru*
-1. ![](https://www.google.com/s2/favicons?domain=https://mastersofcrypto.com) [Mastersofcrypto (https://mastersofcrypto.com)](https://mastersofcrypto.com)*: top 100M, forum*
+1. ![](https://www.google.com/s2/favicons?domain=https://mastersofcrypto.com) [Mastersofcrypto (https://mastersofcrypto.com)](https://mastersofcrypto.com)*: top 100M, forum*, search is disabled
 1. ![](https://www.google.com/s2/favicons?domain=http:/mixlr.com/) [Mixlr (http:/mixlr.com/)](http:/mixlr.com/)*: top 100M, gb*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.munzee.com/) [Munzee (https://www.munzee.com/)](https://www.munzee.com/)*: top 100M, gb*
 1. ![](https://www.google.com/s2/favicons?domain=http://murmansk-life.ru) [MurmanskLife (http://murmansk-life.ru)](http://murmansk-life.ru)*: top 100M, ru*
@@ -2281,8 +2326,8 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://freelance.ru) [freelance.ru (https://freelance.ru)](https://freelance.ru)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://freelansim.ru) [freelansim.ru (https://freelansim.ru)](https://freelansim.ru)*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=http://fotolog.com) [fotolog.com (http://fotolog.com)](http://fotolog.com)*: top 100M, in*
-1. ![](https://www.google.com/s2/favicons?domain=http://thoughts.com) [thoughts.com (http://thoughts.com)](http://thoughts.com)*: top 100M, in*
+1. ![](https://www.google.com/s2/favicons?domain=http://thoughts.com) [thoughts.com (http://thoughts.com)](http://thoughts.com)*: top 100M, blog*
-1. ![](https://www.google.com/s2/favicons?domain=https://hackernoon.com) [hackernoon.com (https://hackernoon.com)](https://hackernoon.com)*: top 100M, in, us*
+1. ![](https://www.google.com/s2/favicons?domain=https://hackernoon.com) [hackernoon.com (https://hackernoon.com)](https://hackernoon.com)*: top 100M, news, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://intigriti.com) [Intigriti (https://intigriti.com)](https://intigriti.com)*: top 100M, hacking, in*
 1. ![](https://www.google.com/s2/favicons?domain=https://yamaya.ru) [yamaya.ru (https://yamaya.ru)](https://yamaya.ru)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.tinkoff.ru/invest/) [Tinkoff Invest (https://www.tinkoff.ru/invest/)](https://www.tinkoff.ru/invest/)*: top 100M, ru*
@@ -2311,7 +2356,7 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=http://mednolit.ru) [mednolit.ru (http://mednolit.ru)](http://mednolit.ru)*: top 100M, ru*
 1. ![](https://www.google.com/s2/favicons?domain=http://mikele-loconte.ru) [mikele-loconte.ru (http://mikele-loconte.ru)](http://mikele-loconte.ru)*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=http://mkuniverse.ru) [mkuniverse.ru (http://mkuniverse.ru)](http://mkuniverse.ru)*: top 100M*
-1. ![](https://www.google.com/s2/favicons?domain=https://hashnode.com) [hashnode.com (https://hashnode.com)](https://hashnode.com)*: top 100M, in*
+1. ![](https://www.google.com/s2/favicons?domain=https://hashnode.com) [hashnode (https://hashnode.com)](https://hashnode.com)*: top 100M, in*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.change.org) [www.change.org (https://www.change.org)](https://www.change.org)*: top 100M, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://www.ifunny.co) [www.ifunny.co (https://www.ifunny.co)](https://www.ifunny.co)*: top 100M, us*
 1. ![](https://www.google.com/s2/favicons?domain=https://localcryptosapi.com) [LocalCryptos (https://localcryptosapi.com)](https://localcryptosapi.com)*: top 100M*
@@ -2597,4 +2642,53 @@ Rank data fetched from Alexa by domains.
 1. ![](https://www.google.com/s2/favicons?domain=https://www.hozpitality.com) [hozpitality (https://www.hozpitality.com)](https://www.hozpitality.com)*: top 100M*
 1. ![](https://www.google.com/s2/favicons?domain=https://kazanlashkigalab.com) [kazanlashkigalab.com (https://kazanlashkigalab.com)](https://kazanlashkigalab.com)*: top 100M, kz*
 
-Alexa.com rank data fetched at (2022-01-03 02:59:10.811535 UTC)
+The list was updated at (2022-03-07 16:44:18.939455 UTC)
+## Statistics
+
+Enabled/total sites: 2483/2638 = 94.12%
+
+Incomplete checks: 523/1892 = 27.64% (false positive risks)
+
+Top 20 profile URLs:
+- (796)	`{urlMain}/index/8-0-{username} (uCoz)`
+- (229)	`/{username}`
+- (221)	`{urlMain}{urlSubpath}/members/?username={username} (XenForo)`
+- (144)	`/user/{username}`
+- (134)	`{urlMain}{urlSubpath}/member.php?username={username} (vBulletin)`
+- (101)	`/profile/{username}`
+- (87)	`{urlMain}/u/{username}/summary (Discourse)`
+- (75)	`/users/{username}`
+- (44)	`{urlMain}{urlSubpath}/search.php?author={username} (phpBB/Search)`
+- (41)	`/members/?username={username}`
+- (40)	`/@{username}`
+- (39)	`SUBDOMAIN`
+- (30)	`/u/{username}`
+- (27)	`/members/{username}`
+- (27)	`{urlMain}{urlSubpath}/memberlist.php?username={username} (phpBB)`
+- (18)	`/forum/members/?username={username}`
+- (18)	`/forum/search.php?keywords=&terms=all&author={username}`
+- (17)	`/search.php?keywords=&terms=all&author={username}`
+- (16)	`/author/{username}`
+- (14)	`/people/{username}`
+
+Top 20 tags:
+- (271)	`forum`
+- (83)	`NO_TAGS` (non-standard)
+- (50)	`gaming`
+- (24)	`photo`
+- (24)	`coding`
+- (18)	`news`
+- (18)	`blog`
+- (18)	`music`
+- (15)	`tech`
+- (13)	`freelance`
+- (12)	`sharing`
+- (12)	`finance`
+- (11)	`shopping`
+- (10)	`dating`
+- (10)	`art`
+- (9)	`hobby`
+- (9)	`movies`
+- (7)	`sport`
+- (7)	`hacking`
+- (5)	`stock`

test-requirements.txt

@@ -1,7 +1,8 @@
+reportlab==3.6.6
 flake8==4.0.1
-pytest==6.2.5
+pytest==7.0.1
 pytest-asyncio==0.16.0;python_version<"3.7"
-pytest-asyncio==0.17.0;python_version>="3.7"
+pytest-asyncio==0.18.2;python_version>="3.7"
 pytest-cov==3.0.0
-pytest-httpserver==1.0.3
+pytest-httpserver==1.0.4
 pytest-rerunfailures==10.2

utils/check_engines.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Maigret: Supported Site Listing with Alexa ranking and country tags
+This module generates the listing of supported sites in file `SITES.md`
+and pretty prints file with sites data.
+"""
+import aiohttp
+import asyncio
+import json
+import sys
+import requests
+import logging
+import threading
+import xml.etree.ElementTree as ET
+from datetime import datetime
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+
+import tqdm.asyncio
+
+from maigret.maigret import get_response, site_self_check
+from maigret.sites import MaigretSite, MaigretDatabase, MaigretEngine
+from maigret.utils import CaseConverter
+
+
+async def check_engine_of_site(site_name, sites_with_engines, future, engine_name, semaphore, logger):
+    async with semaphore:
+        response = await get_response(request_future=future,
+                                      site_name=site_name,
+                                      logger=logger)
+
+        html_text, status_code, error_text, expection_text = response
+
+        if html_text and engine_name in html_text:
+            sites_with_engines.append(site_name)
+            return True
+    return False
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter
+                            )
+    parser.add_argument("--base","-b", metavar="BASE_FILE",
+                        dest="base_file", default="maigret/resources/data.json",
+                        help="JSON file with sites data to update.")
+
+    parser.add_argument('--engine', '-e', help='check only selected engine', type=str)
+
+    args = parser.parse_args()
+
+    log_level = logging.INFO
+    logging.basicConfig(
+        format='[%(filename)s:%(lineno)d] %(levelname)-3s  %(asctime)s %(message)s',
+        datefmt='%H:%M:%S',
+        level=log_level
+    )
+    logger = logging.getLogger('engines-check')
+    logger.setLevel(log_level)
+
+    db = MaigretDatabase()
+    sites_subset = db.load_from_file(args.base_file).sites
+    sites = {site.name: site for site in sites_subset}
+
+    with open(args.base_file, "r", encoding="utf-8") as data_file:
+        sites_info = json.load(data_file)
+        engines = sites_info['engines']
+
+    for engine_name, engine_data in engines.items():
+        if args.engine and args.engine != engine_name:
+            continue
+
+        if not 'presenseStrs' in engine_data:
+            print(f'No features to automatically detect sites on engine {engine_name}')
+            continue
+
+        engine_obj = MaigretEngine(engine_name, engine_data)
+
+        # setup connections for checking both engine and usernames
+        connector = aiohttp.TCPConnector(ssl=False)
+        connector.verify_ssl=False
+        session = aiohttp.ClientSession(connector=connector)
+
+        sem = asyncio.Semaphore(100)
+        loop = asyncio.get_event_loop()
+        tasks = []
+
+        # check sites without engine if they look like sites on this engine
+        new_engine_sites = []
+        for site_name, site_data in sites.items():
+            if site_data.engine:
+                continue
+
+            future = session.get(url=site_data.url_main,
+                                 allow_redirects=True,
+                                 timeout=10,
+                                 )
+
+            check_engine_coro = check_engine_of_site(site_name, new_engine_sites, future, engine_name, sem, logger)
+            future = asyncio.ensure_future(check_engine_coro)
+            tasks.append(future)
+
+        # progress bar
+        for f in tqdm.asyncio.tqdm.as_completed(tasks):
+            loop.run_until_complete(f)
+
+        print(f'Total detected {len(new_engine_sites)} sites on engine {engine_name}')
+        # dict with new found engine sites
+        new_sites = {site_name: sites[site_name] for site_name in new_engine_sites}
+
+        # update sites obj from engine
+        for site_name, site in new_sites.items():
+            site.request_future = None
+            site.engine = engine_name
+            site.update_from_engine(engine_obj)
+
+        async def update_site_data(site_name, site_data, all_sites, logger, no_progressbar):
+            updates = await site_self_check(site_name, site_data, logger, no_progressbar)
+            all_sites[site_name].update(updates)
+
+        tasks = []
+        # for new_site_name, new_site_data in new_sites.items():
+            # coro = update_site_data(new_site_name, new_site_data, new_sites, logger)
+            # future = asyncio.ensure_future(coro)
+            # tasks.append(future)
+
+        # asyncio.gather(*tasks)
+        for new_site_name, new_site_data in new_sites.items():
+            coro = update_site_data(new_site_name, new_site_data, new_sites, logger, no_progressbar=True)
+            loop.run_until_complete(coro)
+
+        updated_sites_count = 0
+
+        for s in new_sites:
+            site = new_sites[s]
+            site.request_future = None
+
+            if site.disabled:
+                print(f'{site.name} failed username checking of engine {engine_name}')
+                continue
+
+            site = site.strip_engine_data()
+
+            db.update_site(site)
+            updated_sites_count += 1
+            db.save_to_file(args.base_file)
+
+            print(f'Site "{s}": ' + json.dumps(site.json, indent=4))
+
+        print(f'Updated total {updated_sites_count} sites!')
+        print(f'Checking all sites on engine {engine_name}')
+
+        loop.run_until_complete(session.close())
+
+    print("\nFinished updating supported site listing!")

utils/import_sites.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+import json
+import random
+import re
+
+import tqdm.asyncio
+from mock import Mock
+import requests
+
+from maigret.maigret import *
+from maigret.result import QueryStatus
+from maigret.sites import MaigretSite
+
+URL_RE = re.compile(r"https?://(www\.)?")
+TIMEOUT = 200
+
+
+async def maigret_check(site, site_data, username, status, logger):
+    query_notify = Mock()
+    logger.debug(f'Checking {site}...')
+
+    for username, status in [(username, status)]:
+        results = await maigret(
+            username,
+            {site: site_data},
+            logger,
+            query_notify,
+            timeout=TIMEOUT,
+            forced=True,
+            no_progressbar=True,
+        )
+
+        if results[site]['status'].status != status:
+            if results[site]['status'].status == QueryStatus.UNKNOWN:
+                msg = site_data.absence_strs
+                etype = site_data.check_type
+                context = results[site]['status'].context
+
+                logger.debug(f'Error while searching {username} in {site}, must be claimed. Context: {context}')
+                # if site_data.get('errors'):
+                #     continue
+                return False
+
+            if status == QueryStatus.CLAIMED:
+                logger.debug(f'Not found {username} in {site}, must be claimed')
+                logger.debug(results[site])
+                pass
+            else:
+                logger.debug(f'Found {username} in {site}, must be available')
+                logger.debug(results[site])
+                pass
+            return False
+
+    return site_data
+
+
+async def check_and_add_maigret_site(site_data, semaphore, logger, ok_usernames, bad_usernames):
+    async with semaphore:
+        sitename = site_data.name
+        positive = False
+        negative = False
+
+        for ok_username in ok_usernames:
+            site_data.username_claimed = ok_username
+            status = QueryStatus.CLAIMED
+            if await maigret_check(sitename, site_data, ok_username, status, logger):
+                # print(f'{sitename} positive case is okay')
+                positive = True
+                break
+
+        for bad_username in bad_usernames:
+            site_data.username_unclaimed = bad_username
+            status = QueryStatus.AVAILABLE
+            if await maigret_check(sitename, site_data, bad_username, status, logger):
+                # print(f'{sitename} negative case is okay')
+                negative = True
+                break
+
+        if positive and negative:
+            site_data = site_data.strip_engine_data()
+
+            db.update_site(site_data)
+            print(site_data.json)
+            try:
+                db.save_to_file(args.base_file)
+            except Exception as e:
+                logging.error(e, exc_info=True)
+            print(f'Saved new site {sitename}...')
+            ok_sites.append(site_data)
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter
+                            )
+    parser.add_argument("--base", "-b", metavar="BASE_FILE",
+                        dest="base_file", default="maigret/resources/data.json",
+                        help="JSON file with sites data to update.")
+
+    parser.add_argument("--add-engine", dest="add_engine", help="Additional engine to check")
+
+    parser.add_argument("--only-engine", dest="only_engine", help="Use only this engine from detected to check")
+
+    parser.add_argument('--check', help='only check sites in database', action='store_true')
+
+    parser.add_argument('--random', help='shuffle list of urls', action='store_true', default=False)
+
+    parser.add_argument('--top', help='top count of records in file', type=int, default=10000)
+
+    parser.add_argument('--filter', help='substring to filter input urls', type=str, default='')
+
+    parser.add_argument('--username', help='preferable username to check with', type=str)
+
+    parser.add_argument(
+        "--info",
+        "-vv",
+        action="store_true",
+        dest="info",
+        default=False,
+        help="Display service information.",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        dest="verbose",
+        default=False,
+        help="Display extra information and metrics.",
+    )
+    parser.add_argument(
+        "-d",
+        "--debug",
+        "-vvv",
+        action="store_true",
+        dest="debug",
+        default=False,
+        help="Saving debugging information and sites responses in debug.txt.",
+    )
+
+    parser.add_argument("urls_file",
+                        metavar='URLS_FILE',
+                        action="store",
+                        help="File with base site URLs"
+                        )
+
+    args = parser.parse_args()
+
+    log_level = logging.ERROR
+    if args.debug:
+        log_level = logging.DEBUG
+    elif args.info:
+        log_level = logging.INFO
+    elif args.verbose:
+        log_level = logging.WARNING
+
+    logging.basicConfig(
+        format='[%(filename)s:%(lineno)d] %(levelname)-3s  %(asctime)s %(message)s',
+        datefmt='%H:%M:%S',
+        level=log_level
+    )
+    logger = logging.getLogger('engines-check')
+    logger.setLevel(log_level)
+
+    db = MaigretDatabase()
+    sites_subset = db.load_from_file(args.base_file).sites
+    sites = {site.name: site for site in sites_subset}
+    engines = db.engines
+
+    # TODO: usernames extractors
+    ok_usernames = ['alex', 'god', 'admin', 'red', 'blue', 'john']
+    if args.username:
+        ok_usernames = [args.username] + ok_usernames
+
+    bad_usernames = ['noonewouldeverusethis7']
+
+    with open(args.urls_file, 'r') as urls_file:
+        urls = urls_file.read().splitlines()
+        if args.random:
+            random.shuffle(urls)
+        urls = urls[:args.top]
+
+    raw_maigret_data = json.dumps({site.name: site.json for site in sites_subset})
+
+    new_sites = []
+    for site in tqdm.asyncio.tqdm(urls):
+        site_lowercase = site.lower()
+
+        domain_raw = URL_RE.sub('', site_lowercase).strip().strip('/')
+        domain_raw = domain_raw.split('/')[0]
+
+        if args.filter and args.filter not in domain_raw:
+            logger.debug('Site %s skipped due to filtering by "%s"', domain_raw, args.filter)
+            continue
+
+        if domain_raw in raw_maigret_data:
+            logger.debug(f'Site {domain_raw} already exists in the Maigret database!')
+            continue
+
+        if '"' in domain_raw:
+            logger.debug(f'Invalid site {domain_raw}')
+            continue
+
+        main_page_url = '/'.join(site.split('/', 3)[:3])
+
+        site_data = {
+            'url': site,
+            'urlMain': main_page_url,
+            'name': domain_raw,
+        }
+
+        try:
+            r = requests.get(main_page_url, timeout=5)
+        except:
+            r = None
+            pass
+
+        detected_engines = []
+
+        for e in engines:
+            strs_to_check = e.__dict__.get('presenseStrs')
+            if strs_to_check and r and r.text:
+                all_strs_in_response = True
+                for s in strs_to_check:
+                    if not s in r.text:
+                        all_strs_in_response = False
+                if all_strs_in_response:
+                    engine_name = e.__dict__.get('name')
+                    detected_engines.append(engine_name)
+                    logger.info(f'Detected engine {engine_name} for site {main_page_url}')
+
+        if args.only_engine and args.only_engine in detected_engines:
+            detected_engines = [args.only_engine]
+        elif not detected_engines and args.add_engine:
+            logging.debug('Could not detect any engine, applying default engine %s...', args.add_engine)
+            detected_engines = [args.add_engine]
+
+        def create_site_from_engine(sitename, data, e):
+            site = MaigretSite(sitename, data)
+            site.update_from_engine(db.engines_dict[e])
+            site.engine = e
+            return site
+
+        for engine_name in detected_engines:
+            site = create_site_from_engine(domain_raw, site_data, engine_name)
+            new_sites.append(site)
+            logger.debug(site.json)
+
+            # if engine_name == "phpBB":
+            #     site_data_with_subpath = dict(site_data)
+            #     site_data_with_subpath["urlSubpath"] = "/forum"
+            #     site = create_site_from_engine(domain_raw, site_data_with_subpath, engine_name)
+            #     new_sites.append(site)
+
+        # except Exception as e:
+        #     print(f'Error: {str(e)}')
+        #     pass
+
+    print(f'Found {len(new_sites)}/{len(urls)} new sites')
+
+    if args.check:
+        for s in new_sites:
+            print(s.url_main)
+        sys.exit(0)
+
+    sem = asyncio.Semaphore(20)
+    loop = asyncio.get_event_loop()
+
+    ok_sites = []
+    tasks = []
+    for site in new_sites:
+        check_coro = check_and_add_maigret_site(site, sem, logger, ok_usernames, bad_usernames)
+        future = asyncio.ensure_future(check_coro)
+        tasks.append(future)
+
+    for f in tqdm.asyncio.tqdm.as_completed(tasks, timeout=TIMEOUT):
+        try:
+            loop.run_until_complete(f)
+        except asyncio.exceptions.TimeoutError:
+            pass
+
+    print(f'Found and saved {len(ok_sites)} sites!')

utils/sites_diff.py

@@ -0,0 +1,36 @@
+import sys
+import difflib
+import requests
+
+
+a = requests.get(sys.argv[1]).text
+b = requests.get(sys.argv[2]).text
+
+
+tokens_a = set(a.split('"'))
+tokens_b = set(b.split('"'))
+
+a_minus_b = tokens_a.difference(tokens_b)
+b_minus_a = tokens_b.difference(tokens_a)
+
+print(a_minus_b)
+print(b_minus_a)
+
+print(len(a_minus_b))
+print(len(b_minus_a))
+
+desired_strings = ["username", "not found", "пользователь", "profile", "lastname", "firstname", "biography",
+"birthday", "репутация", "информация", "e-mail"]
+
+
+def get_match_ratio(x):
+    return round(max([
+    	difflib.SequenceMatcher(a=x.lower(), b=y).ratio()
+    	for y in desired_strings
+    ]), 2)
+
+
+RATIO = 0.6
+
+print(sorted(a_minus_b, key=get_match_ratio, reverse=True)[:10])
+print(sorted(b_minus_a, key=get_match_ratio, reverse=True)[:10])

utils/update_site_data.py

@@ -114,7 +114,7 @@ Rank data fetched from Alexa by domains.
                 sys.stdout.flush()
                 index = index + 1
 
-        sites_full_list = [(s, s.alexa_rank) for s in sites_subset]
+        sites_full_list = [(s, int(s.alexa_rank)) for s in sites_subset]
 
         sites_full_list.sort(reverse=False, key=lambda x: x[1])
 
@@ -137,7 +137,11 @@ Rank data fetched from Alexa by domains.
             site_file.write(f'1. {favicon} [{site}]({url_main})*: top {valid_rank}{tags}*{note}\n')
             db.update_site(site)
 
-        site_file.write(f'\nAlexa.com rank data fetched at ({datetime.utcnow()} UTC)\n')
+        site_file.write(f'\nThe list was updated at ({datetime.utcnow()} UTC)\n')
         db.save_to_file(args.base_file)
 
+        statistics_text = db.get_db_stats(is_markdown=True)
+        site_file.write('## Statistics\n\n')
+        site_file.write(statistics_text)
+
     print("\nFinished updating supported site listing!")