Merge pull request #122 from soxoj/0.2.1-bugfix

Fixed json report generation bug, bump to 0.2.1

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## [Unreleased]
 
+## [0.2.1] - 2021-05-02
+* fixed json reports generation bug, added tests
+
 ## [0.2.0] - 2021-05-02
 * added `--retries` option
 * added `source` feature for sites' mirrors

maigret/maigret.py

@@ -36,7 +36,7 @@ from .sites import MaigretDatabase
 from .submit import submit_dialog
 from .utils import get_dict_ascii_tree
 
-__version__ = '0.2.0'
+__version__ = '0.2.1'
 
 
 def notify_about_errors(search_results, query_notify):

maigret/report.py

@@ -269,6 +269,9 @@ def generate_json_report(username: str, results: dict, file, report_type):
 
         data = dict(site_result)
         data["status"] = data["status"].json()
+        data["site"] = data["site"].json
+        if "future" in data:
+            del data["future"]
 
         if is_report_per_line:
             data["sitename"] = sitename

setup.py

@@ -12,7 +12,7 @@ with open('requirements.txt') as rf:
     requires = rf.read().splitlines()
 
 setup(name='maigret',
-      version='0.2.0',
+      version='0.2.1',
       description='Collect a dossier on a person by username from a huge number of sites',
       long_description=long_description,
       long_description_content_type="text/markdown",

tests/conftest.py

@@ -9,6 +9,7 @@ from maigret.sites import MaigretDatabase
 
 CUR_PATH = os.path.dirname(os.path.realpath(__file__))
 JSON_FILE = os.path.join(CUR_PATH, '../maigret/resources/data.json')
+TEST_JSON_FILE = os.path.join(CUR_PATH, 'db.json')
 empty_mark = Mark('', [], {})
 
 
@@ -38,6 +39,13 @@ def default_db():
     return db
 
 
+@pytest.fixture(scope='function')
+def test_db():
+    db = MaigretDatabase().load_from_file(TEST_JSON_FILE)
+
+    return db
+
+
 @pytest.fixture(autouse=True)
 def reports_autoclean():
     remove_test_reports()

tests/db.json

@@ -0,0 +1,26 @@
+{
+    "engines": {},
+    "sites": {
+        "GooglePlayStore": {
+            "tags": ["global", "us"],
+            "disabled": false,
+            "checkType": "status_code",
+            "alexaRank": 1,
+            "url": "https://play.google.com/store/apps/developer?id={username}",
+            "urlMain": "https://play.google.com/store",
+            "usernameClaimed": "Facebook_nosuchname",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        },
+        "Reddit": {
+            "tags": ["news", "social", "us"],
+            "checkType": "status_code",
+            "presenseStrs": ["totalKarma"],
+            "disabled": true,
+            "alexaRank": 17,
+            "url": "https://www.reddit.com/user/{username}",
+            "urlMain": "https://www.reddit.com/",
+            "usernameClaimed": "blue",
+            "usernameUnclaimed": "noonewouldeverusethis7"
+        }
+    }
+}

tests/test_maigret.py

@@ -4,93 +4,130 @@ import asyncio
 import pytest
 from mock import Mock
 
-from maigret.maigret import self_check
+from maigret.maigret import self_check, maigret
-from maigret.sites import MaigretDatabase
+from maigret.sites import MaigretSite
+from maigret.result import QueryResult, QueryStatus
 
-EXAMPLE_DB = {
+
-    'engines': {},
+@pytest.mark.slow
-    'sites': {
+def test_self_check_db_positive_disable(test_db):
-        "GooglePlayStore": {
+    logger = Mock()
-            "tags": ["global", "us"],
+    assert test_db.sites[0].disabled is False
-            "disabled": False,
+
-            "checkType": "status_code",
+    loop = asyncio.get_event_loop()
-            "alexaRank": 1,
+    loop.run_until_complete(
-            "url": "https://play.google.com/store/apps/developer?id={username}",
+        self_check(test_db, test_db.sites_dict, logger, silent=True)
-            "urlMain": "https://play.google.com/store",
+    )
-            "usernameClaimed": "Facebook_nosuchname",
+
-            "usernameUnclaimed": "noonewouldeverusethis7",
+    assert test_db.sites[0].disabled is True
-        },
+
-        "Reddit": {
+
-            "tags": ["news", "social", "us"],
+@pytest.mark.slow
-            "checkType": "status_code",
+def test_self_check_db_positive_enable(test_db):
-            "presenseStrs": ["totalKarma"],
+    logger = Mock()
-            "disabled": True,
+
-            "alexaRank": 17,
+    test_db.sites[0].disabled = True
-            "url": "https://www.reddit.com/user/{username}",
+    test_db.sites[0].username_claimed = 'Facebook'
-            "urlMain": "https://www.reddit.com/",
+    assert test_db.sites[0].disabled is True
-            "usernameClaimed": "blue",
+
-            "usernameUnclaimed": "noonewouldeverusethis7",
+    loop = asyncio.get_event_loop()
-        },
+    loop.run_until_complete(
-    },
+        self_check(test_db, test_db.sites_dict, logger, silent=True)
+    )
+
+    assert test_db.sites[0].disabled is False
+
+
+@pytest.mark.slow
+def test_self_check_db_negative_disabled(test_db):
+    logger = Mock()
+
+    test_db.sites[0].disabled = True
+    assert test_db.sites[0].disabled is True
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(
+        self_check(test_db, test_db.sites_dict, logger, silent=True)
+    )
+
+    assert test_db.sites[0].disabled is True
+
+
+@pytest.mark.slow
+def test_self_check_db_negative_enabled(test_db):
+    logger = Mock()
+
+    test_db.sites[0].disabled = False
+    test_db.sites[0].username_claimed = 'Facebook'
+    assert test_db.sites[0].disabled is False
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(
+        self_check(test_db, test_db.sites_dict, logger, silent=True)
+    )
+
+    assert test_db.sites[0].disabled is False
+
+
+@pytest.mark.slow
+def test_maigret_results(test_db):
+    logger = Mock()
+
+    username = 'Facebook'
+    loop = asyncio.get_event_loop()
+    results = loop.run_until_complete(
+        maigret(username, site_dict=test_db.sites_dict, logger=logger, timeout=30)
+    )
+
+    assert isinstance(results, dict)
+
+    reddit_site = results['Reddit']['site']
+    assert isinstance(reddit_site, MaigretSite)
+
+    assert reddit_site.json == {
+        'tags': ['news', 'social', 'us'],
+        'checkType': 'status_code',
+        'presenseStrs': ['totalKarma'],
+        'disabled': True,
+        'alexaRank': 17,
+        'url': 'https://www.reddit.com/user/{username}',
+        'urlMain': 'https://www.reddit.com/',
+        'usernameClaimed': 'blue',
+        'usernameUnclaimed': 'noonewouldeverusethis7',
     }
 
+    del results['Reddit']['site']
+    del results['GooglePlayStore']['site']
 
-@pytest.mark.slow
+    reddit_status = results['Reddit']['status']
-def test_self_check_db_positive_disable():
+    assert isinstance(reddit_status, QueryResult)
-    logger = Mock()
+    assert reddit_status.status == QueryStatus.ILLEGAL
-    db = MaigretDatabase()
-    db.load_from_json(EXAMPLE_DB)
 
-    assert db.sites[0].disabled == False
+    playstore_status = results['GooglePlayStore']['status']
+    assert isinstance(playstore_status, QueryResult)
+    assert playstore_status.status == QueryStatus.CLAIMED
 
-    loop = asyncio.get_event_loop()
+    del results['Reddit']['status']
-    loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True))
+    del results['GooglePlayStore']['status']
 
-    assert db.sites[0].disabled == True
+    assert results['Reddit'].get('future') is None
+    del results['GooglePlayStore']['future']
 
-
+    assert results == {
-@pytest.mark.slow
+        'Reddit': {
-def test_self_check_db_positive_enable():
+            'cookies': None,
-    logger = Mock()
+            'parsing_enabled': False,
-    db = MaigretDatabase()
+            'url_main': 'https://www.reddit.com/',
-    db.load_from_json(EXAMPLE_DB)
+            'username': 'Facebook',
-
+        },
-    db.sites[0].disabled = True
+        'GooglePlayStore': {
-    db.sites[0].username_claimed = 'Facebook'
+            'cookies': None,
-    assert db.sites[0].disabled == True
+            'http_status': 200,
-
+            'is_similar': False,
-    loop = asyncio.get_event_loop()
+            'parsing_enabled': False,
-    loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True))
+            'rank': 1,
-
+            'url_main': 'https://play.google.com/store',
-    assert db.sites[0].disabled == False
+            'url_user': 'https://play.google.com/store/apps/developer?id=Facebook',
-
+            'username': 'Facebook',
-
+        },
-@pytest.mark.slow
+    }
-def test_self_check_db_negative_disabled():
-    logger = Mock()
-    db = MaigretDatabase()
-    db.load_from_json(EXAMPLE_DB)
-
-    db.sites[0].disabled = True
-    assert db.sites[0].disabled == True
-
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True))
-
-    assert db.sites[0].disabled == True
-
-
-@pytest.mark.slow
-def test_self_check_db_negative_enabled():
-    logger = Mock()
-    db = MaigretDatabase()
-    db.load_from_json(EXAMPLE_DB)
-
-    db.sites[0].disabled = False
-    db.sites[0].username_claimed = 'Facebook'
-    assert db.sites[0].disabled == False
-
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True))
-
-    assert db.sites[0].disabled == False

tests/test_report.py

@@ -18,6 +18,11 @@ from maigret.report import (
     generate_json_report,
 )
 from maigret.result import QueryResult, QueryStatus
+from maigret.sites import MaigretSite
+
+
+GOOD_RESULT = QueryResult('', '', '', QueryStatus.CLAIMED)
+BAD_RESULT = QueryResult('', '', '', QueryStatus.AVAILABLE)
 
 EXAMPLE_RESULTS = {
     'GitHub': {
@@ -35,12 +40,10 @@ EXAMPLE_RESULTS = {
         'http_status': 200,
         'is_similar': False,
         'rank': 78,
+        'site': MaigretSite('test', {}),
     }
 }
 
-GOOD_RESULT = QueryResult('', '', '', QueryStatus.CLAIMED)
-BAD_RESULT = QueryResult('', '', '', QueryStatus.AVAILABLE)
-
 GOOD_500PX_RESULT = copy.deepcopy(GOOD_RESULT)
 GOOD_500PX_RESULT.tags = ['photo', 'us', 'global']
 GOOD_500PX_RESULT.ids_data = {