Merge pull request #28 from soxoj/reports

Reports refactoring & improving

maigret/__main__.py

@@ -5,8 +5,8 @@ Maigret entrypoint
 """
 
 import asyncio
-import maigret
 
+import maigret
 
 if __name__ == "__main__":
     asyncio.run(maigret.main())

maigret/maigret.py

@@ -2,6 +2,7 @@
 Maigret main module
 """
 
+import aiohttp
 import asyncio
 import csv
 import http.cookiejar as cookielib
@@ -10,26 +11,24 @@ import logging
 import os
 import platform
 import re
+import requests
 import ssl
 import sys
+import tqdm.asyncio
+import xmind
+from aiohttp_socks import ProxyConnector
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
 from http.cookies import SimpleCookie
-
-import aiohttp
-from aiohttp_socks import ProxyConnector
-from python_socks import _errors as proxy_errors
-import requests
-import tqdm.asyncio
 from mock import Mock
-from socid_extractor import parse, extract
+from python_socks import _errors as proxy_errors
+from socid_extractor import parse, extract, __version__ as socid_version
 
 from .activation import ParsingActivator
 from .notify import QueryNotifyPrint
+from .report import save_csv_report, save_xmind_report, save_html_report, save_pdf_report, \
+                    generate_report_context, save_txt_report
 from .result import QueryResult, QueryStatus
 from .sites import MaigretDatabase, MaigretSite
-from .report import save_csv_report, genxmindfile, save_html_pdf_report
-
-import xmind
 
 __version__ = '0.1.10'
 
@@ -517,7 +516,7 @@ def timeout_check(value):
     return timeout
 
 
-async def site_self_check(site, logger, semaphore, db: MaigretDatabase, no_progressbar=False):
+async def site_self_check(site, logger, semaphore, db: MaigretDatabase, silent=False):
     query_notify = Mock()
     changes = {
         'disabled': False,
@@ -579,13 +578,14 @@ async def site_self_check(site, logger, semaphore, db: MaigretDatabase, no_progr
     if changes['disabled'] != site.disabled:
         site.disabled = changes['disabled']
         db.update_site(site)
-        action = 'Disabled' if not site.disabled else 'Enabled'
+        if not silent:
-        print(f'{action} site {site.name}...')
+            action = 'Disabled' if not site.disabled else 'Enabled'
+            print(f'{action} site {site.name}...')
 
     return changes
 
 
-async def self_check(db: MaigretDatabase, site_data: dict, logger):
+async def self_check(db: MaigretDatabase, site_data: dict, logger, silent=False):
     sem = asyncio.Semaphore(10)
     tasks = []
     all_sites = site_data
@@ -596,7 +596,7 @@ async def self_check(db: MaigretDatabase, site_data: dict, logger):
     disabled_old_count = disabled_count(all_sites.values())
 
     for _, site in all_sites.items():
-        check_coro = site_self_check(site, logger, sem, db)
+        check_coro = site_self_check(site, logger, sem, db, silent)
         future = asyncio.ensure_future(check_coro)
         tasks.append(future)
 
@@ -612,13 +612,18 @@ async def self_check(db: MaigretDatabase, site_data: dict, logger):
         message = 'Enabled'
         total_disabled *= -1
 
-    print(f'{message} {total_disabled} checked sites. Run with `--info` flag to get more information')
+    if not silent:
+        print(f'{message} {total_disabled} checked sites. Run with `--info` flag to get more information')
 
 
 async def main():
-    version_string = f"%(prog)s {__version__}\n" + \
+    version_string = '\n'.join([
-                     f"{requests.__description__}:  {requests.__version__}\n" + \
+        f'%(prog)s {__version__}',
-                     f"Python:  {platform.python_version()}"
+        f'Socid-extractor:  {socid_version}',
+        f'Aiohttp:  {aiohttp.__version__}',
+        f'Requests:  {requests.__version__}',
+        f'Python:  {platform.python_version()}',
+    ])
 
     parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,
                             description=f"Maigret v{__version__}"
@@ -627,7 +632,7 @@ async def main():
                         action="version", version=version_string,
                         help="Display version information and dependencies."
                         )
-    parser.add_argument("--info",
+    parser.add_argument("--info", "-vv",
                         action="store_true", dest="info", default=False,
                         help="Display service information."
                         )
@@ -635,21 +640,10 @@ async def main():
                         action="store_true", dest="verbose", default=False,
                         help="Display extra information and metrics."
                         )
-    parser.add_argument("-d", "--debug",
+    parser.add_argument("-d", "--debug", "-vvv",
                         action="store_true", dest="debug", default=False,
                         help="Saving debugging information and sites responses in debug.txt."
                         )
-    parser.add_argument("--folderoutput", "-fo", dest="folderoutput", default="reports",
-                        help="If using multiple usernames, the output of the results will be saved to this folder."
-                        )
-    parser.add_argument("--csv",
-                        action="store_true", dest="csv", default=False,
-                        help="Create Comma-Separated Values (CSV) File."
-                        )
-    parser.add_argument("--html",
-                        action="store_true", dest="html", default=False,
-                        help="Create HTML report file."
-                        )
     parser.add_argument("--site",
                         action="append", metavar='SITE_NAME',
                         dest="site_list", default=[],
@@ -715,17 +709,31 @@ async def main():
                         dest="tags", default='',
                         help="Specify tags of sites."
                         )
-
+    # reports options
-    parser.add_argument("-x","--xmind",
+    parser.add_argument("--folderoutput", "-fo", dest="folderoutput", default="reports",
+                        help="If using multiple usernames, the output of the results will be saved to this folder."
+                        )
+    parser.add_argument("-T", "--txt",
+                        action="store_true", dest="txt", default=False,
+                        help="Create a TXT report (one report per username)."
+                        )
+    parser.add_argument("-C", "--csv",
+                        action="store_true", dest="csv", default=False,
+                        help="Create a CSV report (one report per username)."
+                        )
+    parser.add_argument("-H", "--html",
+                        action="store_true", dest="html", default=False,
+                        help="Create an HTML report file (general report on all usernames)."
+                        )
+    parser.add_argument("-X","--xmind",
                         action="store_true",
                         dest="xmind", default=False,
-                        help="Generate an xmind 8 mindmap"
+                        help="Generate an XMind 8 mindmap report (one report per username)."
                         )
-
     parser.add_argument("-P", "--pdf",
                         action="store_true",
                         dest="pdf", default=False,
-                        help="Generate a pdf report"
+                        help="Generate a PDF report (general report on all usernames)."
                         )
 
     args = parser.parse_args()
@@ -802,6 +810,13 @@ async def main():
         else:
             print('Updates will be applied only for current search session.')
 
+    # Make reports folder is not exists
+    os.makedirs(args.folderoutput, exist_ok=True)
+    report_path = args.folderoutput
+
+    # Define one report filename template
+    report_filepath_tpl = os.path.join(args.folderoutput, 'report_{username}{postfix}')
+
     # Database consistency
     enabled_count = len(list(filter(lambda x: not x.disabled, site_data.values())))
     print(f'Sites in database, enabled/total: {enabled_count}/{len(site_data)}')
@@ -855,51 +870,54 @@ async def main():
                                 logger=logger,
                                 forced=args.use_disabled_sites,
                                 )
+
+        username_result = (username, id_type, results)
         general_results.append((username, id_type, results))
 
-        if args.folderoutput:
+        # TODO: tests
-            # The usernames results should be stored in a targeted folder.
+        for website_name in results:
-            # If the folder doesn't exist, create it first
+            dictionary = results[website_name]
-            os.makedirs(args.folderoutput, exist_ok=True)
+            # TODO: fix no site data issue
-            result_path = os.path.join(args.folderoutput, f"{username}.")
+            if not dictionary:
-        else:
+                continue
-            result_path = os.path.join("reports", f"{username}.")
+            new_usernames = dictionary.get('ids_usernames')
+            if new_usernames:
+                for u, utype in new_usernames.items():
+                    usernames[u] = utype
 
+        # reporting for a one username
         if args.xmind:
-            genxmindfile(result_path+"xmind", username, results)
+            filename = report_filepath_tpl.format(username=username, postfix='.xmind')
-
+            save_xmind_report(filename, username, results)
-
+            print(f'XMind report for {username} saved in {filename}')
-        with open(result_path+"txt", "w", encoding="utf-8") as file:
-            exists_counter = 0
-            for website_name in results:
-                dictionary = results[website_name]
-                # TODO: fix no site data issue
-                if not dictionary:
-                    continue
-                new_usernames = dictionary.get('ids_usernames')
-                if new_usernames:
-                    for u, utype in new_usernames.items():
-                        usernames[u] = utype
-
-                if dictionary.get("status").status == QueryStatus.CLAIMED:
-                    exists_counter += 1
-                    file.write(dictionary["url_user"] + "\n")
-            file.write(f"Total Websites Username Detected On : {exists_counter}")
-            file.close()
 
         if args.csv:
-            save_csv_report(username, results, result_path+"csv")
+            filename = report_filepath_tpl.format(username=username, postfix='.csv')
+            save_csv_report(filename, username, results)
+            print(f'CSV report for {username} saved in {filename}')
 
-        pathPDF = None
+        if args.txt:
-        pathHTML = None
+            filename = report_filepath_tpl.format(username=username, postfix='.txt')
-        if args.html:
+            save_txt_report(filename, username, results)
-            pathHTML = result_path+"html"
+            print(f'TXT report for {username} saved in {filename}')
-        if args.pdf:
-            pathPDF = result_path+"pdf"
 
-        if pathPDF or pathHTML:
+    # reporting for all the result
-            save_html_pdf_report(general_results,pathHTML,pathPDF)
+    report_context = generate_report_context(general_results)
+    # determine main username
+    username = report_context['username']
 
+    if args.html:
+        filename = report_filepath_tpl.format(username=username, postfix='.html')
+        save_html_report(filename, report_context)
+        print(f'HTML report on all usernames saved in {filename}')
+
+    if args.pdf:
+        filename = report_filepath_tpl.format(username=username, postfix='.pdf')
+        save_pdf_report(filename, report_context)
+        print(f'PDF report on all usernames saved in {filename}')
+
+
+    # update database
     db.save_to_file(args.json_file)
 
 

maigret/notify.py

@@ -4,8 +4,8 @@ This module defines the objects for notifying the caller about the
 results of queries.
 """
 import sys
-
 from colorama import Fore, Style, init
+
 from .result import QueryStatus
 
 

maigret/report.py

@@ -1,56 +1,83 @@
 import csv
-from datetime import datetime
+import io
 import logging
 import os
-import xmind
-import io
-
-from xhtml2pdf import pisa
-from jinja2 import Template
-
 import pycountry
+import xmind
+from datetime import datetime
+from jinja2 import Template
+from xhtml2pdf import pisa
+from dateutil.parser import parse as parse_datetime_str
 
 from .result import QueryStatus
 from .utils import is_country_tag, CaseConverter, enrich_link_str
 
-def save_csv_report(username: str, results: dict, filename:str):
-    with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
-        save_csv_report_to_file(username, results, csvfile)
 
-def retrive_timestamp(datestring:str):
+'''
-    first_seen_format = '%Y-%m-%d %H:%M:%S'
+UTILS
-    first_seen_formats = '%Y-%m-%dT%H:%M:%S'
+'''
-    try:
+def filter_supposed_data(data):
-        time = datetime.strptime(datestring, first_seen_format)
-    except:
-        try:
-            time = datetime.strptime(datestring, first_seen_formats)
-        except:
-            time = datetime.min
-    return time
-
-def filterSupposedData(data):
     ### interesting fields
-    allowed_fields = ['fullname', 'gender', 'location']
+    allowed_fields = ['fullname', 'gender', 'location', 'age']
     filtered_supposed_data = {CaseConverter.snake_to_title(k): v[0]
                               for k, v in data.items()
                               if k in allowed_fields}
     return filtered_supposed_data
 
-def generate_template(pdf:bool):
+
-    # template generation
+'''
-    if(pdf):
+REPORTS SAVING
-        template_text = open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
+'''
-                                      "resources/simple_report_pdf.tpl")).read()
+def save_csv_report(filename: str, username: str, results: dict):
+    with open(filename, 'w', newline='', encoding='utf-8') as f:
+        generate_csv_report(username, results, f)
+
+
+def save_txt_report(filename: str, username: str, results: dict):
+    with open(filename, 'w', encoding='utf-8') as f:
+        generate_txt_report(username, results, f)
+
+
+def save_html_report(filename: str, context: dict):
+    template, _ = generate_report_template(is_pdf=False)
+    filled_template = template.render(**context)
+    with open(filename, 'w') as f:
+        f.write(filled_template)
+
+
+def save_pdf_report(filename: str, context: dict):
+    template, css = generate_report_template(is_pdf=True)
+    filled_template = template.render(**context)
+    with open(filename, 'w+b') as f:
+        pisa.pisaDocument(io.StringIO(filled_template), dest=f, default_css=css)
+
+
+'''
+REPORTS GENERATING
+'''
+def generate_report_template(is_pdf: bool):
+    """
+        HTML/PDF template generation
+    """
+    def get_resource_content(filename):
+        return open(os.path.join(maigret_path, 'resources', filename)).read()
+
+    maigret_path = os.path.dirname(os.path.realpath(__file__))
+
+    if is_pdf:
+        template_content = get_resource_content('simple_report_pdf.tpl')
+        css_content = get_resource_content('simple_report_pdf.css')
     else:
-        template_text = open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
+        template_content = get_resource_content('simple_report.tpl')
-                                          "resources/simple_report.tpl")).read()
+        css_content = None
-    template = Template(template_text)
+
+    template = Template(template_content)
     template.globals['title'] = CaseConverter.snake_to_title
     template.globals['detect_link'] = enrich_link_str
-    return template
+    return template, css_content
 
-def save_html_pdf_report(username_results: list, filename:str=None, filenamepdf:str=None):
+
+def generate_report_context(username_results: list):
     brief_text = []
     usernames = {}
     extended_info_count = 0
@@ -84,10 +111,13 @@ def save_html_pdf_report(username_results: list, filename:str=None, filenamepdf:
                     if first_seen is None:
                         first_seen = created_at
                     else:
-                        known_time = retrive_timestamp(first_seen)
+                        try:
-                        new_time = retrive_timestamp(created_at)
+                            known_time = parse_datetime_str(first_seen)
-                        if new_time < known_time:
+                            new_time = parse_datetime_str(created_at)
-                            first_seen = created_at
+                            if new_time < known_time:
+                                first_seen = created_at
+                        except:
+                            logging.debug('Problems with converting datetime %s/%s', first_seen, created_at)
 
                 for k, v in status.ids_data.items():
                     # suppose target data
@@ -149,52 +179,21 @@ def save_html_pdf_report(username_results: list, filename:str=None, filenamepdf:
     countries_lists = list(filter(lambda x: is_country_tag(x[0]), tags.items()))
     interests_list = list(filter(lambda x: not is_country_tag(x[0]), tags.items()))
 
-    filtered_supposed_data = filterSupposedData(supposed_data)
+    filtered_supposed_data = filter_supposed_data(supposed_data)
 
-    # save report in HTML
+    return {
-    if(filename is not None):
+        'username': first_username,
-        template =  generate_template(False)
+        'brief': brief,
-        filled_template = template.render(username=first_username,
+        'results': username_results,
-                                          brief=brief,
+        'first_seen': first_seen,
-                                          results=username_results,
+        'interests_tuple_list': tuple_sort(interests_list),
-                                          first_seen=first_seen,
+        'countries_tuple_list': tuple_sort(countries_lists),
-                                          interests_tuple_list=tuple_sort(interests_list),
+        'supposed_data': filtered_supposed_data,
-                                          countries_tuple_list=tuple_sort(countries_lists),
+        'generated_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-                                          supposed_data=filtered_supposed_data,
+    }
-                                          generated_at=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-                                          )
-        with open(filename, 'w') as f:
-            f.write(filled_template)
-            f.close()
-    # save report in PDF
-    if(filenamepdf is not None):
-        template = generate_template(True)
-        filled_template = template.render(username=first_username,
-                                          brief=brief,
-                                          results=username_results,
-                                          first_seen=first_seen,
-                                          interests_tuple_list=tuple_sort(interests_list),
-                                          countries_tuple_list=tuple_sort(countries_lists),
-                                          supposed_data=filtered_supposed_data,
-                                          generated_at=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-                                          )
-        csstext = ""
-        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
-                                      "resources/simple_report_pdf.css"), "r") as cssfile:
-            cssline = cssfile.readline()
-            csstext += cssline
-            while cssline:
-                cssline = cssfile.readline()
-                csstext += cssline
-            cssfile.close()
-
-        pdffile = open(filenamepdf, "w+b")
-        pisa.pisaDocument(io.StringIO(filled_template), dest=pdffile, default_css=csstext)
-        pdffile.close()
 
 
-def save_csv_report_to_file(username: str, results: dict, csvfile):
+def generate_csv_report(username: str, results: dict, csvfile):
-    print(results)
     writer = csv.writer(csvfile)
     writer.writerow(['username',
                      'name',
@@ -213,11 +212,23 @@ def save_csv_report_to_file(username: str, results: dict, csvfile):
                          results[site]['http_status'],
                         ])
 
+
+def generate_txt_report(username: str, results: dict, file):
+    exists_counter = 0
+    for website_name in results:
+        dictionary = results[website_name]
+        # TODO: fix no site data issue
+        if not dictionary:
+            continue
+        if dictionary.get("status").status == QueryStatus.CLAIMED:
+            exists_counter += 1
+            file.write(dictionary["url_user"] + "\n")
+    file.write(f'Total Websites Username Detected On : {exists_counter}')
+
 '''
 XMIND 8 Functions
 '''
-def genxmindfile(filename, username, results):
+def save_xmind_report(filename, username, results):
-    print(f'Generating XMIND8 file for username {username}')
     if os.path.exists(filename):
         os.remove(filename)
     workbook = xmind.load(filename)
@@ -286,7 +297,7 @@ def design_sheet(sheet, username, results):
                             supposed_data[field].append(currentval)
                             currentsublabel.setTitle("%s: %s" % (k, currentval))
     ### Add Supposed DATA
-    filterede_supposed_data = filterSupposedData(supposed_data)
+    filterede_supposed_data = filter_supposed_data(supposed_data)
     if(len(filterede_supposed_data) >0):
         undefinedsection = root_topic1.addSubTopic()
         undefinedsection.setTitle("SUPPOSED DATA")

maigret/resources/data.json

@@ -8901,12 +8901,14 @@
             "usernameClaimed": "red",
             "usernameUnclaimed": "noonewouldeverusethis7"
         },
-        "NameMC (Minecraft.net skins)": {
+        "NameMC": {
             "tags": [
                 "us"
             ],
+            "regexCheck": "^.{3,16}$",
             "checkType": "message",
-            "absenceStrs": "Profiles: 0 results",
+            "presenseStrs": "/profile/",
+            "absenceStrs": "<div class=\"col-lg-5 order-lg-2\">\n    </div>",
             "alexaRank": 10151,
             "url": "https://namemc.com/profile/{username}",
             "urlMain": "https://namemc.com/",
@@ -13416,15 +13418,11 @@
                 "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"",
                 "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
                 "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
-                "x-guest-token": "1349509919867854849"
+                "x-guest-token": "1350185014307254275"
             },
             "errors": {
                 "Bad guest token": "x-guest-token update required"
             },
-            "urlProbe": "https://twitter.com/i/api/graphql/ZRnOhhXPwue_JGILb9TNug/UserByScreenName?variables=%7B%22screen_name%22%3A%22{username}%22%2C%22withHighlightedLabel%22%3Atrue%7D",
-            "checkType": "message",
-            "absenceStrs": "Not found",
-            "alexaRank": 55,
             "activation": {
                 "method": "twitter",
                 "marks": [
@@ -13434,6 +13432,10 @@
                 "src": "guest_token",
                 "dst": "x-guest-token"
             },
+            "urlProbe": "https://twitter.com/i/api/graphql/ZRnOhhXPwue_JGILb9TNug/UserByScreenName?variables=%7B%22screen_name%22%3A%22{username}%22%2C%22withHighlightedLabel%22%3Atrue%7D",
+            "checkType": "message",
+            "absenceStrs": "Not found",
+            "alexaRank": 55,
             "url": "https://twitter.com/{username}",
             "urlMain": "https://www.twitter.com/",
             "usernameClaimed": "blue",

maigret/resources/simple_report.tpl

@@ -20,7 +20,7 @@
                 <h4 class="mb-0">
                     <a class="blog-header-logo text-dark" href="#">Username search report for {{ username }}</a>
                 </h4>
-                <small class="text-muted">Generated at {{ generated_at }}</small>
+                <small class="text-muted">Generated by <a href="https://github.com/soxoj/maigret">Maigret</a> at {{ generated_at }}</small>
             </div>
         </div>
         <div class="row-mb">

maigret/resources/simple_report_pdf.tpl

@@ -11,9 +11,10 @@
                 <h2 class="mb-0">
                     Username search report for {{ username }}
                 </h2>
-                <small>Generated at {{ generated_at }}</small>
+                <small>Generated by <a href="https://github.com/soxoj/maigret">Maigret</a> at {{ generated_at }}</small>
             </div>
         </div>
+        <br/><br/>
         <div>
                 <div>
                     <div>
@@ -41,6 +42,7 @@
                     </div>
                 </div>
             </div>
+            <br/>
             <div>
                 <div>
                     <div>
@@ -57,16 +59,13 @@
                 {% for k, v in data.items() %}
                     {% if v.found and not v.is_similar %}
             <split></split>
+            <hr>
             <br/>
             <div class="sitebox" style="margin-top: 20px;" >
                 <div>
                     <div>
                         <table>
                             <tr>
-                                <td style="width:201px;" >
-                                    <img   alt="Photo" style="width: 200px; height: 200px; object-fit: scale-down;" src="{{ v.status.ids_data.image or 'https://i.imgur.com/040fmbw.png' }}" data-holder-rendered="true">
-                                </td>
-                                <td style="width:10px;" ></td>
                                 <td valign="top">
                                     <div class="textbox" style="padding-top: 10px;" >
                                         <h3>
@@ -79,29 +78,32 @@
                                             <a href="{{ v.url_user }}" target="_blank">{{ v.url_user }}</a>
                                         </p>
                                     </div>
+                                    {% if v.ids_data %}
+                                    <div style="clear:both;"></div>
+                                    <div style="width:100%">
+                                    <br/>
+                                    <h4>Details</h4>
+                                    <table class="table table-striped;" style="margin-top:5px;">
+                                        <tbody>
+                                        {% for k1, v1 in v.ids_data.items() %}
+                                            {% if k1 != 'image' %}
+                                            <tr>
+                                                <th style="width:200px;">{{ title(k1) }}</th>
+                                                <td>{% if v1 is iterable and (v1 is not string and v1 is not mapping) %}{{ v1 | join(', ') }}{% else %}{{ detect_link(v1) }}{% endif %}</td>
+                                            </tr>
+                                            {% endif %}
+
+                                        {% endfor %}
+                                        </tbody>
+                                    </table>
+                                    </div>
+                                    {% endif %}
+                                </td>
+                                <td style="width:201px; position: relative;" valign="top">
+                                    <img alt="Photo" style="width: 200px; height: 200px; object-fit: scale-down;" src="{{ v.status.ids_data.image or 'https://i.imgur.com/040fmbw.png' }}" data-holder-rendered="true">
                                 </td>
                             </tr>
                         </table>
-                        {% if v.ids_data %}
-                        <div style="clear:both;"></div>
-                        <div style="width:100%">
-                        <br/>
-                        <h4>Details</h4>
-                        <table class="table table-striped;" style="margin-top:5px;">
-                            <tbody>
-                            {% for k1, v1 in v.ids_data.items() %}
-                                {% if k1 != 'image' %}
-                                <tr>
-                                    <th style="width:100px;">{{ title(k1) }}</th>
-                                    <td>{% if v1 is iterable and (v1 is not string and v1 is not mapping) %}{{ v1 | join(', ') }}{% else %}{{ detect_link(v1) }}{% endif %}</td>
-                                </tr>
-                                {% endif %}
-
-                            {% endfor %}
-                            </tbody>
-                        </table>
-                        </div>
-                        {% endif %}
                     </div>
                 </div>
             </div>

maigret/sites.py

@@ -3,9 +3,8 @@
 import copy
 import json
 import operator
-import sys
-
 import requests
+import sys
 
 from .utils import CaseConverter
 

requirements.txt

@@ -8,6 +8,7 @@ bs4==0.0.1
 certifi==2020.12.5
 chardet==3.0.4
 colorama==0.4.4
+python-dateutil==2.8.1
 future==0.18.2
 future-annotations==1.0.0
 html5lib==1.1
@@ -27,7 +28,7 @@ reportlab==3.5.59
 requests==2.25.1
 requests-futures==1.0.0
 six==1.15.0
-socid-extractor==0.0.2
+socid-extractor>0.0.2
 soupsieve==2.1
 stem==1.8.0
 torrequest==0.1.0

tests/conftest.py

@@ -1,11 +1,14 @@
-from _pytest.mark import Mark
+import glob
-from mock import Mock
+import logging
 import os
 import pytest
+from _pytest.mark import Mark
+from mock import Mock
 
 from maigret.sites import MaigretDatabase, MaigretSite
 
-JSON_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../maigret/resources/data.json')
+CUR_PATH = os.path.dirname(os.path.realpath(__file__))
+JSON_FILE = os.path.join(CUR_PATH, '../maigret/resources/data.json')
 empty_mark = Mark('', [], {})
 
 
@@ -17,8 +20,25 @@ def pytest_collection_modifyitems(items):
     items.sort(key=by_slow_marker, reverse=False)
 
 
+def get_test_reports_filenames():
+    return glob.glob(os.path.join('report_*'), recursive=False)
+
+
+def remove_test_reports():
+    reports_list = get_test_reports_filenames()
+    for f in reports_list: os.remove(f)
+    logging.error(f'Removed test reports {reports_list}')
+
+
 @pytest.fixture(scope='session')
 def default_db():
     db = MaigretDatabase().load_from_file(JSON_FILE)
 
     return db
+
+
+@pytest.fixture(autouse=True)
+def reports_autoclean():
+    remove_test_reports()
+    yield
+    remove_test_reports()

tests/test_activation.py

@@ -1,6 +1,6 @@
 """Maigret activation test functions"""
-from mock import Mock
 import pytest
+from mock import Mock
 
 from maigret.activation import ParsingActivator
 

tests/test_maigret.py

@@ -1,11 +1,10 @@
 """Maigret main module test functions"""
 import asyncio
-from mock import Mock
 import pytest
+from mock import Mock
 
-from maigret.sites import MaigretDatabase, MaigretSite
 from maigret.maigret import self_check
-
+from maigret.sites import MaigretDatabase, MaigretSite
 
 EXAMPLE_DB = {
     'engines': {
@@ -54,7 +53,7 @@ def test_self_check_db_positive_disable():
     assert db.sites[0].disabled == False
 
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(self_check(db, db.sites_dict, logger))
+    loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True))
 
     assert db.sites[0].disabled == True
 
@@ -70,7 +69,7 @@ def test_self_check_db_positive_enable():
     assert db.sites[0].disabled == True
 
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(self_check(db, db.sites_dict, logger))
+    loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True))
 
     assert db.sites[0].disabled == False
 
@@ -85,7 +84,7 @@ def test_self_check_db_negative_disabled():
     assert db.sites[0].disabled == True
 
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(self_check(db, db.sites_dict, logger))
+    loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True))
 
     assert db.sites[0].disabled == True
 
@@ -101,6 +100,6 @@ def test_self_check_db_negative_enabled():
     assert db.sites[0].disabled == False
 
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(self_check(db, db.sites_dict, logger))
+    loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True))
 
     assert db.sites[0].disabled == False

tests/test_report.py

@@ -1,14 +1,15 @@
 """Maigret reports test functions"""
-from io import StringIO
 import copy
 import os
+from io import StringIO
 
 import xmind
+from jinja2 import Template
 
-from maigret.report import save_csv_report_to_file, genxmindfile, save_html_pdf_report
+from maigret.report import generate_csv_report, generate_txt_report, save_xmind_report, save_html_report, \
+    save_pdf_report, generate_report_template, generate_report_context
 from maigret.result import QueryResult, QueryStatus
 
-
 EXAMPLE_RESULTS = {
     'GitHub': {
         'username': 'test',
@@ -56,22 +57,47 @@ SUPPOSED_INTERESTS = "Interests: photo <span class=\"text-muted\">(2)</span>, ne
 SUPPOSED_GEO = "Geo: us <span class=\"text-muted\">(3)</span>"
 
 
-def test_save_csv_report_to_file():
+def test_generate_report_template():
+    report_template, css = generate_report_template(is_pdf=True)
+
+    assert isinstance(report_template, Template)
+    assert isinstance(css, str)
+
+    report_template, css = generate_report_template(is_pdf=False)
+
+    assert isinstance(report_template, Template)
+    assert css is None
+
+
+def test_generate_csv_report():
     csvfile = StringIO()
-    save_csv_report_to_file('test', EXAMPLE_RESULTS, csvfile)
+    generate_csv_report('test', EXAMPLE_RESULTS, csvfile)
 
     csvfile.seek(0)
     data = csvfile.readlines()
 
     assert data == [
-         'username,name,url_main,url_user,exists,http_status\r\n',
+        'username,name,url_main,url_user,exists,http_status\r\n',
-         'test,GitHub,https://www.github.com/,https://www.github.com/test,Claimed,200\r\n',
+        'test,GitHub,https://www.github.com/,https://www.github.com/test,Claimed,200\r\n',
+    ]
+
+
+def test_generate_txt_report():
+    txtfile = StringIO()
+    generate_txt_report('test', EXAMPLE_RESULTS, txtfile)
+
+    txtfile.seek(0)
+    data = txtfile.readlines()
+
+    assert data == [
+        'https://www.github.com/test\n',
+        'Total Websites Username Detected On : 1',
     ]
 
 
 def test_save_xmind_report():
-    filename = 'test_report.xmind'
+    filename = 'report_test.xmind'
-    genxmindfile(filename, 'test', EXAMPLE_RESULTS)
+    save_xmind_report(filename, 'test', EXAMPLE_RESULTS)
 
     workbook = xmind.load(filename)
     sheet = workbook.getPrimarySheet()
@@ -87,14 +113,9 @@ def test_save_xmind_report():
 
 
 def test_html_report():
-    report_name = 'report_alexaimephotographycars.html'
+    report_name = 'report_test.html'
-    try:
+    context = generate_report_context(TEST)
-        os.remove(report_name)
+    save_html_report(report_name, context)
-    except:
-        pass
-
-    save_html_pdf_report(TEST,filename=report_name,filenamepdf=None)
-    assert os.path.exists(report_name)
 
     report_text = open(report_name).read()
 
@@ -102,12 +123,10 @@ def test_html_report():
     assert SUPPOSED_GEO in report_text
     assert SUPPOSED_INTERESTS in report_text
 
-def test_pdf_report():
-    report_name_pdf = 'report_alexaimephotographycars.pdf'
-    try:
-        os.remove(report_name_pdf)
-    except:
-        pass
 
-    save_html_pdf_report(TEST,filename=None,filenamepdf=report_name_pdf)
+def test_pdf_report():
-    assert os.path.exists(report_name_pdf)
+    report_name = 'report_test.pdf'
+    context = generate_report_context(TEST)
+    save_pdf_report(report_name, context)
+
+    assert os.path.exists(report_name)