diff --git a/maigret/__main__.py b/maigret/__main__.py index c24c5cf..527d456 100644 --- a/maigret/__main__.py +++ b/maigret/__main__.py @@ -5,8 +5,8 @@ Maigret entrypoint """ import asyncio -import maigret +import maigret if __name__ == "__main__": asyncio.run(maigret.main()) diff --git a/maigret/maigret.py b/maigret/maigret.py index aa4a4be..4f04300 100755 --- a/maigret/maigret.py +++ b/maigret/maigret.py @@ -2,6 +2,7 @@ Maigret main module """ +import aiohttp import asyncio import csv import http.cookiejar as cookielib @@ -10,26 +11,24 @@ import logging import os import platform import re +import requests import ssl import sys +import tqdm.asyncio +import xmind +from aiohttp_socks import ProxyConnector from argparse import ArgumentParser, RawDescriptionHelpFormatter from http.cookies import SimpleCookie - -import aiohttp -from aiohttp_socks import ProxyConnector -from python_socks import _errors as proxy_errors -import requests -import tqdm.asyncio from mock import Mock -from socid_extractor import parse, extract +from python_socks import _errors as proxy_errors +from socid_extractor import parse, extract, __version__ as socid_version from .activation import ParsingActivator from .notify import QueryNotifyPrint +from .report import save_csv_report, save_xmind_report, save_html_report, save_pdf_report, \ + generate_report_context, save_txt_report from .result import QueryResult, QueryStatus from .sites import MaigretDatabase, MaigretSite -from .report import save_csv_report, genxmindfile, save_html_pdf_report - -import xmind __version__ = '0.1.10' @@ -517,7 +516,7 @@ def timeout_check(value): return timeout -async def site_self_check(site, logger, semaphore, db: MaigretDatabase, no_progressbar=False): +async def site_self_check(site, logger, semaphore, db: MaigretDatabase, silent=False): query_notify = Mock() changes = { 'disabled': False, @@ -579,13 +578,14 @@ async def site_self_check(site, logger, semaphore, db: MaigretDatabase, no_progr if changes['disabled'] != site.disabled: site.disabled = changes['disabled'] db.update_site(site) - action = 'Disabled' if not site.disabled else 'Enabled' - print(f'{action} site {site.name}...') + if not silent: + action = 'Disabled' if not site.disabled else 'Enabled' + print(f'{action} site {site.name}...') return changes -async def self_check(db: MaigretDatabase, site_data: dict, logger): +async def self_check(db: MaigretDatabase, site_data: dict, logger, silent=False): sem = asyncio.Semaphore(10) tasks = [] all_sites = site_data @@ -596,7 +596,7 @@ async def self_check(db: MaigretDatabase, site_data: dict, logger): disabled_old_count = disabled_count(all_sites.values()) for _, site in all_sites.items(): - check_coro = site_self_check(site, logger, sem, db) + check_coro = site_self_check(site, logger, sem, db, silent) future = asyncio.ensure_future(check_coro) tasks.append(future) @@ -612,13 +612,18 @@ async def self_check(db: MaigretDatabase, site_data: dict, logger): message = 'Enabled' total_disabled *= -1 - print(f'{message} {total_disabled} checked sites. Run with `--info` flag to get more information') + if not silent: + print(f'{message} {total_disabled} checked sites. Run with `--info` flag to get more information') async def main(): - version_string = f"%(prog)s {__version__}\n" + \ - f"{requests.__description__}: {requests.__version__}\n" + \ - f"Python: {platform.python_version()}" + version_string = '\n'.join([ + f'%(prog)s {__version__}', + f'Socid-extractor: {socid_version}', + f'Aiohttp: {aiohttp.__version__}', + f'Requests: {requests.__version__}', + f'Python: {platform.python_version()}', + ]) parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, description=f"Maigret v{__version__}" @@ -627,7 +632,7 @@ async def main(): action="version", version=version_string, help="Display version information and dependencies." ) - parser.add_argument("--info", + parser.add_argument("--info", "-vv", action="store_true", dest="info", default=False, help="Display service information." ) @@ -635,21 +640,10 @@ async def main(): action="store_true", dest="verbose", default=False, help="Display extra information and metrics." ) - parser.add_argument("-d", "--debug", + parser.add_argument("-d", "--debug", "-vvv", action="store_true", dest="debug", default=False, help="Saving debugging information and sites responses in debug.txt." ) - parser.add_argument("--folderoutput", "-fo", dest="folderoutput", default="reports", - help="If using multiple usernames, the output of the results will be saved to this folder." - ) - parser.add_argument("--csv", - action="store_true", dest="csv", default=False, - help="Create Comma-Separated Values (CSV) File." - ) - parser.add_argument("--html", - action="store_true", dest="html", default=False, - help="Create HTML report file." - ) parser.add_argument("--site", action="append", metavar='SITE_NAME', dest="site_list", default=[], @@ -715,17 +709,31 @@ async def main(): dest="tags", default='', help="Specify tags of sites." ) - - parser.add_argument("-x","--xmind", + # reports options + parser.add_argument("--folderoutput", "-fo", dest="folderoutput", default="reports", + help="If using multiple usernames, the output of the results will be saved to this folder." + ) + parser.add_argument("-T", "--txt", + action="store_true", dest="txt", default=False, + help="Create a TXT report (one report per username)." + ) + parser.add_argument("-C", "--csv", + action="store_true", dest="csv", default=False, + help="Create a CSV report (one report per username)." + ) + parser.add_argument("-H", "--html", + action="store_true", dest="html", default=False, + help="Create an HTML report file (general report on all usernames)." + ) + parser.add_argument("-X","--xmind", action="store_true", dest="xmind", default=False, - help="Generate an xmind 8 mindmap" + help="Generate an XMind 8 mindmap report (one report per username)." ) - parser.add_argument("-P", "--pdf", action="store_true", dest="pdf", default=False, - help="Generate a pdf report" + help="Generate a PDF report (general report on all usernames)." ) args = parser.parse_args() @@ -802,6 +810,13 @@ async def main(): else: print('Updates will be applied only for current search session.') + # Make reports folder is not exists + os.makedirs(args.folderoutput, exist_ok=True) + report_path = args.folderoutput + + # Define one report filename template + report_filepath_tpl = os.path.join(args.folderoutput, 'report_{username}{postfix}') + # Database consistency enabled_count = len(list(filter(lambda x: not x.disabled, site_data.values()))) print(f'Sites in database, enabled/total: {enabled_count}/{len(site_data)}') @@ -855,51 +870,54 @@ async def main(): logger=logger, forced=args.use_disabled_sites, ) + + username_result = (username, id_type, results) general_results.append((username, id_type, results)) - if args.folderoutput: - # The usernames results should be stored in a targeted folder. - # If the folder doesn't exist, create it first - os.makedirs(args.folderoutput, exist_ok=True) - result_path = os.path.join(args.folderoutput, f"{username}.") - else: - result_path = os.path.join("reports", f"{username}.") + # TODO: tests + for website_name in results: + dictionary = results[website_name] + # TODO: fix no site data issue + if not dictionary: + continue + new_usernames = dictionary.get('ids_usernames') + if new_usernames: + for u, utype in new_usernames.items(): + usernames[u] = utype + # reporting for a one username if args.xmind: - genxmindfile(result_path+"xmind", username, results) - - - with open(result_path+"txt", "w", encoding="utf-8") as file: - exists_counter = 0 - for website_name in results: - dictionary = results[website_name] - # TODO: fix no site data issue - if not dictionary: - continue - new_usernames = dictionary.get('ids_usernames') - if new_usernames: - for u, utype in new_usernames.items(): - usernames[u] = utype - - if dictionary.get("status").status == QueryStatus.CLAIMED: - exists_counter += 1 - file.write(dictionary["url_user"] + "\n") - file.write(f"Total Websites Username Detected On : {exists_counter}") - file.close() + filename = report_filepath_tpl.format(username=username, postfix='.xmind') + save_xmind_report(filename, username, results) + print(f'XMind report for {username} saved in {filename}') if args.csv: - save_csv_report(username, results, result_path+"csv") + filename = report_filepath_tpl.format(username=username, postfix='.csv') + save_csv_report(filename, username, results) + print(f'CSV report for {username} saved in {filename}') - pathPDF = None - pathHTML = None - if args.html: - pathHTML = result_path+"html" - if args.pdf: - pathPDF = result_path+"pdf" + if args.txt: + filename = report_filepath_tpl.format(username=username, postfix='.txt') + save_txt_report(filename, username, results) + print(f'TXT report for {username} saved in {filename}') - if pathPDF or pathHTML: - save_html_pdf_report(general_results,pathHTML,pathPDF) + # reporting for all the result + report_context = generate_report_context(general_results) + # determine main username + username = report_context['username'] + if args.html: + filename = report_filepath_tpl.format(username=username, postfix='.html') + save_html_report(filename, report_context) + print(f'HTML report on all usernames saved in {filename}') + + if args.pdf: + filename = report_filepath_tpl.format(username=username, postfix='.pdf') + save_pdf_report(filename, report_context) + print(f'PDF report on all usernames saved in {filename}') + + + # update database db.save_to_file(args.json_file) diff --git a/maigret/notify.py b/maigret/notify.py index 4dc87dc..c5374b9 100644 --- a/maigret/notify.py +++ b/maigret/notify.py @@ -4,8 +4,8 @@ This module defines the objects for notifying the caller about the results of queries. """ import sys - from colorama import Fore, Style, init + from .result import QueryStatus diff --git a/maigret/report.py b/maigret/report.py index 6211995..c847143 100644 --- a/maigret/report.py +++ b/maigret/report.py @@ -1,56 +1,83 @@ import csv -from datetime import datetime +import io import logging import os -import xmind -import io - -from xhtml2pdf import pisa -from jinja2 import Template - import pycountry +import xmind +from datetime import datetime +from jinja2 import Template +from xhtml2pdf import pisa +from dateutil.parser import parse as parse_datetime_str from .result import QueryStatus from .utils import is_country_tag, CaseConverter, enrich_link_str -def save_csv_report(username: str, results: dict, filename:str): - with open(filename, 'w', newline='', encoding='utf-8') as csvfile: - save_csv_report_to_file(username, results, csvfile) -def retrive_timestamp(datestring:str): - first_seen_format = '%Y-%m-%d %H:%M:%S' - first_seen_formats = '%Y-%m-%dT%H:%M:%S' - try: - time = datetime.strptime(datestring, first_seen_format) - except: - try: - time = datetime.strptime(datestring, first_seen_formats) - except: - time = datetime.min - return time - -def filterSupposedData(data): +''' +UTILS +''' +def filter_supposed_data(data): ### interesting fields - allowed_fields = ['fullname', 'gender', 'location'] + allowed_fields = ['fullname', 'gender', 'location', 'age'] filtered_supposed_data = {CaseConverter.snake_to_title(k): v[0] for k, v in data.items() if k in allowed_fields} return filtered_supposed_data -def generate_template(pdf:bool): - # template generation - if(pdf): - template_text = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), - "resources/simple_report_pdf.tpl")).read() + +''' +REPORTS SAVING +''' +def save_csv_report(filename: str, username: str, results: dict): + with open(filename, 'w', newline='', encoding='utf-8') as f: + generate_csv_report(username, results, f) + + +def save_txt_report(filename: str, username: str, results: dict): + with open(filename, 'w', encoding='utf-8') as f: + generate_txt_report(username, results, f) + + +def save_html_report(filename: str, context: dict): + template, _ = generate_report_template(is_pdf=False) + filled_template = template.render(**context) + with open(filename, 'w') as f: + f.write(filled_template) + + +def save_pdf_report(filename: str, context: dict): + template, css = generate_report_template(is_pdf=True) + filled_template = template.render(**context) + with open(filename, 'w+b') as f: + pisa.pisaDocument(io.StringIO(filled_template), dest=f, default_css=css) + + +''' +REPORTS GENERATING +''' +def generate_report_template(is_pdf: bool): + """ + HTML/PDF template generation + """ + def get_resource_content(filename): + return open(os.path.join(maigret_path, 'resources', filename)).read() + + maigret_path = os.path.dirname(os.path.realpath(__file__)) + + if is_pdf: + template_content = get_resource_content('simple_report_pdf.tpl') + css_content = get_resource_content('simple_report_pdf.css') else: - template_text = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), - "resources/simple_report.tpl")).read() - template = Template(template_text) + template_content = get_resource_content('simple_report.tpl') + css_content = None + + template = Template(template_content) template.globals['title'] = CaseConverter.snake_to_title template.globals['detect_link'] = enrich_link_str - return template + return template, css_content -def save_html_pdf_report(username_results: list, filename:str=None, filenamepdf:str=None): + +def generate_report_context(username_results: list): brief_text = [] usernames = {} extended_info_count = 0 @@ -84,10 +111,13 @@ def save_html_pdf_report(username_results: list, filename:str=None, filenamepdf: if first_seen is None: first_seen = created_at else: - known_time = retrive_timestamp(first_seen) - new_time = retrive_timestamp(created_at) - if new_time < known_time: - first_seen = created_at + try: + known_time = parse_datetime_str(first_seen) + new_time = parse_datetime_str(created_at) + if new_time < known_time: + first_seen = created_at + except: + logging.debug('Problems with converting datetime %s/%s', first_seen, created_at) for k, v in status.ids_data.items(): # suppose target data @@ -149,52 +179,21 @@ def save_html_pdf_report(username_results: list, filename:str=None, filenamepdf: countries_lists = list(filter(lambda x: is_country_tag(x[0]), tags.items())) interests_list = list(filter(lambda x: not is_country_tag(x[0]), tags.items())) - filtered_supposed_data = filterSupposedData(supposed_data) + filtered_supposed_data = filter_supposed_data(supposed_data) - # save report in HTML - if(filename is not None): - template = generate_template(False) - filled_template = template.render(username=first_username, - brief=brief, - results=username_results, - first_seen=first_seen, - interests_tuple_list=tuple_sort(interests_list), - countries_tuple_list=tuple_sort(countries_lists), - supposed_data=filtered_supposed_data, - generated_at=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - ) - with open(filename, 'w') as f: - f.write(filled_template) - f.close() - # save report in PDF - if(filenamepdf is not None): - template = generate_template(True) - filled_template = template.render(username=first_username, - brief=brief, - results=username_results, - first_seen=first_seen, - interests_tuple_list=tuple_sort(interests_list), - countries_tuple_list=tuple_sort(countries_lists), - supposed_data=filtered_supposed_data, - generated_at=datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - ) - csstext = "" - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), - "resources/simple_report_pdf.css"), "r") as cssfile: - cssline = cssfile.readline() - csstext += cssline - while cssline: - cssline = cssfile.readline() - csstext += cssline - cssfile.close() - - pdffile = open(filenamepdf, "w+b") - pisa.pisaDocument(io.StringIO(filled_template), dest=pdffile, default_css=csstext) - pdffile.close() + return { + 'username': first_username, + 'brief': brief, + 'results': username_results, + 'first_seen': first_seen, + 'interests_tuple_list': tuple_sort(interests_list), + 'countries_tuple_list': tuple_sort(countries_lists), + 'supposed_data': filtered_supposed_data, + 'generated_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + } -def save_csv_report_to_file(username: str, results: dict, csvfile): - print(results) +def generate_csv_report(username: str, results: dict, csvfile): writer = csv.writer(csvfile) writer.writerow(['username', 'name', @@ -213,11 +212,23 @@ def save_csv_report_to_file(username: str, results: dict, csvfile): results[site]['http_status'], ]) + +def generate_txt_report(username: str, results: dict, file): + exists_counter = 0 + for website_name in results: + dictionary = results[website_name] + # TODO: fix no site data issue + if not dictionary: + continue + if dictionary.get("status").status == QueryStatus.CLAIMED: + exists_counter += 1 + file.write(dictionary["url_user"] + "\n") + file.write(f'Total Websites Username Detected On : {exists_counter}') + ''' XMIND 8 Functions ''' -def genxmindfile(filename, username, results): - print(f'Generating XMIND8 file for username {username}') +def save_xmind_report(filename, username, results): if os.path.exists(filename): os.remove(filename) workbook = xmind.load(filename) @@ -286,7 +297,7 @@ def design_sheet(sheet, username, results): supposed_data[field].append(currentval) currentsublabel.setTitle("%s: %s" % (k, currentval)) ### Add Supposed DATA - filterede_supposed_data = filterSupposedData(supposed_data) + filterede_supposed_data = filter_supposed_data(supposed_data) if(len(filterede_supposed_data) >0): undefinedsection = root_topic1.addSubTopic() undefinedsection.setTitle("SUPPOSED DATA") diff --git a/maigret/resources/data.json b/maigret/resources/data.json index f30deda..1139f95 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -8901,12 +8901,14 @@ "usernameClaimed": "red", "usernameUnclaimed": "noonewouldeverusethis7" }, - "NameMC (Minecraft.net skins)": { + "NameMC": { "tags": [ "us" ], + "regexCheck": "^.{3,16}$", "checkType": "message", - "absenceStrs": "Profiles: 0 results", + "presenseStrs": "/profile/", + "absenceStrs": "
\n
", "alexaRank": 10151, "url": "https://namemc.com/profile/{username}", "urlMain": "https://namemc.com/", @@ -13416,15 +13418,11 @@ "sec-ch-ua": "Google Chrome\";v=\"87\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"87\"", "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA", "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", - "x-guest-token": "1349509919867854849" + "x-guest-token": "1350185014307254275" }, "errors": { "Bad guest token": "x-guest-token update required" }, - "urlProbe": "https://twitter.com/i/api/graphql/ZRnOhhXPwue_JGILb9TNug/UserByScreenName?variables=%7B%22screen_name%22%3A%22{username}%22%2C%22withHighlightedLabel%22%3Atrue%7D", - "checkType": "message", - "absenceStrs": "Not found", - "alexaRank": 55, "activation": { "method": "twitter", "marks": [ @@ -13434,6 +13432,10 @@ "src": "guest_token", "dst": "x-guest-token" }, + "urlProbe": "https://twitter.com/i/api/graphql/ZRnOhhXPwue_JGILb9TNug/UserByScreenName?variables=%7B%22screen_name%22%3A%22{username}%22%2C%22withHighlightedLabel%22%3Atrue%7D", + "checkType": "message", + "absenceStrs": "Not found", + "alexaRank": 55, "url": "https://twitter.com/{username}", "urlMain": "https://www.twitter.com/", "usernameClaimed": "blue", diff --git a/maigret/resources/simple_report.tpl b/maigret/resources/simple_report.tpl index 0908808..df5e216 100644 --- a/maigret/resources/simple_report.tpl +++ b/maigret/resources/simple_report.tpl @@ -20,7 +20,7 @@

- Generated at {{ generated_at }} + Generated by Maigret at {{ generated_at }}
diff --git a/maigret/resources/simple_report_pdf.tpl b/maigret/resources/simple_report_pdf.tpl index aa58ae5..f3db395 100644 --- a/maigret/resources/simple_report_pdf.tpl +++ b/maigret/resources/simple_report_pdf.tpl @@ -11,9 +11,10 @@

Username search report for {{ username }}

- Generated at {{ generated_at }} + Generated by Maigret at {{ generated_at }}
+

@@ -41,6 +42,7 @@
+
@@ -57,16 +59,13 @@ {% for k, v in data.items() %} {% if v.found and not v.is_similar %} +

- - +
- Photo -

@@ -79,29 +78,32 @@ {{ v.url_user }}

+ {% if v.ids_data %} +
+
+
+

Details

+ + + {% for k1, v1 in v.ids_data.items() %} + {% if k1 != 'image' %} + + + + + {% endif %} + + {% endfor %} + +
{{ title(k1) }}{% if v1 is iterable and (v1 is not string and v1 is not mapping) %}{{ v1 | join(', ') }}{% else %}{{ detect_link(v1) }}{% endif %}
+
+ {% endif %} + 		+ Photo
- {% if v.ids_data %} -
-
-
-

Details

- - - {% for k1, v1 in v.ids_data.items() %} - {% if k1 != 'image' %} - - - - - {% endif %} - - {% endfor %} - -
{{ title(k1) }}{% if v1 is iterable and (v1 is not string and v1 is not mapping) %}{{ v1 | join(', ') }}{% else %}{{ detect_link(v1) }}{% endif %}
-
- {% endif %}
diff --git a/maigret/sites.py b/maigret/sites.py index f7db004..02a86ff 100644 --- a/maigret/sites.py +++ b/maigret/sites.py @@ -3,9 +3,8 @@ import copy import json import operator -import sys - import requests +import sys from .utils import CaseConverter diff --git a/requirements.txt b/requirements.txt index f60881d..84706fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ bs4==0.0.1 certifi==2020.12.5 chardet==3.0.4 colorama==0.4.4 +python-dateutil==2.8.1 future==0.18.2 future-annotations==1.0.0 html5lib==1.1 @@ -27,7 +28,7 @@ reportlab==3.5.59 requests==2.25.1 requests-futures==1.0.0 six==1.15.0 -socid-extractor==0.0.2 +socid-extractor>0.0.2 soupsieve==2.1 stem==1.8.0 torrequest==0.1.0 diff --git a/tests/conftest.py b/tests/conftest.py index fe4ebca..754f67c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,14 @@ -from _pytest.mark import Mark -from mock import Mock +import glob +import logging import os import pytest +from _pytest.mark import Mark +from mock import Mock from maigret.sites import MaigretDatabase, MaigretSite -JSON_FILE = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../maigret/resources/data.json') +CUR_PATH = os.path.dirname(os.path.realpath(__file__)) +JSON_FILE = os.path.join(CUR_PATH, '../maigret/resources/data.json') empty_mark = Mark('', [], {}) @@ -17,8 +20,25 @@ def pytest_collection_modifyitems(items): items.sort(key=by_slow_marker, reverse=False) +def get_test_reports_filenames(): + return glob.glob(os.path.join('report_*'), recursive=False) + + +def remove_test_reports(): + reports_list = get_test_reports_filenames() + for f in reports_list: os.remove(f) + logging.error(f'Removed test reports {reports_list}') + + @pytest.fixture(scope='session') def default_db(): db = MaigretDatabase().load_from_file(JSON_FILE) return db + + +@pytest.fixture(autouse=True) +def reports_autoclean(): + remove_test_reports() + yield + remove_test_reports() diff --git a/tests/test_activation.py b/tests/test_activation.py index b733cbd..efe30a9 100644 --- a/tests/test_activation.py +++ b/tests/test_activation.py @@ -1,6 +1,6 @@ """Maigret activation test functions""" -from mock import Mock import pytest +from mock import Mock from maigret.activation import ParsingActivator diff --git a/tests/test_maigret.py b/tests/test_maigret.py index 637a54e..1809bc6 100644 --- a/tests/test_maigret.py +++ b/tests/test_maigret.py @@ -1,11 +1,10 @@ """Maigret main module test functions""" import asyncio -from mock import Mock import pytest +from mock import Mock -from maigret.sites import MaigretDatabase, MaigretSite from maigret.maigret import self_check - +from maigret.sites import MaigretDatabase, MaigretSite EXAMPLE_DB = { 'engines': { @@ -54,7 +53,7 @@ def test_self_check_db_positive_disable(): assert db.sites[0].disabled == False loop = asyncio.get_event_loop() - loop.run_until_complete(self_check(db, db.sites_dict, logger)) + loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True)) assert db.sites[0].disabled == True @@ -70,7 +69,7 @@ def test_self_check_db_positive_enable(): assert db.sites[0].disabled == True loop = asyncio.get_event_loop() - loop.run_until_complete(self_check(db, db.sites_dict, logger)) + loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True)) assert db.sites[0].disabled == False @@ -85,7 +84,7 @@ def test_self_check_db_negative_disabled(): assert db.sites[0].disabled == True loop = asyncio.get_event_loop() - loop.run_until_complete(self_check(db, db.sites_dict, logger)) + loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True)) assert db.sites[0].disabled == True @@ -101,6 +100,6 @@ def test_self_check_db_negative_enabled(): assert db.sites[0].disabled == False loop = asyncio.get_event_loop() - loop.run_until_complete(self_check(db, db.sites_dict, logger)) + loop.run_until_complete(self_check(db, db.sites_dict, logger, silent=True)) assert db.sites[0].disabled == False diff --git a/tests/test_report.py b/tests/test_report.py index 1d4f0d1..f7952a9 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,14 +1,15 @@ """Maigret reports test functions""" -from io import StringIO import copy import os +from io import StringIO import xmind +from jinja2 import Template -from maigret.report import save_csv_report_to_file, genxmindfile, save_html_pdf_report +from maigret.report import generate_csv_report, generate_txt_report, save_xmind_report, save_html_report, \ + save_pdf_report, generate_report_template, generate_report_context from maigret.result import QueryResult, QueryStatus - EXAMPLE_RESULTS = { 'GitHub': { 'username': 'test', @@ -56,23 +57,48 @@ SUPPOSED_INTERESTS = "Interests: photo (2), ne SUPPOSED_GEO = "Geo: us (3)" -def test_save_csv_report_to_file(): +def test_generate_report_template(): + report_template, css = generate_report_template(is_pdf=True) + + assert isinstance(report_template, Template) + assert isinstance(css, str) + + report_template, css = generate_report_template(is_pdf=False) + + assert isinstance(report_template, Template) + assert css is None + + +def test_generate_csv_report(): csvfile = StringIO() - save_csv_report_to_file('test', EXAMPLE_RESULTS, csvfile) + generate_csv_report('test', EXAMPLE_RESULTS, csvfile) csvfile.seek(0) data = csvfile.readlines() assert data == [ - 'username,name,url_main,url_user,exists,http_status\r\n', - 'test,GitHub,https://www.github.com/,https://www.github.com/test,Claimed,200\r\n', + 'username,name,url_main,url_user,exists,http_status\r\n', + 'test,GitHub,https://www.github.com/,https://www.github.com/test,Claimed,200\r\n', + ] + + +def test_generate_txt_report(): + txtfile = StringIO() + generate_txt_report('test', EXAMPLE_RESULTS, txtfile) + + txtfile.seek(0) + data = txtfile.readlines() + + assert data == [ + 'https://www.github.com/test\n', + 'Total Websites Username Detected On : 1', ] def test_save_xmind_report(): - filename = 'test_report.xmind' - genxmindfile(filename, 'test', EXAMPLE_RESULTS) - + filename = 'report_test.xmind' + save_xmind_report(filename, 'test', EXAMPLE_RESULTS) + workbook = xmind.load(filename) sheet = workbook.getPrimarySheet() data = sheet.getData() @@ -87,14 +113,9 @@ def test_save_xmind_report(): def test_html_report(): - report_name = 'report_alexaimephotographycars.html' - try: - os.remove(report_name) - except: - pass - - save_html_pdf_report(TEST,filename=report_name,filenamepdf=None) - assert os.path.exists(report_name) + report_name = 'report_test.html' + context = generate_report_context(TEST) + save_html_report(report_name, context) report_text = open(report_name).read() @@ -102,12 +123,10 @@ def test_html_report(): assert SUPPOSED_GEO in report_text assert SUPPOSED_INTERESTS in report_text -def test_pdf_report(): - report_name_pdf = 'report_alexaimephotographycars.pdf' - try: - os.remove(report_name_pdf) - except: - pass - save_html_pdf_report(TEST,filename=None,filenamepdf=report_name_pdf) - assert os.path.exists(report_name_pdf) +def test_pdf_report(): + report_name = 'report_test.pdf' + context = generate_report_context(TEST) + save_pdf_report(report_name, context) + + assert os.path.exists(report_name)