Fixed DB loading from URL

maigret/maigret.py

@@ -195,7 +195,7 @@ def setup_arguments_parser():
         metavar="DB_FILE",
         dest="db_file",
         default=None,
-        help="Load Maigret database from a JSON file or an online, valid, JSON file.",
+        help="Load Maigret database from a JSON file or HTTP web resource.",
     )
     parser.add_argument(
         "--cookies-jar-file",
@@ -519,7 +519,7 @@ async def main():
     )
 
     # Create object with all information about sites we are aware of.
-    db = MaigretDatabase().load_from_file(args.db_file)
+    db = MaigretDatabase().load_from_path(args.db_file)
     get_top_sites_for_id = lambda x: db.ranked_sites_dict(
         top=args.top_sites,
         tags=args.tags,

maigret/report.py

@@ -139,7 +139,9 @@ def save_graph_report(filename: str, username_results: list, db: MaigretDatabase
             if dictionary["status"].status != QueryStatus.CLAIMED:
                 continue
 
-            site_fallback_name = dictionary.get('url_user', f'{website_name}: {username.lower()}')
+            site_fallback_name = dictionary.get(
+                'url_user', f'{website_name}: {username.lower()}'
+            )
             # site_node_name = dictionary.get('url_user', f'{website_name}: {username.lower()}')
             site_node_name = graph.add_node('site', site_fallback_name)
             graph.link(username_node_name, site_node_name)
@@ -165,11 +167,13 @@ def save_graph_report(filename: str, username_results: list, db: MaigretDatabase
                             data_node_name = graph.add_node(vv, site_fallback_name)
                             graph.link(list_node_name, data_node_name)
 
-                            add_ids = {a: b for b, a in db.extract_ids_from_url(vv).items()}
+                            add_ids = {
+                                a: b for b, a in db.extract_ids_from_url(vv).items()
+                            }
                             if add_ids:
                                 process_ids(data_node_name, add_ids)
                     else:
-                    # value is just a string
+                        # value is just a string
                         # ids_data_name = f'{k}: {v}'
                         # if ids_data_name == parent_node:
                         #     continue

maigret/sites.py

@@ -292,6 +292,9 @@ class MaigretDatabase:
         return self
 
     def save_to_file(self, filename: str) -> "MaigretDatabase":
+        if '://' in filename:
+            return self
+
         db_data = {
             "sites": {site.name: site.strip_engine_data().json for site in self._sites},
             "engines": {engine.name: engine.json for engine in self._engines},
@@ -344,7 +347,13 @@ class MaigretDatabase:
 
         return self.load_from_json(data)
 
-    def load_from_url(self, url: str) -> "MaigretDatabase":
+    def load_from_path(self, path: str) -> "MaigretDatabase":
+        if '://' in path:
+            return self.load_from_http(path)
+        else:
+            return self.load_from_file(path)
+
+    def load_from_http(self, url: str) -> "MaigretDatabase":
         is_url_valid = url.startswith("http://") or url.startswith("https://")
 
         if not is_url_valid:
@@ -400,7 +409,6 @@ class MaigretDatabase:
 
         return found_flags
 
-
     def extract_ids_from_url(self, url: str) -> dict:
         results = {}
         for s in self._sites:
@@ -411,7 +419,6 @@ class MaigretDatabase:
             results[_id] = _type
         return results
 
-
     def get_db_stats(self, sites_dict):
         if not sites_dict:
             sites_dict = self.sites_dict()