mirror of
https://github.com/soxoj/maigret.git
synced 2026-05-07 14:34:33 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Soxoj
|
23c7757dcf | ||
|
Soxoj
|
a163278f89 | ||
|
Soxoj
|
5381a134f0 |
@@ -1,61 +0,0 @@
|
||||
name: Close Invalid Telegram PRs
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Run daily at 2 AM UTC
|
||||
- cron: '0 2 * * *'
|
||||
workflow_dispatch:
|
||||
# Allow manual triggering
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Run in dry-run mode (show what would be closed without closing)'
|
||||
required: false
|
||||
default: 'false'
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
close-invalid-prs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
# Need write permissions for pull requests and issues
|
||||
pull-requests: write
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install requests
|
||||
|
||||
- name: Make script executable
|
||||
run: chmod +x utils/close_invalid_telegram_prs.py
|
||||
|
||||
- name: Run PR closer script (dry-run for manual trigger)
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'true'
|
||||
run: |
|
||||
python utils/close_invalid_telegram_prs.py --dry-run
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Run PR closer script (live for manual trigger)
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run == 'false'
|
||||
run: |
|
||||
python utils/close_invalid_telegram_prs.py
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Run PR closer script (automated daily)
|
||||
if: github.event_name == 'schedule'
|
||||
run: |
|
||||
python utils/close_invalid_telegram_prs.py --dry-run
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
||||
python-version: ["3.10", "3.11", "3.12"]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
poetry run coverage report --fail-under=60
|
||||
poetry run coverage html
|
||||
- name: Upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: htmlcov-${{ strategy.job-index }}
|
||||
name: htmlcov
|
||||
path: htmlcov
|
||||
@@ -1,21 +1,30 @@
|
||||
name: Upload Python Package to PyPI when a Release is Created
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [created]
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-and-publish:
|
||||
pypi-publish:
|
||||
name: Publish release to PyPI
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/p/maigret
|
||||
permissions:
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v3
|
||||
- run: uv build
|
||||
- name: Publish to PyPI (Trusted Publishing)
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
packages-dir: dist
|
||||
python-version: "3.x"
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install setuptools wheel
|
||||
- name: Build package
|
||||
run: |
|
||||
python setup.py sdist bdist_wheel # Could also be python -m build
|
||||
- name: Publish package distributions to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
+1
-249
@@ -1,254 +1,6 @@
|
||||
# Changelog
|
||||
|
||||
## [0.5.0] - 2025-08-10
|
||||
* Site Supression by @C3n7ral051nt4g3ncy in https://github.com/soxoj/maigret/pull/627
|
||||
* Bump yarl from 1.7.2 to 1.8.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/626
|
||||
* Streaming sites by @soxoj in https://github.com/soxoj/maigret/pull/628
|
||||
* Mirrors by @fen0s in https://github.com/soxoj/maigret/pull/630
|
||||
* Added Instagram scrapers by @soxoj in https://github.com/soxoj/maigret/pull/633
|
||||
* Bump psutil from 5.9.1 to 5.9.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/624
|
||||
* Bump pypdf2 from 2.10.4 to 2.10.5 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/625
|
||||
* Invalid results fixes by @soxoj in https://github.com/soxoj/maigret/pull/634
|
||||
* Bump pytest-httpserver from 1.0.5 to 1.0.6 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/638
|
||||
* Bump pypdf2 from 2.10.5 to 2.10.8 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/641
|
||||
* Bump certifi from 2022.6.15 to 2022.9.14 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/644
|
||||
* Bump idna from 3.3 to 3.4 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/640
|
||||
* fix false positives from bot by @fen0s in https://github.com/soxoj/maigret/pull/663
|
||||
* Add pre commit hook by @fen0s in https://github.com/soxoj/maigret/pull/664
|
||||
* site deletion by @C3n7ral051nt4g3ncy in https://github.com/soxoj/maigret/pull/648
|
||||
* Changed docker run to interactive and remove on exit by @dr-BEat in https://github.com/soxoj/maigret/pull/675
|
||||
* Corrected grammar in README.md by @Trkzi-Omar in https://github.com/soxoj/maigret/pull/674
|
||||
* fix sites from issues by @fen0s in https://github.com/soxoj/maigret/pull/680
|
||||
* correct username in usage examples by @LeonGr in https://github.com/soxoj/maigret/pull/673
|
||||
* Update README.md by @johanburati in https://github.com/soxoj/maigret/pull/669
|
||||
* Fix typos by @LorenzoSapora in https://github.com/soxoj/maigret/pull/681
|
||||
* Build docker images for arm64 and amd64 by @krydos in https://github.com/soxoj/maigret/pull/687
|
||||
* Bump certifi from 2022.9.14 to 2022.9.24 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/652
|
||||
* Bump aiohttp from 3.8.1 to 3.8.3 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/651
|
||||
* Bump arabic-reshaper from 2.1.3 to 2.1.4 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/650
|
||||
* Update README.md, Repl.it -> Replit with new badge by @PeterDaveHello in https://github.com/soxoj/maigret/pull/692
|
||||
* Refactor Dockerfile with best practices by @PeterDaveHello in https://github.com/soxoj/maigret/pull/691
|
||||
* Improve README.md Installation section by @PeterDaveHello in https://github.com/soxoj/maigret/pull/690
|
||||
* Bump pytest-cov from 3.0.0 to 4.0.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/688
|
||||
* Bump stem from 1.8.0 to 1.8.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/689
|
||||
* Bump typing-extensions from 4.3.0 to 4.4.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/698
|
||||
* Typo fixes in error.py by @Ben-Chapman in https://github.com/soxoj/maigret/pull/711
|
||||
* Fixed docs about tags by @soxoj in https://github.com/soxoj/maigret/pull/715
|
||||
* Fixed lightstalking.com by @soxoj in https://github.com/soxoj/maigret/pull/716
|
||||
* Fixed YouTube by @soxoj in https://github.com/soxoj/maigret/pull/717
|
||||
* Bump pytest-asyncio from 0.19.0 to 0.20.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/732
|
||||
* Updated snapcraft yaml by @kz6fittycent in https://github.com/soxoj/maigret/pull/720
|
||||
* Bump colorama from 0.4.5 to 0.4.6 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/733
|
||||
* Bump pytest from 7.1.3 to 7.2.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/734
|
||||
* disable not working sites by @fen0s in https://github.com/soxoj/maigret/pull/739
|
||||
* disable broken sites by @fen0s in https://github.com/soxoj/maigret/pull/756
|
||||
* Bump cloudscraper from 1.2.64 to 1.2.66 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/769
|
||||
* fix opensea and shutterstock, disable a few dead sites by @fen0s in https://github.com/soxoj/maigret/pull/798
|
||||
* Fixed documentation URL by @soxoj in https://github.com/soxoj/maigret/pull/799
|
||||
* Small readme fix by @soxoj in https://github.com/soxoj/maigret/pull/857
|
||||
* docs spelling error by @Nadeem-05 in https://github.com/soxoj/maigret/pull/866
|
||||
* Fix Pinterest false positive by @therealchiendat in https://github.com/soxoj/maigret/pull/862
|
||||
* Added new Websites by @codyMar30 in https://github.com/soxoj/maigret/pull/838
|
||||
* Update "future" package to v0.18.3 by @PeterDaveHello in https://github.com/soxoj/maigret/pull/834
|
||||
* Bump certifi from 2022.9.24 to 2022.12.7 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/793
|
||||
* Update dependency - networkx from v2.5.1 to v2.6 by @PeterDaveHello in https://github.com/soxoj/maigret/pull/738
|
||||
* Bump reportlab from 3.6.11 to 3.6.12 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/735
|
||||
* Bump typing-extensions from 4.4.0 to 4.5.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/888
|
||||
* Bump psutil from 5.9.2 to 5.9.4 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/741
|
||||
* Bump attrs from 22.1.0 to 22.2.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/892
|
||||
* Bump multidict from 6.0.2 to 6.0.4 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/891
|
||||
* Fixed false positives, updated networkx dep, some lint fixes by @soxoj in https://github.com/soxoj/maigret/pull/894
|
||||
* Bump lxml from 4.9.1 to 4.9.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/900
|
||||
* Bump yarl from 1.8.1 to 1.8.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/899
|
||||
* Fixed false positives on Mastodon sites by @soxoj in https://github.com/soxoj/maigret/pull/901
|
||||
* Added valid regex for Mastodon instances (#848) by @soxoj in https://github.com/soxoj/maigret/pull/906
|
||||
* Fix missing Mastodon Regex on #906 by @therealchiendat in https://github.com/soxoj/maigret/pull/908
|
||||
* Bump tqdm from 4.64.1 to 4.65.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/905
|
||||
* Bump requests from 2.28.1 to 2.28.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/904
|
||||
* Bump psutil from 5.9.4 to 5.9.5 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/910
|
||||
* fix deployment of tests by @noraj in https://github.com/soxoj/maigret/pull/933
|
||||
* Added 26 ENS and similar domains with tag `crypto` by @soxoj in https://github.com/soxoj/maigret/pull/942
|
||||
* Bump requests from 2.28.2 to 2.31.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/957
|
||||
* Update wizard.py by @engNoori in https://github.com/soxoj/maigret/pull/1016
|
||||
* Improved search through UnstoppableDomains by @soxoj in https://github.com/soxoj/maigret/pull/1040
|
||||
* Added memory.lol (Twitter usernames archive) by @soxoj in https://github.com/soxoj/maigret/pull/1067
|
||||
* Disabled and fixed several sites by @soxoj in https://github.com/soxoj/maigret/pull/1132
|
||||
* Fixed some sites (again) by @soxoj in https://github.com/soxoj/maigret/pull/1133
|
||||
* fix(sec): upgrade reportlab to 3.6.13 by @realize096 in https://github.com/soxoj/maigret/pull/1051
|
||||
* Add compatibility with pytest >= 7.3.0 by @tjni in https://github.com/soxoj/maigret/pull/1117
|
||||
* Additionally fixed sites, win32 build fix by @soxoj in https://github.com/soxoj/maigret/pull/1148
|
||||
* Sites fixes 250823 by @soxoj in https://github.com/soxoj/maigret/pull/1149
|
||||
* Bump reportlab from 3.6.12 to 4.0.4 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1160
|
||||
* Bump certifi from 2022.12.7 to 2023.7.22 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1070
|
||||
* fix(sec): upgrade certifi to 2022.12.07 by @realize096 in https://github.com/soxoj/maigret/pull/1173
|
||||
* Bump cloudscraper from 1.2.66 to 1.2.71 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/914
|
||||
* Some sites fixed & cloudflare detection by @soxoj in https://github.com/soxoj/maigret/pull/1178
|
||||
* EasyInstaller because everyone likes saving time :) by @CatchySmile in https://github.com/soxoj/maigret/pull/1212
|
||||
* Tests fixes + last updates by @soxoj in https://github.com/soxoj/maigret/pull/1228
|
||||
* Bump pypdf2 from 2.10.8 to 3.0.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/815
|
||||
* Bump pyvis from 0.2.1 to 0.3.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/861
|
||||
* Bump xhtml2pdf from 0.2.8 to 0.2.11 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/935
|
||||
* Bump flake8 from 5.0.4 to 6.1.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1091
|
||||
* Bump aiohttp from 3.8.3 to 3.8.6 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1222
|
||||
* Specified pyinstaller version by @soxoj in https://github.com/soxoj/maigret/pull/1230
|
||||
* Pyinstaller fix by @soxoj in https://github.com/soxoj/maigret/pull/1231
|
||||
* Test pyinstaller on dev branch by @soxoj in https://github.com/soxoj/maigret/pull/1233
|
||||
* Update main from dev again by @soxoj in https://github.com/soxoj/maigret/pull/1234
|
||||
* Bump typing-extensions from 4.5.0 to 4.8.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1239
|
||||
* Bump pytest-rerunfailures from 10.2 to 12.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1237
|
||||
* Bump async-timeout from 4.0.2 to 4.0.3 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1238
|
||||
* Changed pyinstaller dir by @soxoj in https://github.com/soxoj/maigret/pull/1245
|
||||
* Bump tqdm from 4.65.0 to 4.66.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1235
|
||||
* Updating site checkers, disabling suspended sites by @MeowyPouncer in https://github.com/soxoj/maigret/pull/1266
|
||||
* Updated site statistics by @soxoj in https://github.com/soxoj/maigret/pull/1273
|
||||
* Compat RegataOS (Opensuse) by @Jeiel0rbit in https://github.com/soxoj/maigret/pull/1308
|
||||
* fix reddit by @hhhtylerw in https://github.com/soxoj/maigret/pull/1296
|
||||
* Added Telegram bot link by @soxoj in https://github.com/soxoj/maigret/pull/1321
|
||||
* Added SOWEL classification by @soxoj in https://github.com/soxoj/maigret/pull/1453
|
||||
* Bump jinja2 from 3.1.2 to 3.1.3 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1358
|
||||
* Fixed/Disabled sites. Update requirements.txt by @rly0nheart in https://github.com/soxoj/maigret/pull/1517
|
||||
* Fixed 4 sites, added 6 sites, disabled 27 sites by @rly0nheart in https://github.com/soxoj/maigret/pull/1536
|
||||
* Fixed 3 sites, disabed 3, added by @rly0nheart in https://github.com/soxoj/maigret/pull/1539
|
||||
* Bump socid-extractor from 0.0.24 to 0.0.26 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1546
|
||||
* Added code conventions to CONTRIBUTING.md by @Lord-Topa in https://github.com/soxoj/maigret/pull/1589
|
||||
* Readme by @Lord-Topa in https://github.com/soxoj/maigret/pull/1588
|
||||
* Update data.json by @ranlo in https://github.com/soxoj/maigret/pull/1559
|
||||
* Adding permutator feature for usernames by @balestek in https://github.com/soxoj/maigret/pull/1575
|
||||
* Alik.cz indirectly requests removal by @ppfeister in https://github.com/soxoj/maigret/pull/1671
|
||||
* Fixed 1 site, PyInstaller workflow, Google Colab example by @Ixve in https://github.com/soxoj/maigret/pull/1558
|
||||
* Bump soupsieve from 2.5 to 2.6 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1708
|
||||
* Added dev documentation, fixed some sites, removed GitHub issue links… by @soxoj in https://github.com/soxoj/maigret/pull/1869
|
||||
* Bump cryptography from 42.0.7 to 43.0.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1870
|
||||
* Bump requests-futures from 1.0.1 to 1.0.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1868
|
||||
* Bump werkzeug from 3.0.3 to 3.0.6 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1846
|
||||
* Added .readthedocs.yaml, fixed Pyinstaller and Docker workflows by @soxoj in https://github.com/soxoj/maigret/pull/1874
|
||||
* Added GitHub and BuyMeACoffee sponsorships by @soxoj in https://github.com/soxoj/maigret/pull/1875
|
||||
* Bump psutil from 5.9.5 to 6.1.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1839
|
||||
* Bump flake8 from 6.1.0 to 7.1.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1692
|
||||
* Bump future from 0.18.3 to 1.0.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1545
|
||||
* Bump urllib3 from 2.2.1 to 2.2.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1600
|
||||
* Bump certifi from 2023.11.17 to 2024.8.30 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1840
|
||||
* Fixed test for aiohttp 3.10 by @soxoj in https://github.com/soxoj/maigret/pull/1876
|
||||
* Bump aiohttp from 3.9.5 to 3.10.5 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1721
|
||||
* Added new badges to README by @soxoj in https://github.com/soxoj/maigret/pull/1877
|
||||
* Show detailed error statistics for `-v` by @soxoj in https://github.com/soxoj/maigret/pull/1879
|
||||
* Disabled unavailable sites by @soxoj in https://github.com/soxoj/maigret/pull/1880
|
||||
* Added 7 sites, implemented integration with Marple, docs update by @soxoj in https://github.com/soxoj/maigret/pull/1881
|
||||
* Bump pefile from 2022.5.30 to 2024.8.26 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1883
|
||||
* Bump lxml from 4.9.4 to 5.3.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1884
|
||||
* New sites added by @soxoj in https://github.com/soxoj/maigret/pull/1888
|
||||
* Improved self-check mode, added 15 sites by @soxoj in https://github.com/soxoj/maigret/pull/1887
|
||||
* Bump pyinstaller from 6.1 to 6.11.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1882
|
||||
* Bump pytest-asyncio from 0.23.7 to 0.23.8 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1885
|
||||
* Pyinstaller bump & pefile fix by @soxoj in https://github.com/soxoj/maigret/pull/1890
|
||||
* Bump python-bidi from 0.4.2 to 0.6.3 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1886
|
||||
* Sites checks fixes by @soxoj in https://github.com/soxoj/maigret/pull/1896
|
||||
* Parallel execution optimization by @soxoj in https://github.com/soxoj/maigret/pull/1897
|
||||
* Maigret bot support (custom progress function fixed) by @soxoj in https://github.com/soxoj/maigret/pull/1898
|
||||
* Bump markupsafe from 2.1.5 to 3.0.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1895
|
||||
* Retries set to 0 by default, refactored code of executor with progress by @soxoj in https://github.com/soxoj/maigret/pull/1899
|
||||
* Bump aiohttp-socks from 0.7.1 to 0.9.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1900
|
||||
* Bump pycountry from 23.12.11 to 24.6.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1903
|
||||
* Bump pytest-cov from 4.1.0 to 6.0.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1902
|
||||
* Bump pyvis from 0.2.1 to 0.3.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1893
|
||||
* Close http connections (#1595) by @soxoj in https://github.com/soxoj/maigret/pull/1905
|
||||
* New logo by @soxoj in https://github.com/soxoj/maigret/pull/1906
|
||||
* Fixed dateutil parsing error for CDT timezone by @soxoj in https://github.com/soxoj/maigret/pull/1907
|
||||
* Bump alive-progress from 2.4.1 to 3.2.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1910
|
||||
* Permutator output and documentation updates by @soxoj in https://github.com/soxoj/maigret/pull/1914
|
||||
* Bump aiohttp from 3.11.7 to 3.11.8 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1912
|
||||
* Bump async-timeout from 4.0.3 to 5.0.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1909
|
||||
* An recursive search animation in README has been updated by @soxoj in https://github.com/soxoj/maigret/pull/1915
|
||||
* Bump pytest-rerunfailures from 12.0 to 15.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1911
|
||||
* Bump attrs from 22.2.0 to 24.2.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1913
|
||||
* Sites fixes by @soxoj in https://github.com/soxoj/maigret/pull/1917
|
||||
* Update README.md by @soxoj in https://github.com/soxoj/maigret/pull/1919
|
||||
* Refactored sites module, updated documentation by @soxoj in https://github.com/soxoj/maigret/pull/1918
|
||||
* Bump aiohttp from 3.11.8 to 3.11.9 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1920
|
||||
* Bump pytest from 7.4.4 to 8.3.4 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1923
|
||||
* Bump yarl from 1.18.0 to 1.18.3 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1922
|
||||
* Bump pytest-asyncio from 0.23.8 to 0.24.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1925
|
||||
* Documentation update by @soxoj in https://github.com/soxoj/maigret/pull/1926
|
||||
* Bump mock from 4.0.3 to 5.1.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1921
|
||||
* Bump pywin32-ctypes from 0.2.1 to 0.2.3 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1924
|
||||
* Installation docs update by @soxoj in https://github.com/soxoj/maigret/pull/1927
|
||||
* Disabled Figma check by @soxoj in https://github.com/soxoj/maigret/pull/1928
|
||||
* Put Windows executable in Releases for each dev and main commit by @soxoj in https://github.com/soxoj/maigret/pull/1929
|
||||
* Updated PyInstaller workflow by @soxoj in https://github.com/soxoj/maigret/pull/1930
|
||||
* Documentation update by @soxoj in https://github.com/soxoj/maigret/pull/1931
|
||||
* Fixed Figma check and some bugs by @soxoj in https://github.com/soxoj/maigret/pull/1932
|
||||
* Bump six from 1.16.0 to 1.17.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1933
|
||||
* Activation mechanism documentation added by @soxoj in https://github.com/soxoj/maigret/pull/1935
|
||||
* Readme/docs update based on GH discussions by @soxoj in https://github.com/soxoj/maigret/pull/1936
|
||||
* Bump aiohttp from 3.11.9 to 3.11.10 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1937
|
||||
* Weibo site check fix, activation mechanism added by @soxoj in https://github.com/soxoj/maigret/pull/1938
|
||||
* Fixed Ebay and BongaCams checks by @soxoj in https://github.com/soxoj/maigret/pull/1939
|
||||
* Sites fixes by @soxoj in https://github.com/soxoj/maigret/pull/1940
|
||||
* Fixed Linktr and discourse.mozilla.org by @soxoj in https://github.com/soxoj/maigret/pull/1941
|
||||
* Refactored self-check method, code formatting, small lint fixes by @soxoj in https://github.com/soxoj/maigret/pull/1942
|
||||
* Refactoring, test coverage increased to 60% by @soxoj in https://github.com/soxoj/maigret/pull/1943
|
||||
* Added a test for submitter by @soxoj in https://github.com/soxoj/maigret/pull/1944
|
||||
* Update README.md by @soxoj in https://github.com/soxoj/maigret/pull/1949
|
||||
* Updated OP.GG checks by @soxoj in https://github.com/soxoj/maigret/pull/1950
|
||||
* Fixed ProductHunt check by @soxoj in https://github.com/soxoj/maigret/pull/1951
|
||||
* Improved check feature extraction function, added tests by @soxoj in https://github.com/soxoj/maigret/pull/1952
|
||||
* Submit improvements and site check fixes by @soxoj in https://github.com/soxoj/maigret/pull/1956
|
||||
* chore: update submit.py by @eltociear in https://github.com/soxoj/maigret/pull/1957
|
||||
* Fixed Gravatar parsing (socid_extractor) by @soxoj in https://github.com/soxoj/maigret/pull/1958
|
||||
* Site check fixes by @soxoj in https://github.com/soxoj/maigret/pull/1962
|
||||
* fix bad linux filename generation by @overcuriousity in https://github.com/soxoj/maigret/pull/1961
|
||||
* Bump pytest-asyncio from 0.24.0 to 0.25.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1963
|
||||
* Fixed flaky tests to check cookies by @soxoj in https://github.com/soxoj/maigret/pull/1965
|
||||
* Preparation of 0.5.0 alpha version by @soxoj in https://github.com/soxoj/maigret/pull/1966
|
||||
* Created web frontend launched via --web flag by @overcuriousity in https://github.com/soxoj/maigret/pull/1967
|
||||
* Bump certifi from 2024.8.30 to 2024.12.14 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1969
|
||||
* Bump attrs from 24.2.0 to 24.3.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1970
|
||||
* Added web interface docs by @soxoj in https://github.com/soxoj/maigret/pull/1972
|
||||
* Small docs and parameters fixes for web interface mode by @soxoj in https://github.com/soxoj/maigret/pull/1973
|
||||
* [ImgBot] Optimize images by @imgbot[bot] in https://github.com/soxoj/maigret/pull/1974
|
||||
* Improving the web interface by @overcuriousity in https://github.com/soxoj/maigret/pull/1975
|
||||
* make graph more meaningful by @overcuriousity in https://github.com/soxoj/maigret/pull/1977
|
||||
* Async generator-executor for site checks by @soxoj in https://github.com/soxoj/maigret/pull/1978
|
||||
* Bump aiohttp from 3.11.10 to 3.11.11 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1979
|
||||
* Bump psutil from 6.1.0 to 6.1.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1980
|
||||
* Bump aiohttp-socks from 0.9.1 to 0.10.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1985
|
||||
* Bump mypy from 1.13.0 to 1.14.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1983
|
||||
* Bump aiohttp-socks from 0.10.0 to 0.10.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1987
|
||||
* Bump jinja2 from 3.1.4 to 3.1.5 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1982
|
||||
* Bump coverage from 7.6.9 to 7.6.10 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1986
|
||||
* Bump pytest-asyncio from 0.25.0 to 0.25.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1989
|
||||
* Bump mypy from 1.14.0 to 1.14.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1988
|
||||
* Bump pytest-asyncio from 0.25.1 to 0.25.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/1990
|
||||
* docs: update usage-examples.rst by @eltociear in https://github.com/soxoj/maigret/pull/1996
|
||||
* upload-artifact action in python test workflow updated to v4 by @soxoj in https://github.com/soxoj/maigret/pull/2024
|
||||
* Pass db_file configuration to web interface by @pykereaper in https://github.com/soxoj/maigret/pull/2019
|
||||
* Fix usage of data.json files from web by @pykereaper in https://github.com/soxoj/maigret/pull/2020
|
||||
* Bump black from 24.10.0 to 25.1.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2001
|
||||
* Important Update Installer.bat by @CatchySmile in https://github.com/soxoj/maigret/pull/1994
|
||||
* Bump cryptography from 44.0.0 to 44.0.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2005
|
||||
* Bump jinja2 from 3.1.5 to 3.1.6 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2011
|
||||
* [#2010] Add 6 more websites to manage by @pylapp in https://github.com/soxoj/maigret/pull/2009
|
||||
* Bump flask from 3.1.0 to 3.1.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2028
|
||||
* Bump requests from 2.32.3 to 2.32.4 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2026
|
||||
* Bump pycares from 4.5.0 to 4.9.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2025
|
||||
* Bump pytest-asyncio from 0.25.2 to 0.26.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2016
|
||||
* Bump urllib3 from 2.2.3 to 2.5.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2027
|
||||
* Disable ICQ site by @Echo-Darlyson in https://github.com/soxoj/maigret/pull/1993
|
||||
* Bump attrs from 24.3.0 to 25.3.0 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2014
|
||||
* Bump certifi from 2024.12.14 to 2025.1.31 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2004
|
||||
* Bump typing-extensions from 4.12.2 to 4.14.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2038
|
||||
* Disable AskFM by @MR-VL in https://github.com/soxoj/maigret/pull/2037
|
||||
* Bump platformdirs from 4.3.6 to 4.3.8 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2033
|
||||
* Bump coverage from 7.6.10 to 7.9.2 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2039
|
||||
* Bump aiohttp from 3.11.11 to 3.12.14 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2041
|
||||
* Bump yarl from 1.18.3 to 1.20.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2032
|
||||
* Fixed test dialog_adds_site_negative by @soxoj in https://github.com/soxoj/maigret/pull/2107
|
||||
* Bump reportlab from 4.2.5 to 4.4.3 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2063
|
||||
* Bump asgiref from 3.8.1 to 3.9.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2040
|
||||
* Bump multidict from 6.1.0 to 6.6.3 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2034
|
||||
* Bump pytest-rerunfailures from 15.0 to 15.1 by @dependabot[bot] in https://github.com/soxoj/maigret/pull/2030
|
||||
|
||||
**Full Changelog**: https://github.com/soxoj/maigret/compare/v0.4.4...v0.5.0
|
||||
## [Unreleased]
|
||||
|
||||
## [0.4.4] - 2022-09-03
|
||||
* Fixed some false positives by @soxoj in https://github.com/soxoj/maigret/pull/433
|
||||
|
||||
+78
-68
@@ -1,61 +1,85 @@
|
||||
@echo off
|
||||
|
||||
REM check if running as admin
|
||||
|
||||
goto check_Permissions
|
||||
|
||||
:check_Permissions
|
||||
echo Administrative permissions required. Detecting permissions...
|
||||
|
||||
net session >nul 2>&1
|
||||
if %errorLevel% == 0 (
|
||||
echo Success: Elevated permissions granted.
|
||||
goto 1
|
||||
) else (
|
||||
echo Failure: Requires elevated permissions.
|
||||
pause >nul
|
||||
cls
|
||||
echo Failure: You MUST run this as administator, otherwise commands will fail.
|
||||
)
|
||||
|
||||
cls
|
||||
echo --------------------------------------------------------
|
||||
echo Python 3.8 or higher and pip3 required.
|
||||
echo --------------------------------------------------------
|
||||
echo Press [I] to begin installation.
|
||||
echo Press [R] If already installed.
|
||||
echo --------------------------------------------------------
|
||||
choice /c IR
|
||||
if %errorlevel%==1 goto check_python
|
||||
if %errorlevel%==2 goto after
|
||||
pause >nul
|
||||
|
||||
:check_python
|
||||
cls
|
||||
for /f "tokens=2 delims= " %%i in ('python --version 2^>nul') do (
|
||||
for /f "tokens=1,2 delims=." %%j in ("%%i") do (
|
||||
if %%j GEQ 3 (
|
||||
if %%k GEQ 8 (
|
||||
goto check_pip
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
echo Python 3.8 or higher is required. Please install it first.
|
||||
pause
|
||||
exit /b
|
||||
|
||||
:check_pip
|
||||
pip --version 2>nul | findstr /r /c:"pip" >nul
|
||||
|
||||
REM Step 2: Check if Python and pip3 are installed
|
||||
python --version >nul 2>&1
|
||||
if %errorlevel% neq 0 (
|
||||
echo pip is required. Please install it first.
|
||||
echo Python is not installed. Please install Python 3.8 or higher.
|
||||
pause
|
||||
exit /b
|
||||
)
|
||||
goto install1
|
||||
|
||||
pip3 --version >nul 2>&1
|
||||
if %errorlevel% neq 0 (
|
||||
echo pip3 is not installed. Please install pip3.
|
||||
pause
|
||||
exit /b
|
||||
)
|
||||
|
||||
REM Step 3: Check Python version
|
||||
python -c "import sys; exit(0) if sys.version_info >= (3,8) else exit(1)"
|
||||
if %errorlevel% neq 0 (
|
||||
echo Python version 3.8 or higher is required.
|
||||
pause
|
||||
exit /b
|
||||
)
|
||||
|
||||
|
||||
:1
|
||||
cls
|
||||
:::===============================================================
|
||||
::: ______ __ __ _ _
|
||||
::: | ____| | \/ | (_) | |
|
||||
::: | |__ __ _ ___ _ _ | \ / | __ _ _ __ _ _ __ ___| |_
|
||||
::: | __| / _` / __| | | | | |\/| |/ _` | |/ _` | '__/ _ \ __|
|
||||
::: | |___| (_| \__ \ |_| | | | | | (_| | | (_| | | | __/ |_
|
||||
::: |______\__,_|___/\__, | |_| |_|\__,_|_|\__, |_| \___|\__|
|
||||
::: __/ | __/ |
|
||||
::: |___/ |___/
|
||||
:::
|
||||
:::===============================================================
|
||||
echo.
|
||||
for /f "delims=: tokens=*" %%A in ('findstr /b ::: "%~f0"') do @echo(%%A
|
||||
echo.
|
||||
echo ----------------------------------------------------------------
|
||||
echo Python 3.8 or higher and pip3 required.
|
||||
echo ----------------------------------------------------------------
|
||||
echo Press [I] to begin installation.
|
||||
echo Press [R] If already installed.
|
||||
echo ----------------------------------------------------------------
|
||||
choice /c IR
|
||||
if %errorlevel%==1 goto install1
|
||||
if %errorlevel%==2 goto after
|
||||
|
||||
:install1
|
||||
cls
|
||||
echo ========================================================
|
||||
echo Maigret Installation
|
||||
echo Maigret Installation Script
|
||||
echo ========================================================
|
||||
echo.
|
||||
echo --------------------------------------------------------
|
||||
echo If your pip installation is outdated, it could cause
|
||||
echo cryptography to fail on installation.
|
||||
echo --------------------------------------------------------
|
||||
echo Check for and install pip 23.3.2 now?
|
||||
echo check for and install pip updates now?
|
||||
echo --------------------------------------------------------
|
||||
choice /c YN
|
||||
if %errorlevel%==1 goto install2
|
||||
@@ -63,56 +87,42 @@ if %errorlevel%==2 goto install3
|
||||
|
||||
:install2
|
||||
cls
|
||||
python -m pip install --upgrade pip==23.3.2
|
||||
if %errorlevel% neq 0 (
|
||||
echo Failed to update pip to version 23.3.2. Please check your installation.
|
||||
pause
|
||||
exit /b
|
||||
)
|
||||
goto install3
|
||||
python -m pip install --upgrade pip
|
||||
goto:install3
|
||||
|
||||
:install3
|
||||
cls
|
||||
echo ========================================================
|
||||
echo Maigret Installation
|
||||
echo Maigret Installation Script
|
||||
echo ========================================================
|
||||
echo.
|
||||
echo --------------------------------------------------------
|
||||
echo Installing Maigret...
|
||||
python -m pip install maigret
|
||||
if %errorlevel% neq 0 (
|
||||
echo Failed to install Maigret. Please check your installation.
|
||||
pause
|
||||
exit /b
|
||||
)
|
||||
echo.
|
||||
echo +------------------------------------------------------+
|
||||
echo Maigret installed successfully.
|
||||
echo +------------------------------------------------------+
|
||||
pause
|
||||
goto after
|
||||
echo Install requirements and maigret?
|
||||
echo --------------------------------------------------------
|
||||
choice /c YN
|
||||
if %errorlevel%==1 goto install4
|
||||
if %errorlevel%==2 goto 1
|
||||
|
||||
:install4
|
||||
cls
|
||||
pip install .
|
||||
pip install maigret
|
||||
goto:after
|
||||
|
||||
:after
|
||||
cls
|
||||
echo ========================================================
|
||||
echo Maigret Usage
|
||||
echo Maigret Background Search
|
||||
echo ========================================================
|
||||
echo.
|
||||
echo +--------------------------------------------------------+
|
||||
echo To use Maigret, you can run the following command:
|
||||
echo --------------------------------------------------------
|
||||
echo Please Enter Username / Email
|
||||
echo --------------------------------------------------------
|
||||
set /p input=
|
||||
maigret %input%
|
||||
echo.
|
||||
echo maigret [options] [username]
|
||||
echo.
|
||||
echo For example, to search for a username:
|
||||
echo.
|
||||
echo maigret example_username
|
||||
echo.
|
||||
echo For more options and usage details, refer to the Maigret documentation.
|
||||
echo.
|
||||
echo https://github.com/soxoj/maigret/blob/5b3b81b4822f6deb2e9c31eb95039907f25beb5e/README.md
|
||||
echo +--------------------------------------------------------+
|
||||
echo.
|
||||
cmd
|
||||
pause
|
||||
exit /b
|
||||
exit /b
|
||||
goto:after
|
||||
@@ -1,121 +0,0 @@
|
||||
# Invalid Telegram PR Auto-Closer
|
||||
|
||||
This repository includes an automated solution to identify and close pull requests with titles matching the pattern "Invalid result https://t.me/...". These PRs are typically auto-generated or spam submissions that should not be processed.
|
||||
|
||||
## Components
|
||||
|
||||
### 1. Python Script (`utils/close_invalid_telegram_prs.py`)
|
||||
|
||||
A utility script that:
|
||||
- Searches for open PRs matching the pattern "Invalid result https://t.me/..."
|
||||
- Optionally closes them with a descriptive comment
|
||||
- Supports dry-run mode for testing
|
||||
- Uses the GitHub API to interact with the repository
|
||||
|
||||
#### Usage
|
||||
|
||||
```bash
|
||||
# Dry run (show what would be closed without closing)
|
||||
python utils/close_invalid_telegram_prs.py --dry-run
|
||||
|
||||
# Close matching PRs interactively
|
||||
python utils/close_invalid_telegram_prs.py
|
||||
|
||||
# Close PRs with custom comment
|
||||
python utils/close_invalid_telegram_prs.py --comment "Custom closure message"
|
||||
|
||||
# Use with different repository
|
||||
python utils/close_invalid_telegram_prs.py --owner username --repo repository
|
||||
```
|
||||
|
||||
#### Requirements
|
||||
|
||||
- Python 3.6+
|
||||
- `requests` library: `pip install requests`
|
||||
- GitHub personal access token with repository access
|
||||
|
||||
#### Authentication
|
||||
|
||||
Set your GitHub token via:
|
||||
- Command line: `--token YOUR_TOKEN`
|
||||
- Environment variable: `export GITHUB_TOKEN=YOUR_TOKEN`
|
||||
|
||||
### 2. GitHub Actions Workflow (`.github/workflows/close-invalid-telegram-prs.yml`)
|
||||
|
||||
An automated workflow that:
|
||||
- Runs daily at 2 AM UTC (in dry-run mode by default)
|
||||
- Can be manually triggered with option to actually close PRs
|
||||
- Uses the repository's `GITHUB_TOKEN` for authentication
|
||||
|
||||
#### Manual Trigger
|
||||
|
||||
1. Go to the Actions tab in your GitHub repository
|
||||
2. Select "Close Invalid Telegram PRs" workflow
|
||||
3. Click "Run workflow"
|
||||
4. Choose whether to run in dry-run mode or actually close PRs
|
||||
|
||||
### 3. Tests (`tests/test_close_invalid_telegram_prs.py`)
|
||||
|
||||
Unit tests that verify:
|
||||
- Correct identification of matching PR titles
|
||||
- Proper rejection of non-matching titles
|
||||
- Case-insensitive pattern matching
|
||||
- Whitespace handling
|
||||
|
||||
Run tests with:
|
||||
```bash
|
||||
python tests/test_close_invalid_telegram_prs.py
|
||||
```
|
||||
|
||||
## Pattern Detection
|
||||
|
||||
The script identifies PRs with titles matching:
|
||||
- `Invalid result https://t.me/...` (case insensitive)
|
||||
- Various whitespace and formatting variations
|
||||
- Any Telegram URL after the pattern
|
||||
|
||||
### Examples of Matching Titles
|
||||
|
||||
- "Invalid result https://t.me/someuser"
|
||||
- "INVALID RESULT https://t.me/channel123"
|
||||
- "Invalid Result https://t.me/bot_name"
|
||||
- " Invalid result https://t.me/user/123 " (with whitespace)
|
||||
|
||||
### Examples of Non-Matching Titles
|
||||
|
||||
- "Valid result https://t.me/someuser" (not "Invalid")
|
||||
- "Invalid results https://t.me/someuser" (plural "results")
|
||||
- "Fix invalid result https://t.me/someuser" (extra words)
|
||||
- "Invalid result http://t.me/someuser" (http instead of https)
|
||||
|
||||
## Security
|
||||
|
||||
- The GitHub Actions workflow only has the minimum required permissions
|
||||
- The script requires explicit confirmation before closing PRs (except in automated mode)
|
||||
- All actions are logged and can be audited
|
||||
- Dry-run mode is available for testing
|
||||
|
||||
## Customization
|
||||
|
||||
You can customize the behavior by:
|
||||
- Modifying the regex pattern in `is_invalid_telegram_pr()` function
|
||||
- Changing the default comment message
|
||||
- Adjusting the GitHub Actions schedule
|
||||
- Adding additional validation logic
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Permission Denied**: Ensure your GitHub token has the required permissions
|
||||
2. **No PRs Found**: This is normal if there are no matching PRs
|
||||
3. **Rate Limiting**: The script handles GitHub API rate limits automatically
|
||||
|
||||
### Debug Mode
|
||||
|
||||
Run with verbose output:
|
||||
```bash
|
||||
python utils/close_invalid_telegram_prs.py --dry-run
|
||||
```
|
||||
|
||||
This will show exactly which PRs match the pattern without closing them.
|
||||
+2
-2
@@ -3,10 +3,10 @@
|
||||
# -- Project information
|
||||
|
||||
project = 'Maigret'
|
||||
copyright = '2025, soxoj'
|
||||
copyright = '2024, soxoj'
|
||||
author = 'soxoj'
|
||||
|
||||
release = '0.5.0'
|
||||
release = '0.5.0a1'
|
||||
version = '0.5'
|
||||
|
||||
# -- General configuration
|
||||
|
||||
@@ -194,10 +194,9 @@ PyPi package.
|
||||
|
||||
2. Update Maigret version in three files manually:
|
||||
|
||||
- pyproject.toml
|
||||
- setup.py
|
||||
- maigret/__version__.py
|
||||
- docs/source/conf.py
|
||||
- snapcraft.yaml
|
||||
|
||||
3. Create a new empty text section in the beginning of the file `CHANGELOG.md` with a current date:
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ Usage examples
|
||||
|
||||
You can use Maigret as:
|
||||
|
||||
- a command line tool: initial and a default mode
|
||||
- a command line tool: inital and a default mode
|
||||
- a `web interface <#web-interface>`_: view the graph with results and download all report formats on a single page
|
||||
- a library: integrate Maigret into your own project
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""Maigret version file"""
|
||||
|
||||
__version__ = '0.5.0'
|
||||
__version__ = '0.5.0a1'
|
||||
|
||||
+36
-3
@@ -48,6 +48,15 @@ SUPPORTED_IDS = (
|
||||
BAD_CHARS = "#"
|
||||
|
||||
|
||||
def is_cloudflare_bypass_active(value) -> bool:
|
||||
"""True if Cloudflare webgate URL rewrite should run (``--cloudflare-bypass`` or settings)."""
|
||||
if value is True:
|
||||
return True
|
||||
if isinstance(value, dict):
|
||||
return bool(value.get("enabled", False))
|
||||
return False
|
||||
|
||||
|
||||
class CheckerBase:
|
||||
pass
|
||||
|
||||
@@ -431,6 +440,11 @@ def make_site_result(
|
||||
|
||||
# workaround to prevent slash errors
|
||||
url = re.sub("(?<!:)/+", "/", url)
|
||||
url_probe = site.url_probe
|
||||
|
||||
if 'cloudflare' in site.tags and options.get("cloudflare_bypass"):
|
||||
url_probe = 'http://localhost:8000/html?url=' + url
|
||||
logger.info(f"Using cloudflare proxy for {site.name}")
|
||||
|
||||
# always clearweb_checker for now
|
||||
checker = options["checkers"][site.protocol]
|
||||
@@ -472,7 +486,6 @@ def make_site_result(
|
||||
else:
|
||||
# URL of user on site (if it exists)
|
||||
results_site["url_user"] = url
|
||||
url_probe = site.url_probe
|
||||
if url_probe is None:
|
||||
# Probe URL is normal one seen by people out on the web.
|
||||
url_probe = url
|
||||
@@ -589,6 +602,7 @@ async def maigret(
|
||||
cookies=None,
|
||||
retries=0,
|
||||
check_domains=False,
|
||||
cloudflare_bypass=False,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> QueryResultWrapper:
|
||||
@@ -687,12 +701,14 @@ async def maigret(
|
||||
options["timeout"] = timeout
|
||||
options["id_type"] = id_type
|
||||
options["forced"] = forced
|
||||
options["cloudflare_bypass"] = is_cloudflare_bypass_active(cloudflare_bypass)
|
||||
|
||||
# results from analysis of all sites
|
||||
all_results: Dict[str, QueryResultWrapper] = {}
|
||||
|
||||
sites = list(site_dict.keys())
|
||||
|
||||
executor_limit = timeout + 0.5
|
||||
attempts = retries + 1
|
||||
while attempts:
|
||||
tasks_dict = {}
|
||||
@@ -707,7 +723,11 @@ async def maigret(
|
||||
sitename,
|
||||
'',
|
||||
MaigretCheckStatus.UNKNOWN,
|
||||
error=CheckError('Request failed'),
|
||||
error=CheckError(
|
||||
'Request timeout',
|
||||
f'No response within {executor_limit:.1f}s per site '
|
||||
f'(increase --timeout or use --no-progressbar)',
|
||||
),
|
||||
),
|
||||
}
|
||||
tasks_dict[sitename] = (
|
||||
@@ -788,6 +808,7 @@ async def site_self_check(
|
||||
i2p_proxy=None,
|
||||
skip_errors=False,
|
||||
cookies=None,
|
||||
cloudflare_bypass=False,
|
||||
):
|
||||
changes = {
|
||||
"disabled": False,
|
||||
@@ -815,6 +836,7 @@ async def site_self_check(
|
||||
tor_proxy=tor_proxy,
|
||||
i2p_proxy=i2p_proxy,
|
||||
cookies=cookies,
|
||||
cloudflare_bypass=cloudflare_bypass,
|
||||
)
|
||||
|
||||
# don't disable entries with other ids types
|
||||
@@ -886,6 +908,7 @@ async def self_check(
|
||||
proxy=None,
|
||||
tor_proxy=None,
|
||||
i2p_proxy=None,
|
||||
cloudflare_bypass=False,
|
||||
) -> bool:
|
||||
sem = asyncio.Semaphore(max_connections)
|
||||
tasks = []
|
||||
@@ -901,7 +924,17 @@ async def self_check(
|
||||
|
||||
for _, site in all_sites.items():
|
||||
check_coro = site_self_check(
|
||||
site, logger, sem, db, silent, proxy, tor_proxy, i2p_proxy, skip_errors=True
|
||||
site,
|
||||
logger,
|
||||
sem,
|
||||
db,
|
||||
silent,
|
||||
proxy,
|
||||
tor_proxy,
|
||||
i2p_proxy,
|
||||
skip_errors=True,
|
||||
cookies=None,
|
||||
cloudflare_bypass=cloudflare_bypass,
|
||||
)
|
||||
future = asyncio.ensure_future(check_coro)
|
||||
tasks.append(future)
|
||||
|
||||
+5
-2
@@ -136,7 +136,10 @@ def extract_and_group(search_res: QueryResultWrapper) -> List[Dict[str, Any]]:
|
||||
|
||||
|
||||
def notify_about_errors(
|
||||
search_results: QueryResultWrapper, query_notify, show_statistics=False
|
||||
search_results: QueryResultWrapper,
|
||||
query_notify,
|
||||
show_statistics=False,
|
||||
print_check_errors=False,
|
||||
) -> List[Tuple]:
|
||||
"""
|
||||
Prepare error notifications in search results, text + symbol,
|
||||
@@ -169,7 +172,7 @@ def notify_about_errors(
|
||||
text = f'{e["err"]}: {round(e["perc"],2)}%'
|
||||
results.append((text, '!'))
|
||||
|
||||
if was_errs_displayed:
|
||||
if was_errs_displayed and not print_check_errors:
|
||||
results.append(
|
||||
('You can see detailed site check errors with a flag `--print-errors`', '-')
|
||||
)
|
||||
|
||||
@@ -110,6 +110,28 @@ class AsyncioProgressbarQueueExecutor(AsyncExecutor):
|
||||
self.progress = None
|
||||
|
||||
# TODO: tests
|
||||
@staticmethod
|
||||
def _emit_timeout_notify(args, default_fallback):
|
||||
"""Print per-site line when wait_for kills check_site_for_username (no query_notify.update ran)."""
|
||||
if (
|
||||
not default_fallback
|
||||
or not isinstance(default_fallback, (tuple, list))
|
||||
or len(default_fallback) < 2
|
||||
):
|
||||
return
|
||||
_, results_dict = default_fallback[0], default_fallback[1]
|
||||
if not isinstance(results_dict, dict):
|
||||
return
|
||||
status = results_dict.get('status')
|
||||
if status is None:
|
||||
return
|
||||
if len(args) < 5:
|
||||
return
|
||||
query_notify = args[4]
|
||||
site = results_dict.get('site')
|
||||
similar = bool(getattr(site, 'similar_search', False)) if site is not None else False
|
||||
query_notify.update(status, similar)
|
||||
|
||||
async def increment_progress(self, count):
|
||||
"""Update progress by calling the provided progress function."""
|
||||
if self.progress:
|
||||
@@ -144,6 +166,7 @@ class AsyncioProgressbarQueueExecutor(AsyncExecutor):
|
||||
result = await asyncio.wait_for(query_task, timeout=self.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
result = kwargs.get('default')
|
||||
self._emit_timeout_notify(args, result)
|
||||
|
||||
self.results.append(result)
|
||||
|
||||
@@ -207,6 +230,7 @@ class AsyncioQueueGeneratorExecutor:
|
||||
result = await asyncio.wait_for(query_task, timeout=self.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
result = kwargs.get('default')
|
||||
AsyncioProgressbarQueueExecutor._emit_timeout_notify(args, result)
|
||||
await self._results.put(result)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error in worker: {e}")
|
||||
|
||||
+22
-16
@@ -254,6 +254,12 @@ def setup_arguments_parser(settings: Settings):
|
||||
default=settings.domain_search,
|
||||
help="Enable (experimental) feature of checking domains on usernames.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cloudflare-bypass",
|
||||
action="store_true",
|
||||
default=settings.cloudflare_bypass,
|
||||
help="Enable Cloudflare bypass (edit settings.json to configure)",
|
||||
)
|
||||
|
||||
filter_group = parser.add_argument_group(
|
||||
'Site filtering', 'Options to set site search scope'
|
||||
@@ -493,6 +499,15 @@ async def main():
|
||||
log_level = logging.WARNING
|
||||
logger.setLevel(log_level)
|
||||
|
||||
if args.web is not None:
|
||||
from maigret.web.app import app
|
||||
|
||||
port = (
|
||||
args.web if args.web else 5000
|
||||
) # args.web is either the specified port or 5000 by default
|
||||
app.run(port=port)
|
||||
return
|
||||
|
||||
# Usernames initial list
|
||||
usernames = {
|
||||
u: args.id_type
|
||||
@@ -520,9 +535,7 @@ async def main():
|
||||
if args.tags:
|
||||
args.tags = list(set(str(args.tags).split(',')))
|
||||
|
||||
db_file = args.db_file \
|
||||
if (args.db_file.startswith("http://") or args.db_file.startswith("https://")) \
|
||||
else path.join(path.dirname(path.realpath(__file__)), args.db_file)
|
||||
db_file = path.join(path.dirname(path.realpath(__file__)), args.db_file)
|
||||
|
||||
if args.top_sites == 0 or args.all_sites:
|
||||
args.top_sites = sys.maxsize
|
||||
@@ -574,6 +587,7 @@ async def main():
|
||||
max_connections=args.connections,
|
||||
tor_proxy=args.tor_proxy,
|
||||
i2p_proxy=args.i2p_proxy,
|
||||
cloudflare_bypass=args.cloudflare_bypass,
|
||||
)
|
||||
if is_need_update:
|
||||
if input('Do you want to save changes permanently? [Yn]\n').lower() in (
|
||||
@@ -602,18 +616,6 @@ async def main():
|
||||
# Define one report filename template
|
||||
report_filepath_tpl = path.join(report_dir, 'report_{username}{postfix}')
|
||||
|
||||
# Web interface
|
||||
if args.web is not None:
|
||||
from maigret.web.app import app
|
||||
|
||||
app.config["MAIGRET_DB_FILE"] = db_file
|
||||
|
||||
port = (
|
||||
args.web if args.web else 5000
|
||||
) # args.web is either the specified port or 5000 by default
|
||||
app.run(port=port)
|
||||
return
|
||||
|
||||
if usernames == {}:
|
||||
# magic params to exit after init
|
||||
query_notify.warning('No usernames to check, exiting.')
|
||||
@@ -686,10 +688,14 @@ async def main():
|
||||
no_progressbar=args.no_progressbar,
|
||||
retries=args.retries,
|
||||
check_domains=args.with_domains,
|
||||
cloudflare_bypass=args.cloudflare_bypass,
|
||||
)
|
||||
|
||||
errs = errors.notify_about_errors(
|
||||
results, query_notify, show_statistics=args.verbose
|
||||
results,
|
||||
query_notify,
|
||||
show_statistics=args.verbose,
|
||||
print_check_errors=args.print_check_errors,
|
||||
)
|
||||
for e in errs:
|
||||
query_notify.warning(*e)
|
||||
|
||||
+8
-28
@@ -144,9 +144,7 @@ def save_graph_report(filename: str, username_results: list, db: MaigretDatabase
|
||||
# base site node
|
||||
site_base_url = website_name
|
||||
if site_base_url not in base_site_nodes:
|
||||
base_site_nodes[site_base_url] = graph.add_node(
|
||||
'site', site_base_url, color='#28a745'
|
||||
) # Green color
|
||||
base_site_nodes[site_base_url] = graph.add_node('site', site_base_url, color='#28a745') # Green color
|
||||
|
||||
site_base_node_name = base_site_nodes[site_base_url]
|
||||
|
||||
@@ -154,9 +152,7 @@ def save_graph_report(filename: str, username_results: list, db: MaigretDatabase
|
||||
account_url = dictionary.get('url_user', f'{site_base_url}/{norm_username}')
|
||||
account_node_id = f"{site_base_url}: {account_url}"
|
||||
if account_node_id not in site_account_nodes:
|
||||
site_account_nodes[account_node_id] = graph.add_node(
|
||||
'account', account_url
|
||||
)
|
||||
site_account_nodes[account_node_id] = graph.add_node('account', account_url)
|
||||
|
||||
account_node_name = site_account_nodes[account_node_id]
|
||||
|
||||
@@ -166,12 +162,7 @@ def save_graph_report(filename: str, username_results: list, db: MaigretDatabase
|
||||
|
||||
def process_ids(parent_node, ids):
|
||||
for k, v in ids.items():
|
||||
if (
|
||||
k.endswith('_count')
|
||||
or k.startswith('is_')
|
||||
or k.endswith('_at')
|
||||
or k in 'image'
|
||||
):
|
||||
if k.endswith('_count') or k.startswith('is_') or k.endswith('_at') or k in 'image':
|
||||
continue
|
||||
|
||||
# Normalize value if string
|
||||
@@ -196,9 +187,7 @@ def save_graph_report(filename: str, username_results: list, db: MaigretDatabase
|
||||
data_node_name = graph.add_node(vv, site_base_url)
|
||||
graph.link(list_node_name, data_node_name)
|
||||
|
||||
add_ids = {
|
||||
a: b for b, a in db.extract_ids_from_url(vv).items()
|
||||
}
|
||||
add_ids = {a: b for b, a in db.extract_ids_from_url(vv).items()}
|
||||
if add_ids:
|
||||
process_ids(data_node_name, add_ids)
|
||||
ids_data_name = list_node_name
|
||||
@@ -209,17 +198,11 @@ def save_graph_report(filename: str, username_results: list, db: MaigretDatabase
|
||||
if 'username' in k or k in SUPPORTED_IDS:
|
||||
new_username_key = f"username:{norm_v}"
|
||||
if new_username_key not in processed_values:
|
||||
new_username_node_name = graph.add_node(
|
||||
'username', norm_v
|
||||
)
|
||||
processed_values[new_username_key] = (
|
||||
new_username_node_name
|
||||
)
|
||||
new_username_node_name = graph.add_node('username', norm_v)
|
||||
processed_values[new_username_key] = new_username_node_name
|
||||
graph.link(ids_data_name, new_username_node_name)
|
||||
|
||||
add_ids = {
|
||||
k: v for v, k in db.extract_ids_from_url(v).items()
|
||||
}
|
||||
add_ids = {k: v for v, k in db.extract_ids_from_url(v).items()}
|
||||
if add_ids:
|
||||
process_ids(ids_data_name, add_ids)
|
||||
|
||||
@@ -233,14 +216,11 @@ def save_graph_report(filename: str, username_results: list, db: MaigretDatabase
|
||||
G.remove_nodes_from(nodes_to_remove)
|
||||
|
||||
# Remove site nodes with only one connection
|
||||
single_degree_sites = [
|
||||
n for n, deg in G.degree() if n.startswith("site:") and deg <= 1
|
||||
]
|
||||
single_degree_sites = [n for n, deg in G.degree() if n.startswith("site:") and deg <= 1]
|
||||
G.remove_nodes_from(single_degree_sites)
|
||||
|
||||
# Generate interactive visualization
|
||||
from pyvis.network import Network
|
||||
|
||||
nt = Network(notebook=True, height="750px", width="100%")
|
||||
nt.from_nx(G)
|
||||
nt.show(filename)
|
||||
|
||||
+19
-76
@@ -1451,7 +1451,6 @@
|
||||
"usernameUnclaimed": "noonewouldeverusethis7"
|
||||
},
|
||||
"AskFM": {
|
||||
"disabled": true,
|
||||
"tags": [
|
||||
"eg",
|
||||
"in",
|
||||
@@ -3150,6 +3149,7 @@
|
||||
},
|
||||
"ChaturBate": {
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"us"
|
||||
],
|
||||
"checkType": "status_code",
|
||||
@@ -3329,6 +3329,7 @@
|
||||
},
|
||||
"CloudflareCommunity": {
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"forum",
|
||||
"tech"
|
||||
],
|
||||
@@ -4203,6 +4204,7 @@
|
||||
},
|
||||
"Discogs": {
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"music",
|
||||
"us"
|
||||
],
|
||||
@@ -4919,6 +4921,7 @@
|
||||
},
|
||||
"Etsy": {
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"shopping",
|
||||
"us"
|
||||
],
|
||||
@@ -5038,6 +5041,7 @@
|
||||
"usernameUnclaimed": "noonewouldeverusethis7",
|
||||
"alexaRank": 240,
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"design"
|
||||
]
|
||||
},
|
||||
@@ -6164,16 +6168,6 @@
|
||||
"usernameClaimed": "adam",
|
||||
"usernameUnclaimed": "noonewouldeverusethis7"
|
||||
},
|
||||
"Framapiaf": {
|
||||
"tags": [
|
||||
"mastodon"
|
||||
],
|
||||
"checkType": "status_code",
|
||||
"urlMain": "https://framapiaf.org",
|
||||
"url": "https://framapiaf.org/@{username}",
|
||||
"usernameClaimed": "pylapp",
|
||||
"usernameUnclaimed": "noonewouldeverusethis42"
|
||||
},
|
||||
"Free-lance.ua": {
|
||||
"tags": [
|
||||
"freelance",
|
||||
@@ -6281,6 +6275,7 @@
|
||||
"Freepik": {
|
||||
"tags": [
|
||||
"art",
|
||||
"cloudflare",
|
||||
"photo",
|
||||
"stock"
|
||||
],
|
||||
@@ -7896,7 +7891,6 @@
|
||||
"usernameUnclaimed": "noonewouldeverusethis7"
|
||||
},
|
||||
"ICQ": {
|
||||
"disabled": true,
|
||||
"tags": [
|
||||
"ch",
|
||||
"ru",
|
||||
@@ -8679,6 +8673,7 @@
|
||||
},
|
||||
"Kickstarter": {
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"finance",
|
||||
"us"
|
||||
],
|
||||
@@ -9792,16 +9787,6 @@
|
||||
"usernameUnclaimed": "noonewouldeverusethis7",
|
||||
"alexaRank": 4938389
|
||||
},
|
||||
"Mamot": {
|
||||
"tags": [
|
||||
"mastodon"
|
||||
],
|
||||
"checkType": "status_code",
|
||||
"urlMain": "https://mamot.fr",
|
||||
"url": "https://mamot.fr/@{username}",
|
||||
"usernameClaimed": "pylapp",
|
||||
"usernameUnclaimed": "noonewouldeverusethis42"
|
||||
},
|
||||
"Mamuli": {
|
||||
"tags": [
|
||||
"ru",
|
||||
@@ -11807,16 +11792,6 @@
|
||||
"usernameClaimed": "uehkon89",
|
||||
"usernameUnclaimed": "noonewouldeverusethis7"
|
||||
},
|
||||
"Outgress": {
|
||||
"checkType": "message",
|
||||
"absenceStrs": [
|
||||
"Outgress - Error"
|
||||
],
|
||||
"url": "https://outgress.com/agents/{username}",
|
||||
"urlMain": "https://outgress.com/",
|
||||
"usernameClaimed": "pylapp",
|
||||
"usernameUnclaimed": "noonewouldeverusethis42"
|
||||
},
|
||||
"Overclockers": {
|
||||
"tags": [
|
||||
"ru"
|
||||
@@ -12054,6 +12029,7 @@
|
||||
},
|
||||
"Patreon": {
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"finance"
|
||||
],
|
||||
"checkType": "status_code",
|
||||
@@ -12436,17 +12412,6 @@
|
||||
"usernameClaimed": "adam",
|
||||
"usernameUnclaimed": "noonewouldeverusethis7"
|
||||
},
|
||||
"pixelfed.social": {
|
||||
"tags": [
|
||||
"art",
|
||||
"pixelfed"
|
||||
],
|
||||
"checkType": "status_code",
|
||||
"usernameClaimed": "pylapp",
|
||||
"usernameUnclaimed": "noonewouldeverusethis42",
|
||||
"urlMain": "https://pixelfed.social/",
|
||||
"url": "https://pixelfed.social/{username}/"
|
||||
},
|
||||
"PlanetMinecraft": {
|
||||
"tags": [
|
||||
"us"
|
||||
@@ -13119,19 +13084,6 @@
|
||||
"usernameClaimed": "Sinkler",
|
||||
"usernameUnclaimed": "noonewouldeverusethis7"
|
||||
},
|
||||
"programming.dev": {
|
||||
"tags": [
|
||||
"lemmy"
|
||||
],
|
||||
"checkType": "message",
|
||||
"absenceStrs": [
|
||||
"Error!"
|
||||
],
|
||||
"url": "https://programming.dev/u/{username}",
|
||||
"urlMain": "https://programming.dev",
|
||||
"usernameClaimed": "pylapp",
|
||||
"usernameUnclaimed": "noonewouldeverusethis42"
|
||||
},
|
||||
"Qbn": {
|
||||
"tags": [
|
||||
"in",
|
||||
@@ -13570,6 +13522,7 @@
|
||||
},
|
||||
"Redbubble": {
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"shopping",
|
||||
"us"
|
||||
],
|
||||
@@ -15114,6 +15067,7 @@
|
||||
},
|
||||
"SourceForge": {
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"coding",
|
||||
"us"
|
||||
],
|
||||
@@ -16966,10 +16920,14 @@
|
||||
},
|
||||
"Twitch": {
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"streaming",
|
||||
"us"
|
||||
],
|
||||
"urlProbe": "https://twitchtracker.com/{username}",
|
||||
"headers": {
|
||||
"cookies": "_gat=1; _ga_4ZM72D0Y59=GS1.2.1734305902.1.0.1734305902.0.0.0; _ga=GA1.2.1051951095.1734305902; _gid=GA1.2.30506583.1734305902; cf_clearance=xo5uTOkBRsAYb4so4QSu1h8tlFcFJyYSA2SBEHyYA2U-1734305900-1.2.1.1-l9mQ677uPsBenPceAasuW_ZVBqRgQqy4df.13gRl6y4aFBf._3bLo1c3.uVZOXwMxL_iVN.EvEHEBiNczBNMM6riJrVWgiLx1O1jGRbhIiGP.tsomZgyl_bNupNbWxZNzHy454hC0iUigDrE5jkJJoazDRJNc5532wj9nT.U9DDBxW3RplVCdj4x5sMt3K3IXADYvAGabBQnzvS3rEr_w66KClwAehy69tWHVSPDkc.ww7QnxdDItYqmtL8bz9IScdouTAvU_MWK6oxvxcLc6GQFCQZnoToeX8Fgeui2flhV.kXXjEQ4NjypxSFakcCPIysHZOUjKfv93.W9Vfl7id.Y8DUpsmxEPVOpfcrGY6YvFtk6yJhvUQryJftS5b7E5P5jVPW_pPlMWSTWL9IaysG7INm6ZjDyjsVG7OBJIUujSSMlyoKiR8sv0L2ueHt6",
|
||||
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
|
||||
},
|
||||
"checkType": "status_code",
|
||||
"alexaRank": 34,
|
||||
"urlMain": "https://www.twitch.tv/",
|
||||
@@ -17135,7 +17093,7 @@
|
||||
},
|
||||
"Udemy": {
|
||||
"tags": [
|
||||
"in"
|
||||
"cloudflare"
|
||||
],
|
||||
"checkType": "response_url",
|
||||
"alexaRank": 131,
|
||||
@@ -17532,9 +17490,6 @@
|
||||
],
|
||||
"method": "vimeo"
|
||||
},
|
||||
"headers": {
|
||||
"Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3MzQxMTc1NDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbCwianRpIjoiNDc4Y2ZhZGUtZjI0Yy00MDVkLTliYWItN2RlNGEzNGM4MzI5In0.guN7Fg8dqq7EYdckrJ-6Rdkj_5MOl6FaC4YUSOceDpU"
|
||||
},
|
||||
"urlProbe": "https://api.vimeo.com/users/{username}?fields=name%2Cgender%2Cbio%2Curi%2Clink%2Cbackground_video%2Clocation_details%2Cpictures%2Cverified%2Cmetadata.public_videos.total%2Cavailable_for_hire%2Ccan_work_remotely%2Cmetadata.connections.videos.total%2Cmetadata.connections.albums.total%2Cmetadata.connections.followers.total%2Cmetadata.connections.following.total%2Cmetadata.public_videos.total%2Cmetadata.connections.vimeo_experts.is_enrolled%2Ctotal_collection_count%2Ccreated_time%2Cprofile_preferences%2Cmembership%2Cclients%2Cskills%2Cproject_types%2Crates%2Ccategories%2Cis_expert%2Cprofile_discovery%2Cwebsites%2Ccontact_emails&fetch_user_profile=1",
|
||||
"checkType": "status_code",
|
||||
"alexaRank": 148,
|
||||
@@ -27300,6 +27255,7 @@
|
||||
},
|
||||
"upwork.com": {
|
||||
"tags": [
|
||||
"cloudflare",
|
||||
"us"
|
||||
],
|
||||
"engine": "engine404",
|
||||
@@ -35600,16 +35556,6 @@
|
||||
"tags": [
|
||||
"tr"
|
||||
]
|
||||
},
|
||||
"write.as": {
|
||||
"tags": [
|
||||
"writefreely"
|
||||
],
|
||||
"checkType": "status_code",
|
||||
"url": "https://write.as/{username}",
|
||||
"urlMain": "https://write.as",
|
||||
"usernameClaimed": "pylapp",
|
||||
"usernameUnclaimed": "noonewouldeverusethis42"
|
||||
}
|
||||
},
|
||||
"engines": {
|
||||
@@ -35835,6 +35781,7 @@
|
||||
"tags": [
|
||||
"gaming",
|
||||
"coding",
|
||||
"cloudflare",
|
||||
"photo",
|
||||
"music",
|
||||
"blog",
|
||||
@@ -35891,10 +35838,6 @@
|
||||
"i2p",
|
||||
"q&a",
|
||||
"crypto",
|
||||
"ai",
|
||||
"mastodon",
|
||||
"writefreely",
|
||||
"lemmy",
|
||||
"pixelfed"
|
||||
"ai"
|
||||
]
|
||||
}
|
||||
@@ -54,5 +54,15 @@
|
||||
"graph_report": false,
|
||||
"pdf_report": false,
|
||||
"html_report": false,
|
||||
"web_interface_port": 5000
|
||||
"web_interface_port": 5000,
|
||||
"cloudflare_bypass": {
|
||||
"enabled": true,
|
||||
"modules": [
|
||||
{
|
||||
"name": "chrome_webgate",
|
||||
"method": "url_rewrite",
|
||||
"url": "http://localhost:8000/html?url={url}&retries=1"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -110,6 +110,7 @@ class Submitter:
|
||||
cookies=self.args.cookie_file,
|
||||
# Don't skip errors in submit mode - we need check both false positives/true negatives
|
||||
skip_errors=False,
|
||||
cloudflare_bypass=getattr(self.args, 'cloudflare_bypass', False),
|
||||
)
|
||||
return changes
|
||||
|
||||
|
||||
+24
-34
@@ -21,15 +21,18 @@ from maigret.report import generate_report_context
|
||||
app = Flask(__name__)
|
||||
app.secret_key = 'your-secret-key-here'
|
||||
|
||||
# add background job tracking
|
||||
#add background job tracking
|
||||
background_jobs = {}
|
||||
job_results = {}
|
||||
|
||||
# Configuration
|
||||
app.config["MAIGRET_DB_FILE"] = os.path.join('maigret', 'resources', 'data.json')
|
||||
app.config["COOKIES_FILE"] = "cookies.txt"
|
||||
app.config["UPLOAD_FOLDER"] = 'uploads'
|
||||
app.config["REPORTS_FOLDER"] = os.path.abspath('/tmp/maigret_reports')
|
||||
MAIGRET_DB_FILE = os.path.join('maigret', 'resources', 'data.json')
|
||||
COOKIES_FILE = "cookies.txt"
|
||||
UPLOAD_FOLDER = 'uploads'
|
||||
REPORTS_FOLDER = os.path.abspath('/tmp/maigret_reports')
|
||||
|
||||
os.makedirs(UPLOAD_FOLDER, exist_ok=True)
|
||||
os.makedirs(REPORTS_FOLDER, exist_ok=True)
|
||||
|
||||
|
||||
def setup_logger(log_level, name):
|
||||
@@ -41,14 +44,14 @@ def setup_logger(log_level, name):
|
||||
async def maigret_search(username, options):
|
||||
logger = setup_logger(logging.WARNING, 'maigret')
|
||||
try:
|
||||
db = MaigretDatabase().load_from_path(app.config["MAIGRET_DB_FILE"])
|
||||
db = MaigretDatabase().load_from_path(MAIGRET_DB_FILE)
|
||||
|
||||
top_sites = int(options.get('top_sites') or 500)
|
||||
if options.get('all_sites'):
|
||||
top_sites = 999999999 # effectively all
|
||||
|
||||
tags = options.get('tags', [])
|
||||
site_list = options.get('site_list', [])
|
||||
site_list= options.get('site_list', [])
|
||||
logger.info(f"Filtering sites by tags: {tags}")
|
||||
|
||||
sites = db.ranked_sites_dict(
|
||||
@@ -56,7 +59,7 @@ async def maigret_search(username, options):
|
||||
tags=tags,
|
||||
names=site_list,
|
||||
disabled=False,
|
||||
id_type='username',
|
||||
id_type='username'
|
||||
)
|
||||
|
||||
logger.info(f"Found {len(sites)} sites matching the tag criteria")
|
||||
@@ -67,11 +70,9 @@ async def maigret_search(username, options):
|
||||
timeout=int(options.get('timeout', 30)),
|
||||
logger=logger,
|
||||
id_type='username',
|
||||
cookies=app.config["COOKIES_FILE"] if options.get('use_cookies') else None,
|
||||
cookies=COOKIES_FILE if options.get('use_cookies') else None,
|
||||
is_parsing_enabled=(not options.get('disable_extracting', False)),
|
||||
recursive_search_enabled=(
|
||||
not options.get('disable_recursive_search', False)
|
||||
),
|
||||
recursive_search_enabled=(not options.get('disable_recursive_search', False)),
|
||||
check_domains=options.get('with_domains', False),
|
||||
proxy=options.get('proxy', None),
|
||||
tor_proxy=options.get('tor_proxy', None),
|
||||
@@ -103,17 +104,14 @@ def process_search_task(usernames, options, timestamp):
|
||||
search_multiple_usernames(usernames, options)
|
||||
)
|
||||
|
||||
os.makedirs(app.config["REPORTS_FOLDER"], exist_ok=True)
|
||||
session_folder = os.path.join(
|
||||
app.config["REPORTS_FOLDER"], f"search_{timestamp}"
|
||||
)
|
||||
session_folder = os.path.join(REPORTS_FOLDER, f"search_{timestamp}")
|
||||
os.makedirs(session_folder, exist_ok=True)
|
||||
|
||||
graph_path = os.path.join(session_folder, "combined_graph.html")
|
||||
maigret.report.save_graph_report(
|
||||
graph_path,
|
||||
general_results,
|
||||
MaigretDatabase().load_from_path(app.config["MAIGRET_DB_FILE"]),
|
||||
MaigretDatabase().load_from_path(MAIGRET_DB_FILE),
|
||||
)
|
||||
|
||||
individual_reports = []
|
||||
@@ -190,18 +188,18 @@ def process_search_task(usernames, options, timestamp):
|
||||
|
||||
@app.route('/')
|
||||
def index():
|
||||
# load site data for autocomplete
|
||||
db = MaigretDatabase().load_from_path(app.config["MAIGRET_DB_FILE"])
|
||||
#load site data for autocomplete
|
||||
db = MaigretDatabase().load_from_path(MAIGRET_DB_FILE)
|
||||
site_options = []
|
||||
|
||||
for site in db.sites:
|
||||
# add main site name
|
||||
#add main site name
|
||||
site_options.append(site.name)
|
||||
# add URL if different from name
|
||||
#add URL if different from name
|
||||
if site.url_main and site.url_main not in site_options:
|
||||
site_options.append(site.url_main)
|
||||
|
||||
# sort and deduplicate
|
||||
#sort and deduplicate
|
||||
site_options = sorted(set(site_options))
|
||||
|
||||
return render_template('index.html', site_options=site_options)
|
||||
@@ -239,14 +237,10 @@ def search():
|
||||
'i2p_proxy': request.form.get('i2p_proxy', None) or None,
|
||||
'permute': 'permute' in request.form,
|
||||
'tags': selected_tags, # Pass selected tags as a list
|
||||
'site_list': [
|
||||
s.strip() for s in request.form.get('site', '').split(',') if s.strip()
|
||||
],
|
||||
'site_list': [s.strip() for s in request.form.get('site', '').split(',') if s.strip()],
|
||||
}
|
||||
|
||||
logging.info(
|
||||
f"Starting search for usernames: {usernames} with tags: {selected_tags}"
|
||||
)
|
||||
logging.info(f"Starting search for usernames: {usernames} with tags: {selected_tags}")
|
||||
|
||||
# Start background job
|
||||
background_jobs[timestamp] = {
|
||||
@@ -259,7 +253,6 @@ def search():
|
||||
|
||||
return redirect(url_for('status', timestamp=timestamp))
|
||||
|
||||
|
||||
@app.route('/status/<timestamp>')
|
||||
def status(timestamp):
|
||||
logging.info(f"Status check for timestamp: {timestamp}")
|
||||
@@ -320,11 +313,8 @@ def results(session_id):
|
||||
@app.route('/reports/<path:filename>')
|
||||
def download_report(filename):
|
||||
try:
|
||||
os.makedirs(app.config["REPORTS_FOLDER"], exist_ok=True)
|
||||
file_path = os.path.normpath(
|
||||
os.path.join(app.config["REPORTS_FOLDER"], filename)
|
||||
)
|
||||
if not file_path.startswith(app.config["REPORTS_FOLDER"]):
|
||||
file_path = os.path.normpath(os.path.join(REPORTS_FOLDER, filename))
|
||||
if not file_path.startswith(REPORTS_FOLDER):
|
||||
raise Exception("Invalid file path")
|
||||
return send_file(file_path)
|
||||
except Exception as e:
|
||||
|
||||
Generated
+746
-932
File diff suppressed because it is too large
Load Diff
+21
-21
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.poetry]
|
||||
name = "maigret"
|
||||
version = "0.5.0"
|
||||
version = "0.5.0a1"
|
||||
description = "🕵️♂️ Collect a dossier on a person by username from thousands of sites."
|
||||
authors = ["Soxoj <soxoj@protonmail.com>"]
|
||||
readme = "README.md"
|
||||
@@ -32,28 +32,28 @@ classifiers = [
|
||||
# poetry install --with dev
|
||||
python = "^3.10"
|
||||
aiodns = "^3.0.0"
|
||||
aiohttp = "^3.12.14"
|
||||
aiohttp-socks = "^0.10.1"
|
||||
aiohttp = "^3.11.11"
|
||||
aiohttp-socks = "^0.9.1"
|
||||
arabic-reshaper = "^3.0.0"
|
||||
async-timeout = "^5.0.1"
|
||||
attrs = "^25.3.0"
|
||||
certifi = "^2025.6.15"
|
||||
attrs = "^24.3.0"
|
||||
certifi = "^2024.12.14"
|
||||
chardet = "^5.0.0"
|
||||
colorama = "^0.4.6"
|
||||
future = "^1.0.0"
|
||||
future-annotations= "^1.0.0"
|
||||
html5lib = "^1.1"
|
||||
idna = "^3.4"
|
||||
Jinja2 = "^3.1.6"
|
||||
lxml = ">=5.3,<7.0"
|
||||
Jinja2 = "^3.1.3"
|
||||
lxml = "^5.3.0"
|
||||
MarkupSafe = "^3.0.2"
|
||||
mock = "^5.1.0"
|
||||
multidict = "^6.6.3"
|
||||
multidict = "^6.0.4"
|
||||
pycountry = "^24.6.1"
|
||||
PyPDF2 = "^3.0.1"
|
||||
PySocks = "^1.7.1"
|
||||
python-bidi = "^0.6.3"
|
||||
requests = "^2.32.4"
|
||||
requests = "^2.31.0"
|
||||
requests-futures = "^1.0.2"
|
||||
six = "^1.17.0"
|
||||
socid-extractor = "^0.0.27"
|
||||
@@ -61,18 +61,18 @@ soupsieve = "^2.6"
|
||||
stem = "^1.8.1"
|
||||
torrequest = "^0.1.0"
|
||||
alive_progress = "^3.2.0"
|
||||
typing-extensions = "^4.14.1"
|
||||
typing-extensions = "^4.8.0"
|
||||
webencodings = "^0.5.1"
|
||||
xhtml2pdf = "^0.2.11"
|
||||
XMind = "^1.2.0"
|
||||
yarl = "^1.20.1"
|
||||
yarl = "^1.18.3"
|
||||
networkx = "^2.6.3"
|
||||
pyvis = "^0.3.2"
|
||||
reportlab = "^4.4.3"
|
||||
reportlab = "^4.2.0"
|
||||
cloudscraper = "^1.2.71"
|
||||
flask = {extras = ["async"], version = "^3.1.1"}
|
||||
asgiref = "^3.9.1"
|
||||
platformdirs = "^4.3.8"
|
||||
flask = {extras = ["async"], version = "^3.1.0"}
|
||||
asgiref = "^3.8.1"
|
||||
platformdirs = "^4.3.6"
|
||||
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
@@ -80,15 +80,15 @@ platformdirs = "^4.3.8"
|
||||
# Install dev dependencies with: poetry install --with dev
|
||||
flake8 = "^7.1.1"
|
||||
pytest = "^8.3.4"
|
||||
pytest-asyncio = "^1.0.0"
|
||||
pytest-asyncio = "^0.25.0"
|
||||
pytest-cov = "^6.0.0"
|
||||
pytest-httpserver = "^1.0.0"
|
||||
pytest-rerunfailures = "^15.1"
|
||||
reportlab = "^4.4.3"
|
||||
mypy = "^1.14.1"
|
||||
pytest-rerunfailures = "^15.0"
|
||||
reportlab = "^4.2.0"
|
||||
mypy = "^1.13.0"
|
||||
tuna = "^0.5.11"
|
||||
coverage = "^7.9.2"
|
||||
black = "^25.1.0"
|
||||
coverage = "^7.6.9"
|
||||
black = "^24.10.0"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
# Run with: poetry run maigret <username>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
|
||||
## List of supported sites (search methods): total 3143
|
||||
## List of supported sites (search methods): total 3137
|
||||
|
||||
Rank data fetched from Alexa by domains.
|
||||
|
||||
@@ -21,7 +21,7 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [VK (by id) (https://vk.com/)](https://vk.com/)*: top 50, ru*
|
||||
1.  [BongaCams (https://sbongacams.com)](https://sbongacams.com)*: top 50, cz, webcam*
|
||||
1.  [Instagram (https://www.instagram.com/)](https://www.instagram.com/)*: top 50, photo*, search is disabled
|
||||
1.  [Twitch (https://www.twitch.tv/)](https://www.twitch.tv/)*: top 50, streaming, us*
|
||||
1.  [Twitch (https://www.twitch.tv/)](https://www.twitch.tv/)*: top 50, cloudflare, streaming, us*
|
||||
1.  [YandexCollections API (https://yandex.ru/collections/)](https://yandex.ru/collections/)*: top 50, ru, sharing*, search is disabled
|
||||
1.  [StackOverflow (https://stackoverflow.com)](https://stackoverflow.com)*: top 50, coding*
|
||||
1.  [Ebay (https://www.ebay.com/)](https://www.ebay.com/)*: top 50, shopping, us*
|
||||
@@ -62,7 +62,7 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [community.adobe.com (https://community.adobe.com)](https://community.adobe.com)*: top 100, us*
|
||||
1.  [TradingView (https://www.tradingview.com/)](https://www.tradingview.com/)*: top 100, trading, us*
|
||||
1.  [Aparat (https://www.aparat.com)](https://www.aparat.com)*: top 100, ir, video*
|
||||
1.  [ChaturBate (https://chaturbate.com)](https://chaturbate.com)*: top 100, us*
|
||||
1.  [ChaturBate (https://chaturbate.com)](https://chaturbate.com)*: top 100, cloudflare, us*
|
||||
1.  [Medium (https://medium.com/)](https://medium.com/)*: top 100, blog, us*, search is disabled
|
||||
1.  [Livejasmin (https://www.livejasmin.com/)](https://www.livejasmin.com/)*: top 100, us, webcam*
|
||||
1.  [Pornhub (https://pornhub.com/)](https://pornhub.com/)*: top 100, porn*
|
||||
@@ -72,7 +72,7 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [BleachFandom (https://bleach.fandom.com/ru)](https://bleach.fandom.com/ru)*: top 100, ru, wiki*
|
||||
1.  [Fandom (https://www.fandom.com/)](https://www.fandom.com/)*: top 100, us*
|
||||
1.  [FandomCommunityCentral (https://community.fandom.com)](https://community.fandom.com)*: top 100, wiki*
|
||||
1.  [Etsy (https://www.etsy.com/)](https://www.etsy.com/)*: top 100, shopping, us*
|
||||
1.  [Etsy (https://www.etsy.com/)](https://www.etsy.com/)*: top 100, cloudflare, shopping, us*
|
||||
1.  [GitHub (https://www.github.com/)](https://www.github.com/)*: top 100, coding*
|
||||
1.  [Spotify (https://open.spotify.com/)](https://open.spotify.com/)*: top 100, music, us*, search is disabled
|
||||
1.  [TikTok (https://www.tiktok.com/)](https://www.tiktok.com/)*: top 100, video*
|
||||
@@ -80,14 +80,14 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [Tumblr (https://www.tumblr.com)](https://www.tumblr.com)*: top 500, blog*
|
||||
1.  [Roblox (https://www.roblox.com/)](https://www.roblox.com/)*: top 500, gaming, us*
|
||||
1.  [SoundCloud (https://soundcloud.com/)](https://soundcloud.com/)*: top 500, music*
|
||||
1.  [Udemy (https://www.udemy.com)](https://www.udemy.com)*: top 500, in*
|
||||
1.  [Udemy (https://www.udemy.com)](https://www.udemy.com)*: top 500, cloudflare*
|
||||
1.  [discourse.mozilla.org (https://discourse.mozilla.org)](https://discourse.mozilla.org)*: top 500*
|
||||
1.  [linktr.ee (https://linktr.ee)](https://linktr.ee)*: top 500, links*
|
||||
1.  [xHamster (https://xhamster.com)](https://xhamster.com)*: top 500, porn, us*
|
||||
1.  [Zhihu (https://www.zhihu.com/)](https://www.zhihu.com/)*: top 500, cn*, search is disabled
|
||||
1.  [Blogger (by GAIA id) (https://www.blogger.com)](https://www.blogger.com)*: top 500, blog*
|
||||
1.  [ResearchGate (https://www.researchgate.net/)](https://www.researchgate.net/)*: top 500, in, us*
|
||||
1.  [Freepik (https://www.freepik.com)](https://www.freepik.com)*: top 500, art, photo, stock*
|
||||
1.  [Freepik (https://www.freepik.com)](https://www.freepik.com)*: top 500, art, cloudflare, photo, stock*
|
||||
1.  [Vimeo (https://vimeo.com)](https://vimeo.com)*: top 500, video*
|
||||
1.  [Pinterest (https://www.pinterest.com/)](https://www.pinterest.com/)*: top 500, art, photo, sharing*
|
||||
1.  [Fiverr (https://www.fiverr.com/)](https://www.fiverr.com/)*: top 500, shopping, us*
|
||||
@@ -101,9 +101,9 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [Wix (https://wix.com/)](https://wix.com/)*: top 500, us*
|
||||
1.  [Slack (https://slack.com)](https://slack.com)*: top 500, messaging*
|
||||
1.  [Chess (https://www.chess.com)](https://www.chess.com)*: top 500, gaming, hobby*
|
||||
1.  [upwork.com (https://upwork.com)](https://upwork.com)*: top 500, us*
|
||||
1.  [upwork.com (https://upwork.com)](https://upwork.com)*: top 500, cloudflare, us*
|
||||
1.  [Archive.org (https://archive.org)](https://archive.org)*: top 500*, search is disabled
|
||||
1.  [Figma (https://www.figma.com/)](https://www.figma.com/)*: top 500, design*
|
||||
1.  [Figma (https://www.figma.com/)](https://www.figma.com/)*: top 500, cloudflare, design*
|
||||
1.  [iStock (https://www.istockphoto.com)](https://www.istockphoto.com)*: top 500, photo, stock*
|
||||
1.  [Scribd (https://www.scribd.com/)](https://www.scribd.com/)*: top 500, reading*
|
||||
1.  [opensea.io (https://opensea.io)](https://opensea.io)*: top 500, us*
|
||||
@@ -112,7 +112,7 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [Yelp (http://www.yelp.com)](http://www.yelp.com)*: top 500, review*, search is disabled
|
||||
1.  [Yelp (by id) (https://www.yelp.com)](https://www.yelp.com)*: top 500, review*
|
||||
1.  [Blogger (https://www.blogger.com/)](https://www.blogger.com/)*: top 500, blog*
|
||||
1.  [Patreon (https://www.patreon.com/)](https://www.patreon.com/)*: top 500, finance*
|
||||
1.  [Patreon (https://www.patreon.com/)](https://www.patreon.com/)*: top 500, cloudflare, finance*
|
||||
1.  [GoodReads (https://www.goodreads.com/)](https://www.goodreads.com/)*: top 500, books, us*
|
||||
1.  [OP.GG [LeagueOfLegends] Brazil (https://www.op.gg/)](https://www.op.gg/)*: top 500, br, gaming*
|
||||
1.  [OP.GG [LeagueOfLegends] North America (https://www.op.gg/)](https://www.op.gg/)*: top 500, gaming*
|
||||
@@ -145,7 +145,7 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [Weebly (http://weebly.com)](http://weebly.com)*: top 500, business*
|
||||
1.  [RamblerDating (https://dating.rambler.ru/)](https://dating.rambler.ru/)*: top 500, dating, ru*, search is disabled
|
||||
1.  [LiveJournal (https://www.livejournal.com/)](https://www.livejournal.com/)*: top 500, blog, ru*
|
||||
1.  [SourceForge (https://sourceforge.net/)](https://sourceforge.net/)*: top 500, coding, us*
|
||||
1.  [SourceForge (https://sourceforge.net/)](https://sourceforge.net/)*: top 500, cloudflare, coding, us*
|
||||
1.  [Genius (https://genius.com/)](https://genius.com/)*: top 500, music, us*
|
||||
1.  [Issuu (https://issuu.com/)](https://issuu.com/)*: top 500, business*
|
||||
1.  [9GAG (https://www.9gag.com/)](https://www.9gag.com/)*: top 500, sharing*
|
||||
@@ -162,7 +162,7 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [cyber.harvard.edu (https://cyber.harvard.edu)](https://cyber.harvard.edu)*: top 1K, us*
|
||||
1.  [Duolingo (https://duolingo.com/)](https://duolingo.com/)*: top 1K, us*
|
||||
1.  [Rottentomatoes (https://www.rottentomatoes.com)](https://www.rottentomatoes.com)*: top 1K, movies, us*
|
||||
1.  [Kickstarter (https://www.kickstarter.com)](https://www.kickstarter.com)*: top 1K, finance, us*
|
||||
1.  [Kickstarter (https://www.kickstarter.com)](https://www.kickstarter.com)*: top 1K, cloudflare, finance, us*
|
||||
1.  [forums.ea.com (https://forums.ea.com)](https://forums.ea.com)*: top 1K, forum, gaming, us*, search is disabled
|
||||
1.  [Envato (https://forums.envato.com)](https://forums.envato.com)*: top 1K, au, forum, in*
|
||||
1.  [Ultimate-Guitar (https://ultimate-guitar.com/)](https://ultimate-guitar.com/)*: top 1K, us*
|
||||
@@ -181,17 +181,17 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [Gamespot (https://www.gamespot.com/)](https://www.gamespot.com/)*: top 1K, gaming, us*
|
||||
1.  [note (https://note.com/)](https://note.com/)*: top 1K, jp*
|
||||
1.  [AfreecaTV (http://bjapi.afreecatv.com)](http://bjapi.afreecatv.com)*: top 1K, streaming*
|
||||
1.  [Redbubble (https://www.redbubble.com/)](https://www.redbubble.com/)*: top 1K, shopping, us*
|
||||
1.  [Redbubble (https://www.redbubble.com/)](https://www.redbubble.com/)*: top 1K, cloudflare, shopping, us*
|
||||
1.  [Tom's guide (http://forums.tomsguide.com)](http://forums.tomsguide.com)*: top 1K, forum, tech*
|
||||
1.  [Yumpu (https://www.yumpu.com)](https://www.yumpu.com)*: top 1K, stock*, search is disabled
|
||||
1.  [community.brave.com (https://community.brave.com)](https://community.brave.com)*: top 1K, forum, us*
|
||||
1.  [Tinder (https://tinder.com/)](https://tinder.com/)*: top 1K, dating, us*
|
||||
1.  [CloudflareCommunity (https://community.cloudflare.com/)](https://community.cloudflare.com/)*: top 1K, forum, tech*
|
||||
1.  [CloudflareCommunity (https://community.cloudflare.com/)](https://community.cloudflare.com/)*: top 1K, cloudflare, forum, tech*
|
||||
1.  [Eksisozluk (https://eksisozluk.com)](https://eksisozluk.com)*: top 1K, tr*
|
||||
1.  [AllRecipes (https://www.allrecipes.com/)](https://www.allrecipes.com/)*: top 1K, us*
|
||||
1.  [T-MobileSupport (https://support.t-mobile.com)](https://support.t-mobile.com)*: top 1K, us*, search is disabled
|
||||
1.  [Tinkoff Invest (https://www.tinkoff.ru/invest/)](https://www.tinkoff.ru/invest/)*: top 5K, ru*
|
||||
1.  [Discogs (https://www.discogs.com/)](https://www.discogs.com/)*: top 5K, music, us*
|
||||
1.  [Discogs (https://www.discogs.com/)](https://www.discogs.com/)*: top 5K, cloudflare, music, us*
|
||||
1.  [DiscussPython (https://discuss.python.org/)](https://discuss.python.org/)*: top 5K, coding, forum, us*
|
||||
1.  [Nairaland Forum (https://www.nairaland.com/)](https://www.nairaland.com/)*: top 5K, ng*
|
||||
1.  [Redtube (https://ru.redtube.com/)](https://ru.redtube.com/)*: top 5K, porn, us*
|
||||
@@ -336,11 +336,11 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [Jigsawplanet (https://www.jigsawplanet.com)](https://www.jigsawplanet.com)*: top 5K, fr, us*
|
||||
1.  [hackernoon.com (https://hackernoon.com)](https://hackernoon.com)*: top 5K, news, us*
|
||||
1.  [PCPartPicker (https://pcpartpicker.com)](https://pcpartpicker.com)*: top 5K, us*, search is disabled
|
||||
1.  [AskFM (https://ask.fm/)](https://ask.fm/)*: top 5K, eg, in, ru*, search is disabled
|
||||
1.  [AskFM (https://ask.fm/)](https://ask.fm/)*: top 5K, eg, in, ru*
|
||||
1.  [GitLab (https://gitlab.com/)](https://gitlab.com/)*: top 5K, coding*
|
||||
1.  [DEV Community (https://dev.to/)](https://dev.to/)*: top 5K, coding*
|
||||
1.  [Gumroad (https://www.gumroad.com/)](https://www.gumroad.com/)*: top 5K, us*
|
||||
1.  [Gramho (https://gramho.com/)](https://gramho.com/)*: top 5K, photo*, search is disabled
|
||||
1.  [Gramho (https://gramho.com/)](https://gramho.com/)*: top 5K, photo*
|
||||
1.  [Taplink (https://taplink.cc/)](https://taplink.cc/)*: top 5K, links, ru*, search is disabled
|
||||
1.  [BuyMeACoffee (https://www.buymeacoffee.com/)](https://www.buymeacoffee.com/)*: top 5K, in*
|
||||
1.  [Muckrack (https://muckrack.com)](https://muckrack.com)*: top 5K, us*
|
||||
@@ -750,7 +750,7 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [forum.eksmo.ru (http://forum.eksmo.ru)](http://forum.eksmo.ru)*: top 100K, forum, ru*
|
||||
1.  [Davesgarden (https://davesgarden.com)](https://davesgarden.com)*: top 100K, us*
|
||||
1.  [forum.cxem.net (https://forum.cxem.net/)](https://forum.cxem.net/)*: top 100K, forum, ru*, search is disabled
|
||||
1.  [ICQ (https://icq.com)](https://icq.com)*: top 100K, ch, ru, tr*, search is disabled
|
||||
1.  [ICQ (https://icq.com)](https://icq.com)*: top 100K, ch, ru, tr*
|
||||
1.  [d3 (https://d3.ru/)](https://d3.ru/)*: top 100K, ru*
|
||||
1.  [dwg (https://forum.dwg.ru/)](https://forum.dwg.ru/)*: top 100K, forum, ru*
|
||||
1.  [Fotki (https://fotki.com)](https://fotki.com)*: top 100K, photo*
|
||||
@@ -2161,7 +2161,6 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [Finanzfrage ()]()*: top 100M*
|
||||
1.  [Forum.quake2.com.ru (http://forum.quake2.com.ru/)](http://forum.quake2.com.ru/)*: top 100M, forum, ru*
|
||||
1.  [ForumTauck (https://forums.tauck.com)](https://forums.tauck.com)*: top 100M, forum, us*
|
||||
1.  [Framapiaf (https://framapiaf.org)](https://framapiaf.org)*: top 100M, mastodon*
|
||||
1.  [G2g.com ()]()*: top 100M*
|
||||
1.  [Gam1ng (https://gam1ng.com.br)](https://gam1ng.com.br)*: top 100M, br, webcam*, search is disabled
|
||||
1.  [GeniusArtists ()]()*: top 100M*
|
||||
@@ -2203,7 +2202,6 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [Macqa (https://macqa.ru)](https://macqa.ru)*: top 100M, ru*, search is disabled
|
||||
1.  [Maga-Chat ()]()*: top 100M*
|
||||
1.  [Magabook ()]()*: top 100M*
|
||||
1.  [Mamot (https://mamot.fr)](https://mamot.fr)*: top 100M, mastodon*
|
||||
1.  [Mapify.travel ()]()*: top 100M*
|
||||
1.  [MapMyTracks ()]()*: top 100M*
|
||||
1.  [Marshmallow ()]()*: top 100M*
|
||||
@@ -2226,12 +2224,10 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [Oglaszamy24h ()]()*: top 100M*
|
||||
1.  [Olx.pl ()]()*: top 100M*
|
||||
1.  [Ourfreedombook ()]()*: top 100M*
|
||||
1.  [Outgress (https://outgress.com/)](https://outgress.com/)*: top 100M*
|
||||
1.  [Ow.ly ()]()*: top 100M*
|
||||
1.  [Patronite ()]()*: top 100M*
|
||||
1.  [Pewex.pl ()]()*: top 100M*
|
||||
1.  [Piekielni ()]()*: top 100M*
|
||||
1.  [pixelfed.social (https://pixelfed.social/)](https://pixelfed.social/)*: top 100M, art, pixelfed*
|
||||
1.  [Pol.social ()]()*: top 100M*
|
||||
1.  [Polczat.pl ()]()*: top 100M*
|
||||
1.  [Policja2009 ()]()*: top 100M*
|
||||
@@ -2242,7 +2238,6 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [PalexaRankru (https://palexaRankru.net/)](https://palexaRankru.net/)*: top 100M, forum, ru*, search is disabled
|
||||
1.  [ProgrammersForum (https://www.programmersforum)](https://www.programmersforum)*: top 100M, forum, ru*, search is disabled
|
||||
1.  [Prv.pl ()]()*: top 100M*
|
||||
1.  [programming.dev (https://programming.dev)](https://programming.dev)*: top 100M, lemmy*
|
||||
1.  [Quitter.pl ()]()*: top 100M*
|
||||
1.  [Quizlet ()]()*: top 100M*
|
||||
1.  [Rammclan (http://www.rammclan.ru)](http://www.rammclan.ru)*: top 100M, ru*
|
||||
@@ -3145,18 +3140,17 @@ Rank data fetched from Alexa by domains.
|
||||
1.  [Substack (https://substack.com)](https://substack.com)*: top 100M, blog*
|
||||
1.  [OP.GG [PUBG] (https://pubg.op.gg)](https://pubg.op.gg)*: top 100M, gaming*
|
||||
1.  [OP.GG [Valorant] (https://valorant.op.gg)](https://valorant.op.gg)*: top 100M, gaming*
|
||||
1.  [write.as (https://write.as)](https://write.as)*: top 100M, writefreely*
|
||||
|
||||
The list was updated at (2025-08-10)
|
||||
The list was updated at (2026-03-22)
|
||||
## Statistics
|
||||
|
||||
Enabled/total sites: 2687/3143 = 85.49%
|
||||
Enabled/total sites: 2684/3137 = 85.56%
|
||||
|
||||
Incomplete message checks: 394/2687 = 14.66% (false positive risks)
|
||||
Incomplete message checks: 394/2684 = 14.68% (false positive risks)
|
||||
|
||||
Status code checks: 618/2687 = 23.0% (false positive risks)
|
||||
Status code checks: 615/2684 = 22.91% (false positive risks)
|
||||
|
||||
False positive risk (total): 37.66%
|
||||
False positive risk (total): 37.59%
|
||||
|
||||
Sites with probing: 500px, Aparat, BinarySearch (disabled), BongaCams, BuyMeACoffee, Cent, Disqus, Docker Hub, Duolingo, Gab, GitHub, GitLab, Google Plus (archived), Gravatar, Imgur, Issuu, Keybase, Livejasmin, LocalCryptos (disabled), MixCloud, Niftygateway, Reddit Search (Pushshift) (disabled), SportsTracker, Spotify (disabled), TAP'D, Trello, Twitch, Twitter, Twitter Shadowban (disabled), UnstoppableDomains, Vimeo, Weibo, Yapisal (disabled), YouNow, nightbot, notabug.org, polarsteps, qiwi.me (disabled)
|
||||
|
||||
@@ -3164,17 +3158,17 @@ Sites with activation: Spotify (disabled), Twitter, Vimeo, Weibo
|
||||
|
||||
Top 20 profile URLs:
|
||||
- (796) `{urlMain}/index/8-0-{username} (uCoz)`
|
||||
- (303) `/{username}`
|
||||
- (301) `/{username}`
|
||||
- (221) `{urlMain}{urlSubpath}/members/?username={username} (XenForo)`
|
||||
- (161) `/user/{username}`
|
||||
- (133) `{urlMain}{urlSubpath}/member.php?username={username} (vBulletin)`
|
||||
- (127) `{urlMain}{urlSubpath}/search.php?author={username} (phpBB/Search)`
|
||||
- (118) `/profile/{username}`
|
||||
- (112) `/u/{username}`
|
||||
- (111) `/u/{username}`
|
||||
- (88) `/users/{username}`
|
||||
- (87) `{urlMain}/u/{username}/summary (Discourse)`
|
||||
- (54) `/@{username}`
|
||||
- (54) `/wiki/User:{username}`
|
||||
- (52) `/@{username}`
|
||||
- (41) `/members/?username={username}`
|
||||
- (41) `SUBDOMAIN`
|
||||
- (32) `/members/{username}`
|
||||
@@ -3186,7 +3180,7 @@ Top 20 profile URLs:
|
||||
|
||||
|
||||
Top 20 tags:
|
||||
- (1106) `NO_TAGS` (non-standard)
|
||||
- (1105) `NO_TAGS` (non-standard)
|
||||
- (735) `forum`
|
||||
- (92) `gaming`
|
||||
- (48) `photo`
|
||||
@@ -3198,11 +3192,11 @@ Top 20 tags:
|
||||
- (19) `finance`
|
||||
- (18) `crypto`
|
||||
- (16) `sharing`
|
||||
- (16) `art`
|
||||
- (16) `freelance`
|
||||
- (15) `art`
|
||||
- (15) `shopping`
|
||||
- (13) `sport`
|
||||
- (13) `business`
|
||||
- (13) `cloudflare` (non-standard)
|
||||
- (12) `movies`
|
||||
- (11) `hobby`
|
||||
- (11) `education`
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ description: |
|
||||
|
||||
Currently supported more than 3000 sites, search is launched against 500 popular sites in descending order of popularity by default. Also supported checking of Tor sites, I2P sites, and domains (via DNS resolving).
|
||||
|
||||
version: 0.5.0
|
||||
version: 0.5.0a1
|
||||
license: MIT
|
||||
base: core22
|
||||
confinement: strict
|
||||
|
||||
@@ -45,6 +45,17 @@ DEFAULT_ARGS: Dict[str, Any] = {
|
||||
'web': None,
|
||||
'with_domains': False,
|
||||
'xmind': False,
|
||||
# Mirrors maigret/resources/settings.json (flag --cloudflare-bypass overrides with True)
|
||||
'cloudflare_bypass': {
|
||||
"enabled": True,
|
||||
"modules": [
|
||||
{
|
||||
"name": "chrome_webgate",
|
||||
"method": "url_rewrite",
|
||||
"url": "http://localhost:8000/html?url={url}&retries=1"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -60,6 +71,15 @@ def test_args_search_mode(argparser):
|
||||
assert getattr(args, arg) == want_args[arg]
|
||||
|
||||
|
||||
def test_args_cloudflare_bypass_flag(argparser):
|
||||
args = argparser.parse_args('--cloudflare-bypass username'.split())
|
||||
|
||||
want_args = dict(DEFAULT_ARGS)
|
||||
want_args.update({'username': ['username'], 'cloudflare_bypass': True})
|
||||
|
||||
assert args == Namespace(**want_args)
|
||||
|
||||
|
||||
def test_args_search_mode_several_usernames(argparser):
|
||||
args = argparser.parse_args('username1 username2'.split())
|
||||
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
"""Tests for the close_invalid_telegram_prs utility."""
|
||||
|
||||
import unittest
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add the utils directory to the path
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'utils'))
|
||||
|
||||
from close_invalid_telegram_prs import is_invalid_telegram_pr
|
||||
|
||||
|
||||
class TestCloseInvalidTelegramPRs(unittest.TestCase):
|
||||
"""Test cases for the invalid Telegram PR detection."""
|
||||
|
||||
def test_valid_invalid_telegram_pr_titles(self):
|
||||
"""Test that valid invalid Telegram PR titles are correctly identified."""
|
||||
valid_titles = [
|
||||
"Invalid result https://t.me/someuser",
|
||||
"invalid result https://t.me/channel123",
|
||||
"Invalid Result https://t.me/bot_name",
|
||||
"INVALID RESULT https://t.me/test",
|
||||
"Invalid result https://t.me/user/123",
|
||||
"Invalid result https://t.me/s/channel_name",
|
||||
]
|
||||
|
||||
for title in valid_titles:
|
||||
with self.subTest(title=title):
|
||||
self.assertTrue(is_invalid_telegram_pr(title),
|
||||
f"Title should be identified as invalid: {title}")
|
||||
|
||||
def test_invalid_telegram_pr_titles_not_matching(self):
|
||||
"""Test that non-matching titles are correctly rejected."""
|
||||
invalid_titles = [
|
||||
"Valid result https://t.me/someuser", # "Valid" instead of "Invalid"
|
||||
"Invalid results https://t.me/someuser", # "results" instead of "result"
|
||||
"Invalid result http://t.me/someuser", # "http" instead of "https"
|
||||
"Invalid result https://telegram.me/someuser", # Wrong domain
|
||||
"Fix invalid result https://t.me/someuser", # Extra words before
|
||||
"Invalid result for https://t.me/someuser", # Extra words in between
|
||||
"Added telegram site", # Completely different
|
||||
"Fix false positives", # Unrelated
|
||||
"", # Empty title
|
||||
"Invalid result", # Missing URL
|
||||
"https://t.me/someuser", # Missing "Invalid result"
|
||||
]
|
||||
|
||||
for title in invalid_titles:
|
||||
with self.subTest(title=title):
|
||||
self.assertFalse(is_invalid_telegram_pr(title),
|
||||
f"Title should NOT be identified as invalid: {title}")
|
||||
|
||||
def test_whitespace_handling(self):
|
||||
"""Test that whitespace is handled correctly."""
|
||||
titles_with_whitespace = [
|
||||
" Invalid result https://t.me/someuser ", # Leading/trailing spaces
|
||||
"\tInvalid result https://t.me/someuser\t", # Tabs
|
||||
"Invalid\tresult\thttps://t.me/someuser", # Tabs between words
|
||||
"Invalid result https://t.me/someuser", # Multiple spaces
|
||||
]
|
||||
|
||||
for title in titles_with_whitespace:
|
||||
with self.subTest(title=title):
|
||||
self.assertTrue(is_invalid_telegram_pr(title),
|
||||
f"Title with whitespace should be identified: {title}")
|
||||
|
||||
def test_case_insensitive(self):
|
||||
"""Test that the pattern matching is case insensitive."""
|
||||
case_variations = [
|
||||
"invalid result https://t.me/someuser",
|
||||
"Invalid Result https://t.me/someuser",
|
||||
"INVALID RESULT https://t.me/someuser",
|
||||
"Invalid result https://T.ME/someuser",
|
||||
"iNvAlId ReSuLt https://t.me/someuser",
|
||||
]
|
||||
|
||||
for title in case_variations:
|
||||
with self.subTest(title=title):
|
||||
self.assertTrue(is_invalid_telegram_pr(title),
|
||||
f"Case variation should be identified: {title}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -36,7 +36,7 @@ async def test_asyncio_progressbar_executor():
|
||||
# no guarantees for the results order
|
||||
assert sorted(await executor.run(tasks)) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
|
||||
assert executor.execution_time > 0.2
|
||||
assert executor.execution_time < 0.3
|
||||
assert executor.execution_time < 0.6
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -271,7 +271,7 @@ async def test_dialog_adds_site_negative(settings):
|
||||
]
|
||||
|
||||
with patch('builtins.input', side_effect=user_inputs):
|
||||
result = await submitter.dialog("https://icq.com/sokrat", None)
|
||||
result = await submitter.dialog("https://icq.im/sokrat", None)
|
||||
await submitter.close()
|
||||
|
||||
assert result is False
|
||||
|
||||
@@ -1,205 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Utility script to close pull requests with titles matching "Invalid result https://t.me/..."
|
||||
|
||||
This script identifies and closes PRs that follow the pattern of invalid telegram results,
|
||||
which are typically auto-generated or spam PRs that should not be processed.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import List, Optional
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
print("Error: requests library is required. Install with: pip install requests")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
class GitHubAPI:
|
||||
"""Simple GitHub API wrapper for managing pull requests."""
|
||||
|
||||
def __init__(self, token: str, owner: str, repo: str):
|
||||
self.token = token
|
||||
self.owner = owner
|
||||
self.repo = repo
|
||||
self.base_url = "https://api.github.com"
|
||||
self.headers = {
|
||||
"Authorization": f"token {token}",
|
||||
"Accept": "application/vnd.github.v3+json"
|
||||
}
|
||||
|
||||
def get_open_prs(self) -> List[dict]:
|
||||
"""Get all open pull requests."""
|
||||
url = f"{self.base_url}/repos/{self.owner}/{self.repo}/pulls"
|
||||
params = {"state": "open", "per_page": 100}
|
||||
|
||||
all_prs = []
|
||||
page = 1
|
||||
|
||||
while True:
|
||||
params["page"] = page
|
||||
response = requests.get(url, headers=self.headers, params=params)
|
||||
response.raise_for_status()
|
||||
|
||||
prs = response.json()
|
||||
if not prs:
|
||||
break
|
||||
|
||||
all_prs.extend(prs)
|
||||
page += 1
|
||||
|
||||
return all_prs
|
||||
|
||||
def close_pr(self, pr_number: int, comment: Optional[str] = None) -> bool:
|
||||
"""Close a pull request with an optional comment."""
|
||||
try:
|
||||
# Add comment if provided
|
||||
if comment:
|
||||
comment_url = f"{self.base_url}/repos/{self.owner}/{self.repo}/issues/{pr_number}/comments"
|
||||
comment_data = {"body": comment}
|
||||
response = requests.post(comment_url, headers=self.headers, json=comment_data)
|
||||
response.raise_for_status()
|
||||
|
||||
# Close the PR
|
||||
close_url = f"{self.base_url}/repos/{self.owner}/{self.repo}/pulls/{pr_number}"
|
||||
close_data = {"state": "closed"}
|
||||
response = requests.patch(close_url, headers=self.headers, json=close_data)
|
||||
response.raise_for_status()
|
||||
|
||||
return True
|
||||
except requests.RequestException as e:
|
||||
print(f"Error closing PR #{pr_number}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def is_invalid_telegram_pr(title: str) -> bool:
|
||||
"""
|
||||
Check if a PR title matches the pattern "Invalid result https://t.me/..."
|
||||
|
||||
Args:
|
||||
title: The PR title to check
|
||||
|
||||
Returns:
|
||||
True if the title matches the pattern, False otherwise
|
||||
"""
|
||||
# Pattern: "Invalid result https://t.me/..." (case insensitive)
|
||||
pattern = r"^invalid\s+result\s+https://t\.me/.*"
|
||||
return bool(re.match(pattern, title.strip(), re.IGNORECASE))
|
||||
|
||||
|
||||
def find_invalid_telegram_prs(github_api: GitHubAPI) -> List[dict]:
|
||||
"""
|
||||
Find all open PRs that match the invalid telegram pattern.
|
||||
|
||||
Args:
|
||||
github_api: GitHub API wrapper instance
|
||||
|
||||
Returns:
|
||||
List of PR dictionaries that match the pattern
|
||||
"""
|
||||
all_prs = github_api.get_open_prs()
|
||||
matching_prs = []
|
||||
|
||||
for pr in all_prs:
|
||||
if is_invalid_telegram_pr(pr["title"]):
|
||||
matching_prs.append(pr)
|
||||
|
||||
return matching_prs
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to find and close invalid telegram PRs."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Close pull requests with titles matching 'Invalid result https://t.me/...'"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--token",
|
||||
required=False,
|
||||
help="GitHub personal access token (or set GITHUB_TOKEN env var)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--owner",
|
||||
default="soxoj",
|
||||
help="Repository owner (default: soxoj)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo",
|
||||
default="maigret",
|
||||
help="Repository name (default: maigret)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Show what would be closed without actually closing PRs"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--comment",
|
||||
default="Automatically closing this PR as it appears to be an invalid result for a Telegram URL. "
|
||||
"If this is a legitimate PR, please reopen it with a more descriptive title.",
|
||||
help="Comment to add when closing PRs"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get GitHub token
|
||||
token = args.token or os.getenv("GITHUB_TOKEN")
|
||||
if not token:
|
||||
print("Error: GitHub token is required. Provide via --token or GITHUB_TOKEN env var")
|
||||
sys.exit(1)
|
||||
|
||||
# Initialize GitHub API
|
||||
try:
|
||||
github_api = GitHubAPI(token, args.owner, args.repo)
|
||||
except Exception as e:
|
||||
print(f"Error initializing GitHub API: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# Find matching PRs
|
||||
print(f"Searching for PRs matching pattern in {args.owner}/{args.repo}...")
|
||||
try:
|
||||
matching_prs = find_invalid_telegram_prs(github_api)
|
||||
except Exception as e:
|
||||
print(f"Error fetching PRs: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
if not matching_prs:
|
||||
print("No PRs found matching the pattern 'Invalid result https://t.me/...'")
|
||||
return
|
||||
|
||||
print(f"Found {len(matching_prs)} PR(s) matching the pattern:")
|
||||
|
||||
for pr in matching_prs:
|
||||
print(f" - PR #{pr['number']}: {pr['title']}")
|
||||
print(f" Created by: {pr['user']['login']}")
|
||||
print(f" URL: {pr['html_url']}")
|
||||
print()
|
||||
|
||||
if args.dry_run:
|
||||
print("Dry run mode: No PRs were actually closed.")
|
||||
return
|
||||
|
||||
# Confirm before closing
|
||||
response = input(f"Close {len(matching_prs)} PR(s)? [y/N]: ")
|
||||
if response.lower() != 'y':
|
||||
print("Cancelled.")
|
||||
return
|
||||
|
||||
# Close PRs
|
||||
closed_count = 0
|
||||
for pr in matching_prs:
|
||||
print(f"Closing PR #{pr['number']}: {pr['title']}")
|
||||
if github_api.close_pr(pr['number'], args.comment):
|
||||
closed_count += 1
|
||||
print(f" ✓ Closed successfully")
|
||||
else:
|
||||
print(f" ✗ Failed to close")
|
||||
|
||||
print(f"\nClosed {closed_count} out of {len(matching_prs)} PRs.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user