From c69393727108d4c4c1b21729fede5da62eda99db Mon Sep 17 00:00:00 2001 From: "emdee@spm.plastiras.org" Date: Sat, 24 Feb 2024 07:32:50 +0000 Subject: [PATCH] updates --- Makefile | 16 ++++++++- README.md | 20 +++++------ exclude_badExits-installer.bash | 0 exclude_badExits-pki.bash | 0 exclude_badExits.bash | 12 +++++-- pyproject.toml | 2 +- src/exclude_badExits/__init__.py | 6 ++++ .../{exclude_badExits.py => __main__.py} | 33 +++++++++---------- src/exclude_badExits/exclude_utils.py | 23 ++++++------- src/exclude_badExits/torcontactinfo.py | 2 +- src/exclude_badExits/trustor_poc.py | 10 +++--- 11 files changed, 73 insertions(+), 51 deletions(-) mode change 100644 => 100755 exclude_badExits-installer.bash mode change 100644 => 100755 exclude_badExits-pki.bash mode change 100644 => 100755 exclude_badExits.bash rename src/exclude_badExits/{exclude_badExits.py => __main__.py} (98%) diff --git a/Makefile b/Makefile index d41c6a5..085b88a 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,24 @@ PREFIX=/usr/local PYTHON_EXE_MSYS=${PREFIX}/bin/python3.sh PIP_EXE_MSYS=${PREFIX}/bin/pip3.sh +PYTHON_MINOR=`python3 --version 2>&1 | sed -e 's@^.* @@' -e 's@\.[0-9]*$$@@'` LOCAL_DOCTEST=${PREFIX}/bin/toxcore_run_doctest3.bash DOCTEST=${LOCAL_DOCTEST} MOD=exclude_badExits +all:: README.md exclude_badExits.hlp + +exclude_badExits.hlp:: src/exclude_badExits/__main__.py + PYTHONPATH=${PWD}/src \ + ${PYTHON_EXE_MSYS} -m exclude_badExits --help > exclude_badExits.hlp + +README.md:: src/exclude_badExits/__main__.py + PYTHONPATH=${PWD}/src \ + ${PYTHON_EXE_MSYS} -c 'from exclude_badExits.__main__ import __doc__; print(__doc__)' \ + > README.md + check:: - sh python3.sh -c "import ${MOD}" + sh ${PYTHON_EXE_MSYS} -c "import ${MOD}" lint:: sh .pylint.sh @@ -16,6 +28,8 @@ install:: --no-deps \ --target ${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/ \ --upgrade . + sed -i -e "1s@/usr/bin/python${PYTHON_MINOR}@${PYTHON_EXE_MSYS}@" \ + ${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/bin/* rsync:: bash .rsync.sh diff --git a/README.md b/README.md index ae692d4..2162ff7 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ Use the GoodNodes/Onions list to list onion services you want the Introduction Points whitelisted - these points may change daily Look in tor's notice.log for warnings of 'Every introduction point for service' -```--hs_dir``` ```default='/var/lib/tor'``` will make the program -parse the files named ```hostname``` below this dir to find +```--hs_dir``` ```default='/var/lib/tor'``` will make the program +parse the files named ```hostname``` below this dir to find Hidden Services to whitelist. The Introduction Points can change during the day, so you may want to @@ -86,9 +86,12 @@ Use the GoodNodes/Onions list in goodnodes.yaml to list onion services you want the Introduction Points whitelisted - these points may change daily. Look in tor's notice.log for 'Every introduction point for service' -```notice_log``` will parse the notice log for warnings about relays and +```notice_log``` will parse the notice log for warnings about relays and services that will then be whitelisted. +```--torrc``` will read a file like /etc/tor/torrc and make some +suggestions based on what it finds; it will not edit or change the file. + ```--torrc_output``` will write the torrc ExcludeNodes configuration to a file. ```--good_contacts``` will write the contact info as a ciiss dictionary @@ -129,15 +132,10 @@ To be "good" the ContactInfo must: ```--wellknown_output``` will make the program write the well-known files (```/.well-known/tor-relay/rsa-fingerprint.txt```) to a directory. -```--torrc_output``` will write a file of the commands that it sends to -the Tor controller, so you can include it in a ```/etc/toc/torrc```. - -```--relays_output``` write the download relays in json to a file. The relays +```--relays_output write the download relays in json to a file. The relays are downloaded from https://onionoo.torproject.org/details -For usage, do ```python3 exclude_badExits.py --help``` +For usage, do ```python3 exclude_badExits.py --help` +See [exclude_badExits.txt](./exclude_badExits.txt) -See [exclude_badExits.hlp](./exclude_badExits.hlp) -or there's a doctest file in [exclude_badExits.txt](./exclude_badExits.txt) -Up-to-date code is on https://git.plastiras.org/emdee/exclude_badExits diff --git a/exclude_badExits-installer.bash b/exclude_badExits-installer.bash old mode 100644 new mode 100755 diff --git a/exclude_badExits-pki.bash b/exclude_badExits-pki.bash old mode 100644 new mode 100755 diff --git a/exclude_badExits.bash b/exclude_badExits.bash old mode 100644 new mode 100755 index eeea54d..d2155be --- a/exclude_badExits.bash +++ b/exclude_badExits.bash @@ -6,16 +6,22 @@ SOCKS_PORT=9050 SOCKS_HOST=127.0.0.1 CAFILE=/etc/ssl/certs/ca-certificates.crt # you may have a special python for installed packages -EXE=`which python3.bash` +EXE=`which python3.sh` MOD=exclude_badExits -[ -f exclude_badExits.hlp -a exclude_badExits.hlp -nt src/exclude_badExits/exclude_badExits.py] || \ +if [ ! -f exclude_badExits.hlp ] || \ + [ exclude_badExits.hlp -nt src/exclude_badExits/exclude_badExits.py ] ; then PYTHONPATH=$PWD/src \ $EXE src/${MOD}/exclude_badExits.py --help > exclude_badExits.hlp -[ -f README.md -a README.md -nt src/exclude_badExits/exclude_badExits.py] || \ +fi + +if [ ! -f README.md ] || \ + [ README.md -nt src/exclude_badExits/exclude_badExits.py ] ; then PYTHONPATH=$PWD/src \ $EXE -c 'from exclude_badExits.exclude_badExits import __doc__; print(__doc__)' \ > README.md +fi + # an example of running exclude_badExits with full debugging # expected to 20 minutes or so declare -a LARGS diff --git a/pyproject.toml b/pyproject.toml index b374fea..85841a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ dynamic = ["version", "readme", "dependencies"] # cannot be dynamic ['license'] [project.scripts] -exclude_badExits = "exclude_badExits.exclude_badExits:iMain" +exclude_badExits = "exclude_badExits.__main__:iMain" [tool.setuptools.dynamic] version = {attr = "exclude_badExits.__version__"} diff --git a/src/exclude_badExits/__init__.py b/src/exclude_badExits/__init__.py index 2bcf11b..ad9c9f6 100644 --- a/src/exclude_badExits/__init__.py +++ b/src/exclude_badExits/__init__.py @@ -1,2 +1,8 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- __version__ = "1.0.0" + +__all__ = [ + 'exclude_utils', + 'torcontactinfo', + 'trustor_poc'] diff --git a/src/exclude_badExits/exclude_badExits.py b/src/exclude_badExits/__main__.py similarity index 98% rename from src/exclude_badExits/exclude_badExits.py rename to src/exclude_badExits/__main__.py index 7c21a28..e39b839 100755 --- a/src/exclude_badExits/exclude_badExits.py +++ b/src/exclude_badExits/__main__.py @@ -197,30 +197,29 @@ from toxygen_wrapper.tests.support_onions import ( bAreWeConnected, icheck_torrc, lIntroductionPoints, yKNOWN_NODNS, zResolveDomain) -from trustor_poc import TrustorError, idns_validate -try: - import xxxhttpx +from exclude_badExits.trustor_poc import TrustorError, idns_validate +if False: + import httpx import asyncio - from trustor_poc import oDownloadUrlHttpx -except: + from exclude_badExits.trustor_poc import oDownloadUrlHttpx +else: httpx = None - from trustor_poc import oDownloadUrlUrllib3Socks as oDownloadUrl + from exclude_badExits.trustor_poc import oDownloadUrlUrllib3Socks as oDownloadUrl -try: - from torcontactinfo import TorContactInfoParser - oPARSER = TorContactInfoParser() -except ImportError: - oPARSER = None +from exclude_badExits.torcontactinfo import TorContactInfoParser -from exclude_utils import (aCleanContact, sCleanEmail, aParseContact, - oStemController, oMainArgparser, - vwrite_goodnodes, vwrite_badnodes, vwrite_good_contacts, - vwritefinale, vsetup_logging) +from exclude_badExits.exclude_utils import ( + aCleanContact, sCleanEmail, aParseContact, + oStemController, oMainArgparser, + vwrite_goodnodes, vwrite_badnodes, vwrite_good_contacts, + vwritefinale, vsetup_logging ) warnings.filterwarnings('ignore') global LOG LOG = logging.getLogger() +oPARSER = TorContactInfoParser() + aGOOD_CONTACTS_DB = {} aGOOD_CONTACTS_FPS = {} aBAD_CONTACTS_DB = {} @@ -555,10 +554,10 @@ def aContactFps(oargs, a, fp, o, domain): if oargs.wellknown_output: sdir = os.path.join(oargs.wellknown_output, domain, '.well-known', 'tor-relay') + sfile = os.path.join(sdir, "rsa-fingerprint.txt") try: if not os.path.isdir(sdir): os.makedirs(sdir) - sfile = os.path.join(sdir, "rsa-fingerprint.txt") with open(sfile, 'wt') as oFd: oFd.write(data) except Exception as e: @@ -665,6 +664,7 @@ def bProcessContact(b, texclude_set, aBadContacts, iFakeContact=0): return True +lNOT_IN_RELAYS_DB = [] def bCheckFp(relay, sofar, lConds, texclude_set): global aGOOD_CONTACTS_DB global aGOOD_CONTACTS_FPS @@ -853,7 +853,6 @@ def iMain(lArgs): iFakeContact = 0 iTotalContacts = 0 aBadContacts = {} - lNOT_IN_RELAYS_DB = [] iR = 0 relays = controller.get_server_descriptors() lqueue = [] diff --git a/src/exclude_badExits/exclude_utils.py b/src/exclude_badExits/exclude_utils.py index 7301759..0968274 100644 --- a/src/exclude_badExits/exclude_utils.py +++ b/src/exclude_badExits/exclude_utils.py @@ -11,7 +11,7 @@ import logging import warnings global LOG -from toxygen_wrapper.tests.support_onions import (woGetStemController, +from toxygen_wrapper.tests.support_onions import (oGetStemController, vwait_for_controller,) try: @@ -20,8 +20,6 @@ try: yaml.indent(mapping=2, sequence=2) safe_load = yaml.load except: - yaml = None -if yaml is None: try: import yaml safe_load = yaml.safe_load @@ -36,6 +34,15 @@ try: except ImportError: coloredlogs = False +lMORONS = ['hoster:Quintex Alliance Consulting '] +oCONTACT_RE = re.compile(r'([^:]*)(\s+)(email|url|proof|ciissversion|abuse|gpg):') +lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory'] +lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone', + 'sandbox', 'offlinemasterkey'] +lEMAILS = ['abuse', 'email'] + +ETC_DIR = '/usr/local/etc/tor/yaml' + def aCleanContact(a, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> dict: # cleanups for elt in lINTS: @@ -78,15 +85,6 @@ def sCleanEmail(s, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> str: s = s.replace(elt, '?') return s -lMORONS = ['hoster:Quintex Alliance Consulting '] -oCONTACT_RE = re.compile(r'([^:]*)(\s+)(email|url|proof|ciissversion|abuse|gpg):') -lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory'] -lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone', - 'sandbox', 'offlinemasterkey'] -lEMAILS = ['abuse', 'email'] - -ETC_DIR = '/usr/local/etc/tor/yaml' - def oStemController(oargs, sEXCLUDE_EXIT_GROUP): if os.path.exists(oargs.proxy_ctl): controller = oGetStemController(log_level=oargs.log_level, @@ -261,7 +259,6 @@ def vsetup_logging(theLOG, log_level, logfile='', stream=sys.stdout) -> None: kwargs = dict(level=log_level, force=True, format='%(levelname)s %(message)s') - logging.basicConfig(**kwargs) if add and logfile: oHandler = logging.StreamHandler(stream) diff --git a/src/exclude_badExits/torcontactinfo.py b/src/exclude_badExits/torcontactinfo.py index 169bfe8..ec67339 100644 --- a/src/exclude_badExits/torcontactinfo.py +++ b/src/exclude_badExits/torcontactinfo.py @@ -29,7 +29,7 @@ import textwrap # HAS_RICH = True if True: def rprint(value='', *args, **kwargs): - if value not in [None, False, True] and + if value not in [None, False, True] and \ isinstance(value, (dict, list, set, tuple)): value = json.dumps(value, indent=4) return LOG.debug(value, *args, **kwargs) diff --git a/src/exclude_badExits/trustor_poc.py b/src/exclude_badExits/trustor_poc.py index d78388f..ac8648e 100644 --- a/src/exclude_badExits/trustor_poc.py +++ b/src/exclude_badExits/trustor_poc.py @@ -9,7 +9,7 @@ import re import sys import ipaddress import warnings - +import logging import urllib3.util from urllib3.util import parse_url as urlparse @@ -24,7 +24,6 @@ except: ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None global LOG -import logging warnings.filterwarnings('ignore') LOG = logging.getLogger() @@ -36,7 +35,7 @@ logging.getLogger("urllib3").setLevel(logging.INFO) # https://github.com/erans/torcontactinfoparser # sys.path.append('/home/....') try: - from torcontactinfo import TorContactInfoParser + from exclude_badExits.torcontactinfo import TorContactInfoParser except: TorContactInfoParser = None @@ -266,7 +265,10 @@ def oDownloadUrlRequests(uri, sCAfile: str, timeout: int = 30, host: str = '127. return oReqResp # There's no point in using asyncio because of duplicate urls in the tasks -async def oDownloadUrlHttpx(uri:str, sCAfile:str, timeout;int = 30, host:str = '127.0.0.1', port:int = 9050, content_type:str = 'text/plain'): +async def oDownloadUrlHttpx(uri:str, sCAfile:str, timeout:int = 30, + host:str = '127.0.0.1', port:int = 9050, + content_type:str = 'text/plain'): + import httpcore import asyncio import httpx