From 36e0102dcda4a9fedfc97ffb5249b3726d0fee16 Mon Sep 17 00:00:00 2001 From: emdee Date: Wed, 16 Nov 2022 21:00:16 +0000 Subject: [PATCH] add exclude_badExits.py --- README.md | 76 +++++++++++++++++++++++++++++++++++++++++++++ exclude_badExits.py | 52 +++++++++++++++++++------------ support_onions.py | 20 +++++++----- trustor_poc.py | 2 +- 4 files changed, 123 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 5d92ee8..8be6c22 100644 --- a/README.md +++ b/README.md @@ -71,3 +71,79 @@ You can expect it to take an hour or two the first time this is run: >700 domains. For usage, do ```python3 exclude_badExits.py --help` + +## Usage +``` +usage: exclude_badExits.py [-h] [--https_cafile HTTPS_CAFILE] + [--proxy_host PROXY_HOST] [--proxy_port PROXY_PORT] + [--proxy_ctl PROXY_CTL] [--torrc TORRC] + [--timeout TIMEOUT] [--good_nodes GOOD_NODES] + [--bad_nodes BAD_NODES] [--contact CONTACT] + [--bad_contacts BAD_CONTACTS] + [--strict_nodes {0,1}] [--wait_boot WAIT_BOOT] + [--points_timeout POINTS_TIMEOUT] + [--log_level LOG_LEVEL] + [--bad_sections BAD_SECTIONS] + [--white_services WHITE_SERVICES] + [--torrc_output TORRC_OUTPUT] + [--proof_output PROOF_OUTPUT] +``` + +### Optional arguments: + +``` + -h, --help show this help message and exit + --https_cafile HTTPS_CAFILE + Certificate Authority file (in PEM) +``` +``` + --proxy_host PROXY_HOST, --proxy-host PROXY_HOST + proxy host + --proxy_port PROXY_PORT, --proxy-port PROXY_PORT + proxy control port + --proxy_ctl PROXY_CTL, --proxy-ctl PROXY_CTL + control socket - or port +``` +``` + --torrc TORRC torrc to check for suggestions + --timeout TIMEOUT proxy download connect timeout +``` +``` + --good_nodes GOOD_NODES + Yaml file of good info that should not be excluded + --bad_nodes BAD_NODES + Yaml file of bad nodes that should also be excluded +``` +``` + --contact CONTACT comma sep list of conditions - Empty,NoEmail + --bad_contacts BAD_CONTACTS + Yaml file of bad contacts that bad FPs are using +``` +``` + --strict_nodes {0,1} Set StrictNodes: 1 is less anonymous but more secure, + although some sites may be unreachable + --wait_boot WAIT_BOOT + Seconds to wait for Tor to booststrap + --points_timeout POINTS_TIMEOUT + Timeout for getting introduction points - must be long + >120sec. 0 means disabled looking for IPs +``` +``` + --log_level LOG_LEVEL + 10=debug 20=info 30=warn 40=error + --bad_sections BAD_SECTIONS + sections of the badnodes.yaml to use, comma separated, + '' BROKEN +``` +``` + --white_services WHITE_SERVICES + comma sep. list of onions to whitelist their + introduction points - BROKEN +``` +``` + --torrc_output TORRC_OUTPUT + Write the torrc configuration to a file + --proof_output PROOF_OUTPUT + Write the proof data of the included nodes to a YAML + file +``` diff --git a/exclude_badExits.py b/exclude_badExits.py index 00e7a01..50b9ecf 100644 --- a/exclude_badExits.py +++ b/exclude_badExits.py @@ -48,7 +48,7 @@ exclusion: the ```--contact``` commandline arg is a comma sep list of conditions More may be added later. Because you don't want to exclude the introduction points to any onion -you want to connect to, ```--white_services``` should whitelist the +you want to connect to, ```--white_onions``` should whitelist the introduction points to a comma sep list of onions, but is currently broken in stem 1.8.0: see: * https://github.com/torproject/stem/issues/96 @@ -88,7 +88,7 @@ import time import argparse import string from io import StringIO -import ipaddr +import ipaddress # list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12')) from urllib3.util.ssl_match_hostname import CertificateError @@ -98,15 +98,23 @@ from stem.control import Controller from stem.connection import IncorrectPassword from stem.util.tor_tools import is_valid_fingerprint try: - import yaml + from ruamel.yaml import YAML + yaml = YAML() + safe_load = yaml.load except: yaml = None +if yaml is None: + try: + import yaml + safe_load = yaml.safe_load + except: + yaml = None + try: from unbound import ub_ctx,RR_TYPE_TXT,RR_CLASS_IN except: ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None - global LOG import logging import warnings @@ -158,7 +166,7 @@ def lYamlBadNodes(sFile, if not yaml: return l if os.path.exists(sFile): with open(sFile, 'rt') as oFd: - oBAD_NODES = yaml.safe_load(oFd) + oBAD_NODES = safe_load(oFd) # BROKEN # root = 'ExcludeNodes' @@ -170,7 +178,7 @@ def lYamlBadNodes(sFile, root = 'ExcludeDomains' if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]: - lMAYBE_NODNS = yaml.safe_load(StringIO(yKNOWN_NODNS)) + lMAYBE_NODNS = safe_load(StringIO(yKNOWN_NODNS)) else: lMAYBE_NODNS = oBAD_NODES[oBAD_ROOT][root] return l @@ -184,7 +192,7 @@ def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'): if not yaml: return l if os.path.exists(sFile): with open(sFile, 'rt') as oFd: - o = yaml.safe_load(oFd) + o = safe_load(oFd) oGOOD_NODES = o if 'GuardNodes' in o[oGOOD_ROOT].keys(): l = o[oGOOD_ROOT]['GuardNodes'] @@ -271,7 +279,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050) LOG.warn(f"{domain} is bad from {a['url']}") LOG.debug(f"{fp} is bad from {a}") return a - + ip = zResolveDomain(domain) if ip == '': aFP_EMAIL[fp] = a['email'] @@ -373,7 +381,7 @@ def aParseContact(contact, fp): s += '\n'.join([f" {line}\"".replace(':',': \"', 1) for line in l]) oFd = StringIO(s) - a = yaml.safe_load(oFd) + a = safe_load(oFd) return a def oMainArgparser(_=None): @@ -414,10 +422,10 @@ def oMainArgparser(_=None): help='proxy download connect timeout') parser.add_argument('--good_nodes', type=str, - default=os.path.join(ETC_DIR, 'torrc-goodnodes.yaml'), + default=os.path.join(ETC_DIR, 'goodnodes.yaml'), help="Yaml file of good info that should not be excluded") parser.add_argument('--bad_nodes', type=str, - default=os.path.join(ETC_DIR, 'torrc-badnodes.yaml'), + default=os.path.join(ETC_DIR, 'badnodes.yaml'), help="Yaml file of bad nodes that should also be excluded") parser.add_argument('--contact', type=str, default='Empty,NoEmail', help="comma sep list of conditions - Empty,NoEmail") @@ -437,13 +445,13 @@ def oMainArgparser(_=None): parser.add_argument('--bad_sections', type=str, default='MyBadExit', help="sections of the badnodes.yaml to use, comma separated, '' BROKEN") - parser.add_argument('--white_services', type=str, + parser.add_argument('--white_onions', type=str, default='', help="comma sep. list of onions to whitelist their introduction points - BROKEN") parser.add_argument('--torrc_output', type=str, default=os.path.join(ETC_DIR, 'torrc.new'), help="Write the torrc configuration to a file") - parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'proof.yaml'), + parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'), help="Write the proof data of the included nodes to a YAML file") return parser @@ -493,8 +501,7 @@ def iMain(lArgs): if sFile and os.path.exists(sFile): try: with open(sFile, 'rt') as oFd: - aTRUST_DB = yaml.safe_load(oFd) - assert type(aTRUST_DB) == dict + aTRUST_DB = safe_load(oFd) LOG.info(f"{len(aTRUST_DB.keys())} trusted contacts from {sFile}") # reverse lookup of fps to contacts # but... @@ -541,10 +548,17 @@ def iMain(lArgs): t = set() if 'IntroductionPoints' in oGOOD_NODES[oGOOD_ROOT]['Relays'].keys(): t = set(oGOOD_NODES[oGOOD_ROOT]['Relays']['IntroductionPoints']) - # not working = maybe when stem is updated - w = set(oGOOD_NODES[oGOOD_ROOT]['Services']) - if oArgs.white_services: - w.update(oArgs.white_services.split(',')) + w = set() + if 'Services' in oGOOD_NODES[oGOOD_ROOT].keys(): + # 'Onions' can I use the IntroductionPoints for Services too? + # w = set(oGOOD_NODES[oGOOD_ROOT]['Services']) + pass + if 'Onions' in oGOOD_NODES[oGOOD_ROOT].keys(): + # Provides the descriptor for a hidden service. The **address** is the + # '.onion' address of the hidden service + w = set(oGOOD_NODES[oGOOD_ROOT]['Onions']) + if oArgs.white_onions: + w.update(oArgs.white_onions.split(',')) if oArgs.points_timeout > 0: LOG.info(f"{len(w)} services will be checked from IntroductionPoints") t.update(lIntroductionPoints(controller, w, itimeout=oArgs.points_timeout)) diff --git a/support_onions.py b/support_onions.py index 68d3ee0..d42feba 100644 --- a/support_onions.py +++ b/support_onions.py @@ -14,10 +14,12 @@ if False: import cepa as stem from cepa.control import Controller from cepa.connection import MissingPassword + from cepa.util.tor_tools import is_valid_fingerprint else: import stem from stem.control import Controller from stem.connection import MissingPassword + from stem.util.tor_tools import is_valid_fingerprint global LOG import logging @@ -96,7 +98,7 @@ def oGetStemController(log_level=10, sock_or_pair='/run/tor/control'): controller.authenticate(p) oSTEM_CONTROLER = controller LOG.debug(f"{controller}") - return oSTEM_CONTROLER + return oSTEM_CONTROLER def bAreWeConnected(): # FixMe: Linux only @@ -144,7 +146,11 @@ def bin_to_hex(raw_id, length=None): return res.upper() def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10): - """now working !!! stem 1.8.x timeout must be huge >120""" + """now working !!! stem 1.8.x timeout must be huge >120 + 'Provides the descriptor for a hidden service. The **address** is the + '.onion' address of the hidden service ' + What about Services? + """ try: from cryptography.utils import int_from_bytes except ImportError: @@ -154,12 +160,12 @@ def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10) # this will fai if the trick above didnt work from stem.prereq import is_crypto_available is_crypto_available(ed25519 = True) - + from stem.descriptor.hidden_service import HiddenServiceDescriptorV3 from stem.client.datatype import LinkByFingerprint from stem import Timeout from queue import Empty - + if type(lOnions) not in [set, tuple, list]: lOnions = list(lOnions) if controller is None: @@ -277,7 +283,7 @@ def sTorResolve(target, LOG.info(sLabel +f"{i} on {sHost}:{iPort}" ) sock.close() raise SystemExit(5) - + assert len(data) >= 8 packet_sf = data[1] if packet_sf == 90: @@ -292,7 +298,7 @@ def sTorResolve(target, # os.system("strace tor-resolve -4 "+target+" 2>&1|grep '^sen\|^rec'") return '' - + def getaddrinfo(sHost, sPort): # do this the explicit way = Ive seen the compact connect fail # >>> sHost, sPort = 'l27.0.0.1', 33446 @@ -393,4 +399,4 @@ def lExitExcluder(oArgs, iPort=9051, log_level=10): if __name__ == '__main__': target = 'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad' controller = oGetStemController(log_level=10) - lIntroductionPoints(controller, [target], itimeout=120) + lIntroductionPoints(controller, [target], itimeout=120) diff --git a/trustor_poc.py b/trustor_poc.py index 0f9406a..4ddc6c1 100644 --- a/trustor_poc.py +++ b/trustor_poc.py @@ -355,7 +355,7 @@ def _my_match_hostname(cert, asserted_hostname): try: my_match_hostname(cert, asserted_hostname) except CertificateError as e: - log.warning( + LOG.warning( "Certificate did not match hostname: %s. Certificate: %s", asserted_hostname, cert,