add exclude_badExits.py

This commit is contained in:
emdee 2022-11-16 21:00:16 +00:00
parent 6cf32758c1
commit 36e0102dcd
4 changed files with 123 additions and 27 deletions

View File

@ -71,3 +71,79 @@ You can expect it to take an hour or two the first time this is run:
>700 domains. >700 domains.
For usage, do ```python3 exclude_badExits.py --help` For usage, do ```python3 exclude_badExits.py --help`
## Usage
```
usage: exclude_badExits.py [-h] [--https_cafile HTTPS_CAFILE]
[--proxy_host PROXY_HOST] [--proxy_port PROXY_PORT]
[--proxy_ctl PROXY_CTL] [--torrc TORRC]
[--timeout TIMEOUT] [--good_nodes GOOD_NODES]
[--bad_nodes BAD_NODES] [--contact CONTACT]
[--bad_contacts BAD_CONTACTS]
[--strict_nodes {0,1}] [--wait_boot WAIT_BOOT]
[--points_timeout POINTS_TIMEOUT]
[--log_level LOG_LEVEL]
[--bad_sections BAD_SECTIONS]
[--white_services WHITE_SERVICES]
[--torrc_output TORRC_OUTPUT]
[--proof_output PROOF_OUTPUT]
```
### Optional arguments:
```
-h, --help show this help message and exit
--https_cafile HTTPS_CAFILE
Certificate Authority file (in PEM)
```
```
--proxy_host PROXY_HOST, --proxy-host PROXY_HOST
proxy host
--proxy_port PROXY_PORT, --proxy-port PROXY_PORT
proxy control port
--proxy_ctl PROXY_CTL, --proxy-ctl PROXY_CTL
control socket - or port
```
```
--torrc TORRC torrc to check for suggestions
--timeout TIMEOUT proxy download connect timeout
```
```
--good_nodes GOOD_NODES
Yaml file of good info that should not be excluded
--bad_nodes BAD_NODES
Yaml file of bad nodes that should also be excluded
```
```
--contact CONTACT comma sep list of conditions - Empty,NoEmail
--bad_contacts BAD_CONTACTS
Yaml file of bad contacts that bad FPs are using
```
```
--strict_nodes {0,1} Set StrictNodes: 1 is less anonymous but more secure,
although some sites may be unreachable
--wait_boot WAIT_BOOT
Seconds to wait for Tor to booststrap
--points_timeout POINTS_TIMEOUT
Timeout for getting introduction points - must be long
>120sec. 0 means disabled looking for IPs
```
```
--log_level LOG_LEVEL
10=debug 20=info 30=warn 40=error
--bad_sections BAD_SECTIONS
sections of the badnodes.yaml to use, comma separated,
'' BROKEN
```
```
--white_services WHITE_SERVICES
comma sep. list of onions to whitelist their
introduction points - BROKEN
```
```
--torrc_output TORRC_OUTPUT
Write the torrc configuration to a file
--proof_output PROOF_OUTPUT
Write the proof data of the included nodes to a YAML
file
```

View File

@ -48,7 +48,7 @@ exclusion: the ```--contact``` commandline arg is a comma sep list of conditions
More may be added later. More may be added later.
Because you don't want to exclude the introduction points to any onion Because you don't want to exclude the introduction points to any onion
you want to connect to, ```--white_services``` should whitelist the you want to connect to, ```--white_onions``` should whitelist the
introduction points to a comma sep list of onions, but is introduction points to a comma sep list of onions, but is
currently broken in stem 1.8.0: see: currently broken in stem 1.8.0: see:
* https://github.com/torproject/stem/issues/96 * https://github.com/torproject/stem/issues/96
@ -88,7 +88,7 @@ import time
import argparse import argparse
import string import string
from io import StringIO from io import StringIO
import ipaddr import ipaddress
# list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12')) # list(ipaddress._find_address_range(ipaddress.IPv4Network('172.16.0.0/12'))
from urllib3.util.ssl_match_hostname import CertificateError from urllib3.util.ssl_match_hostname import CertificateError
@ -98,15 +98,23 @@ from stem.control import Controller
from stem.connection import IncorrectPassword from stem.connection import IncorrectPassword
from stem.util.tor_tools import is_valid_fingerprint from stem.util.tor_tools import is_valid_fingerprint
try: try:
import yaml from ruamel.yaml import YAML
yaml = YAML()
safe_load = yaml.load
except: except:
yaml = None yaml = None
if yaml is None:
try:
import yaml
safe_load = yaml.safe_load
except:
yaml = None
try: try:
from unbound import ub_ctx,RR_TYPE_TXT,RR_CLASS_IN from unbound import ub_ctx,RR_TYPE_TXT,RR_CLASS_IN
except: except:
ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
global LOG global LOG
import logging import logging
import warnings import warnings
@ -158,7 +166,7 @@ def lYamlBadNodes(sFile,
if not yaml: return l if not yaml: return l
if os.path.exists(sFile): if os.path.exists(sFile):
with open(sFile, 'rt') as oFd: with open(sFile, 'rt') as oFd:
oBAD_NODES = yaml.safe_load(oFd) oBAD_NODES = safe_load(oFd)
# BROKEN # BROKEN
# root = 'ExcludeNodes' # root = 'ExcludeNodes'
@ -170,7 +178,7 @@ def lYamlBadNodes(sFile,
root = 'ExcludeDomains' root = 'ExcludeDomains'
if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]: if root not in oBAD_NODES[oBAD_ROOT] or not oBAD_NODES[oBAD_ROOT][root]:
lMAYBE_NODNS = yaml.safe_load(StringIO(yKNOWN_NODNS)) lMAYBE_NODNS = safe_load(StringIO(yKNOWN_NODNS))
else: else:
lMAYBE_NODNS = oBAD_NODES[oBAD_ROOT][root] lMAYBE_NODNS = oBAD_NODES[oBAD_ROOT][root]
return l return l
@ -184,7 +192,7 @@ def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
if not yaml: return l if not yaml: return l
if os.path.exists(sFile): if os.path.exists(sFile):
with open(sFile, 'rt') as oFd: with open(sFile, 'rt') as oFd:
o = yaml.safe_load(oFd) o = safe_load(oFd)
oGOOD_NODES = o oGOOD_NODES = o
if 'GuardNodes' in o[oGOOD_ROOT].keys(): if 'GuardNodes' in o[oGOOD_ROOT].keys():
l = o[oGOOD_ROOT]['GuardNodes'] l = o[oGOOD_ROOT]['GuardNodes']
@ -271,7 +279,7 @@ def aVerifyContact(a, fp, https_cafile, timeout=20, host='127.0.0.1', port=9050)
LOG.warn(f"{domain} is bad from {a['url']}") LOG.warn(f"{domain} is bad from {a['url']}")
LOG.debug(f"{fp} is bad from {a}") LOG.debug(f"{fp} is bad from {a}")
return a return a
ip = zResolveDomain(domain) ip = zResolveDomain(domain)
if ip == '': if ip == '':
aFP_EMAIL[fp] = a['email'] aFP_EMAIL[fp] = a['email']
@ -373,7 +381,7 @@ def aParseContact(contact, fp):
s += '\n'.join([f" {line}\"".replace(':',': \"', 1) s += '\n'.join([f" {line}\"".replace(':',': \"', 1)
for line in l]) for line in l])
oFd = StringIO(s) oFd = StringIO(s)
a = yaml.safe_load(oFd) a = safe_load(oFd)
return a return a
def oMainArgparser(_=None): def oMainArgparser(_=None):
@ -414,10 +422,10 @@ def oMainArgparser(_=None):
help='proxy download connect timeout') help='proxy download connect timeout')
parser.add_argument('--good_nodes', type=str, parser.add_argument('--good_nodes', type=str,
default=os.path.join(ETC_DIR, 'torrc-goodnodes.yaml'), default=os.path.join(ETC_DIR, 'goodnodes.yaml'),
help="Yaml file of good info that should not be excluded") help="Yaml file of good info that should not be excluded")
parser.add_argument('--bad_nodes', type=str, parser.add_argument('--bad_nodes', type=str,
default=os.path.join(ETC_DIR, 'torrc-badnodes.yaml'), default=os.path.join(ETC_DIR, 'badnodes.yaml'),
help="Yaml file of bad nodes that should also be excluded") help="Yaml file of bad nodes that should also be excluded")
parser.add_argument('--contact', type=str, default='Empty,NoEmail', parser.add_argument('--contact', type=str, default='Empty,NoEmail',
help="comma sep list of conditions - Empty,NoEmail") help="comma sep list of conditions - Empty,NoEmail")
@ -437,13 +445,13 @@ def oMainArgparser(_=None):
parser.add_argument('--bad_sections', type=str, parser.add_argument('--bad_sections', type=str,
default='MyBadExit', default='MyBadExit',
help="sections of the badnodes.yaml to use, comma separated, '' BROKEN") help="sections of the badnodes.yaml to use, comma separated, '' BROKEN")
parser.add_argument('--white_services', type=str, parser.add_argument('--white_onions', type=str,
default='', default='',
help="comma sep. list of onions to whitelist their introduction points - BROKEN") help="comma sep. list of onions to whitelist their introduction points - BROKEN")
parser.add_argument('--torrc_output', type=str, parser.add_argument('--torrc_output', type=str,
default=os.path.join(ETC_DIR, 'torrc.new'), default=os.path.join(ETC_DIR, 'torrc.new'),
help="Write the torrc configuration to a file") help="Write the torrc configuration to a file")
parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'proof.yaml'), parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'),
help="Write the proof data of the included nodes to a YAML file") help="Write the proof data of the included nodes to a YAML file")
return parser return parser
@ -493,8 +501,7 @@ def iMain(lArgs):
if sFile and os.path.exists(sFile): if sFile and os.path.exists(sFile):
try: try:
with open(sFile, 'rt') as oFd: with open(sFile, 'rt') as oFd:
aTRUST_DB = yaml.safe_load(oFd) aTRUST_DB = safe_load(oFd)
assert type(aTRUST_DB) == dict
LOG.info(f"{len(aTRUST_DB.keys())} trusted contacts from {sFile}") LOG.info(f"{len(aTRUST_DB.keys())} trusted contacts from {sFile}")
# reverse lookup of fps to contacts # reverse lookup of fps to contacts
# but... # but...
@ -541,10 +548,17 @@ def iMain(lArgs):
t = set() t = set()
if 'IntroductionPoints' in oGOOD_NODES[oGOOD_ROOT]['Relays'].keys(): if 'IntroductionPoints' in oGOOD_NODES[oGOOD_ROOT]['Relays'].keys():
t = set(oGOOD_NODES[oGOOD_ROOT]['Relays']['IntroductionPoints']) t = set(oGOOD_NODES[oGOOD_ROOT]['Relays']['IntroductionPoints'])
# not working = maybe when stem is updated w = set()
w = set(oGOOD_NODES[oGOOD_ROOT]['Services']) if 'Services' in oGOOD_NODES[oGOOD_ROOT].keys():
if oArgs.white_services: # 'Onions' can I use the IntroductionPoints for Services too?
w.update(oArgs.white_services.split(',')) # w = set(oGOOD_NODES[oGOOD_ROOT]['Services'])
pass
if 'Onions' in oGOOD_NODES[oGOOD_ROOT].keys():
# Provides the descriptor for a hidden service. The **address** is the
# '.onion' address of the hidden service
w = set(oGOOD_NODES[oGOOD_ROOT]['Onions'])
if oArgs.white_onions:
w.update(oArgs.white_onions.split(','))
if oArgs.points_timeout > 0: if oArgs.points_timeout > 0:
LOG.info(f"{len(w)} services will be checked from IntroductionPoints") LOG.info(f"{len(w)} services will be checked from IntroductionPoints")
t.update(lIntroductionPoints(controller, w, itimeout=oArgs.points_timeout)) t.update(lIntroductionPoints(controller, w, itimeout=oArgs.points_timeout))

View File

@ -14,10 +14,12 @@ if False:
import cepa as stem import cepa as stem
from cepa.control import Controller from cepa.control import Controller
from cepa.connection import MissingPassword from cepa.connection import MissingPassword
from cepa.util.tor_tools import is_valid_fingerprint
else: else:
import stem import stem
from stem.control import Controller from stem.control import Controller
from stem.connection import MissingPassword from stem.connection import MissingPassword
from stem.util.tor_tools import is_valid_fingerprint
global LOG global LOG
import logging import logging
@ -96,7 +98,7 @@ def oGetStemController(log_level=10, sock_or_pair='/run/tor/control'):
controller.authenticate(p) controller.authenticate(p)
oSTEM_CONTROLER = controller oSTEM_CONTROLER = controller
LOG.debug(f"{controller}") LOG.debug(f"{controller}")
return oSTEM_CONTROLER return oSTEM_CONTROLER
def bAreWeConnected(): def bAreWeConnected():
# FixMe: Linux only # FixMe: Linux only
@ -144,7 +146,11 @@ def bin_to_hex(raw_id, length=None):
return res.upper() return res.upper()
def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10): def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10):
"""now working !!! stem 1.8.x timeout must be huge >120""" """now working !!! stem 1.8.x timeout must be huge >120
'Provides the descriptor for a hidden service. The **address** is the
'.onion' address of the hidden service '
What about Services?
"""
try: try:
from cryptography.utils import int_from_bytes from cryptography.utils import int_from_bytes
except ImportError: except ImportError:
@ -154,12 +160,12 @@ def lIntroductionPoints(controller=None, lOnions=[], itimeout=120, log_level=10)
# this will fai if the trick above didnt work # this will fai if the trick above didnt work
from stem.prereq import is_crypto_available from stem.prereq import is_crypto_available
is_crypto_available(ed25519 = True) is_crypto_available(ed25519 = True)
from stem.descriptor.hidden_service import HiddenServiceDescriptorV3 from stem.descriptor.hidden_service import HiddenServiceDescriptorV3
from stem.client.datatype import LinkByFingerprint from stem.client.datatype import LinkByFingerprint
from stem import Timeout from stem import Timeout
from queue import Empty from queue import Empty
if type(lOnions) not in [set, tuple, list]: if type(lOnions) not in [set, tuple, list]:
lOnions = list(lOnions) lOnions = list(lOnions)
if controller is None: if controller is None:
@ -277,7 +283,7 @@ def sTorResolve(target,
LOG.info(sLabel +f"{i} on {sHost}:{iPort}" ) LOG.info(sLabel +f"{i} on {sHost}:{iPort}" )
sock.close() sock.close()
raise SystemExit(5) raise SystemExit(5)
assert len(data) >= 8 assert len(data) >= 8
packet_sf = data[1] packet_sf = data[1]
if packet_sf == 90: if packet_sf == 90:
@ -292,7 +298,7 @@ def sTorResolve(target,
# os.system("strace tor-resolve -4 "+target+" 2>&1|grep '^sen\|^rec'") # os.system("strace tor-resolve -4 "+target+" 2>&1|grep '^sen\|^rec'")
return '' return ''
def getaddrinfo(sHost, sPort): def getaddrinfo(sHost, sPort):
# do this the explicit way = Ive seen the compact connect fail # do this the explicit way = Ive seen the compact connect fail
# >>> sHost, sPort = 'l27.0.0.1', 33446 # >>> sHost, sPort = 'l27.0.0.1', 33446
@ -393,4 +399,4 @@ def lExitExcluder(oArgs, iPort=9051, log_level=10):
if __name__ == '__main__': if __name__ == '__main__':
target = 'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad' target = 'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad'
controller = oGetStemController(log_level=10) controller = oGetStemController(log_level=10)
lIntroductionPoints(controller, [target], itimeout=120) lIntroductionPoints(controller, [target], itimeout=120)

View File

@ -355,7 +355,7 @@ def _my_match_hostname(cert, asserted_hostname):
try: try:
my_match_hostname(cert, asserted_hostname) my_match_hostname(cert, asserted_hostname)
except CertificateError as e: except CertificateError as e:
log.warning( LOG.warning(
"Certificate did not match hostname: %s. Certificate: %s", "Certificate did not match hostname: %s. Certificate: %s",
asserted_hostname, asserted_hostname,
cert, cert,