exclude_badExits/exclude_badExits.py

# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-

# https://github.com/nusenu/noContactInfo_Exit_Excluder
# https://github.com/TheSmashy/TorExitRelayExclude
"""
This extends nusenu's basic idea of using the stem library to
dynamically exclude nodes that are likely to be bad by putting them
on the ExcludeNodes or ExcludeExitNodes setting of a running Tor.
* https://github.com/nusenu/noContactInfo_Exit_Excluder
* https://github.com/TheSmashy/TorExitRelayExclude

The basic cut is to exclude Exit nodes that do not have a contact.
That can be extended to nodes that do not have an email in the contact etc.
"""
"""
But there's a problem, and your Tor notice.log will tell you about it:
you could exclude the nodes needed to access hidden services or
directorues. So we need to add to the process the concept of a whitelist. 
In addition, we may have our own blacklist of nodes we want to exclude,
or use these lists for other applications like selektor.

So we make two files that are structured in YAML:
```
/etc/tor/torrc-goodnodes.yaml
Nodes:
  IntroductionPoints:
    - $NODEFINGERPRINT
  ...
By default all sections of the goodnodes.yaml are used as a whitelist.

/etc/tor/torrc-badnodes.yaml
Nodes:
  ExcludeExitNodes:
    BadExit:
      # $0000000000000000000000000000000000000007
```
That part requires [PyYAML](https://pyyaml.org/wiki/PyYAML)
https://github.com/yaml/pyyaml/

Right now only the ExcludeExitNodes section is used by we may add ExcludeNodes
later, and by default all sub-sections of the badnodes.yaml are used as a
ExcludeExitNodes but it can be customized with the lWanted commandline arg.

The original idea has also been extended to add different conditions for
exclusion: the ```--contact``` commandline arg is a comma sep list of conditions:
* Empty - no contact info
* NoEmail - no @ sign in the contact',
More may be added later.

Because you don't want to exclude the introduction points to any onion
you want to connect to, ```--white_onions``` should whitelist the 
introduction points to a  comma sep list of onions, but is 
currently broken in stem 1.8.0: see:
* https://github.com/torproject/stem/issues/96
* https://gitlab.torproject.org/legacy/trac/-/issues/25417

```--bad_output``` will write the torrc ExcludeNodes configuration to a file.

```--details_output``` will write the lookup URLs of the excluded nodes to a file

```--proof_output``` will write the contact info as a ciiss dictionary
to a YAML file. If the proof is uri-rsa, the well-known file of fingerprints
is downloaded and the fingerprints are added to the on the 'fps' field
 of that fingerprint entry of the YAML dictionary. This file is read at the 
beginning of the program to start with a trust database, and only new
relays are added to the dictionary. The 'fps' field is emptied if the
host fails to provide the well-known file. You can expect it to take
an hour or two the first time this is run: >700 domains.

For usage, do ```python3 exclude_badExits.py --help`

"""

import sys

import os
import getpass
import re
import time
import argparse
from io import StringIO

from stem.control import Controller
from stem.connection import IncorrectPassword
from stem.util.tor_tools import is_valid_fingerprint
try:
    import yaml
except:
    yaml = None
    
try:
    import coloredlogs
    if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ:
        os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
    # https://pypi.org/project/coloredlogs/
except ImportError as e:
    coloredlogs = False
from trustor_poc import lDownloadUrlFps

global LOG
import logging
LOG = logging.getLogger()

aTRUST_DB = {}
sDETAILS_URL = "https://metrics.torproject.org/rs.html#details/"
# You can call this while bootstrapping

def oMakeController(sSock='', port=9051):
    if sSock and os.path.exists(sSock):
        controller = Controller.from_socket_file(path=sSock)
    else:
        controller = Controller.from_port(port=port)
    sys.stdout.flush()
    p = getpass.unix_getpass(prompt='Controller Password: ', stream=sys.stderr)
    controller.authenticate(p)
    return controller

def lYamlBadNodes(sFile='/etc/tor/torrc-badnodes.yaml',
                  section='ExcludeExitNodes',
                  lWanted=['Hetzner','BadExit']):
    root = 'ExcludeNodes'
    l = []
    if not yaml: return l
    if os.path.exists(sFile):
        with open(sFile, 'rt') as oFd:
            o = yaml.safe_load(oFd)
        for elt in o[root][section].keys():
            if lWanted and elt not in lWanted: continue
            l += o[root][section][elt]
    # yq '.ExcludeNodes.Hetzner' < /etc/tor/torrc-badnodes.yaml |sed -e 's/^[[]/ExcludeNodesHetzner = [/'
    # yq '.ExcludeNodes.Hetzner|.[]' < /etc/tor/torrc-badnodes.yaml
    # yq '.ExcludeNodes.BadExit|.[]' < /etc/tor/torrc-badnodes.yaml
    return l

def lYamlGoodNodes(sFile='/etc/tor/torrc-goodnodes.yaml'):
    root='IncludeNodes'
    l = []
    if not yaml: return l
    if os.path.exists(sFile):
        with open(sFile, 'rt') as oFd:
            o = yaml.safe_load(oFd)
        for elt in o[root].keys():
            l += o[root][elt]
    # yq '.Nodes.IntroductionPoints|.[]' < /etc/tor/torrc-goodnodes.yaml
    return l

def lIntroductionPoints(lOnions):
    """not working in stem 1.8.3"""
    l = []
    for elt in lOnions:
        desc = controller.get_hidden_service_descriptor(elt, await_result=True, timeout=None)
        l = desc.introduction_points()
        if l:
            LOG.warn(f"{elt} NO introduction points\n")
            continue
        LOG.info(f"{elt} introduction points are...\n")

        for introduction_point in l:
            LOG.info('  %s:%s => %s' % (introduction_point.address,
                                        introduction_point.port,
                                        introduction_point.identifier))
            l += [introduction_point.address]
    return l

# memory?
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime']
lBOOLS =  ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone'
           'sandbox', 'offlinemasterkey']
def aVerifyContact(a, fp, timeout=20, host='127.0.0.1', port=9050):
    for elt in lINTS:
        if elt in a:
            a[elt] = int(a[elt])
    for elt in lBOOLS:
        if elt in a:
            if a[elt] in ['y','yes', 'true', 'True']:
                a[elt] = True
            else:
                a[elt] = False
        
    # just stick fp in for now
    a.update({'fps': [fp]})
    # test the url for fps and add it to the array
    if 'proof' not  in a:
        # only support uri for now
        LOG.warn(f"{fp} 'proof'  not in {list(a.keys())}")
        return a
    if a['proof'] not in ['uri-rsa']:
        # only support uri for now
        LOG.warn(f"{fp} proof={a['proof']} not supported yet")
        return a
    if 'url' not in a:
        LOG.warn(f"{fp} 'proof' is 'uri-rsa' but url not in {list(a.keys())}")
        return a
    if a['url'].startswith('http:'):
        a['url'] = 'https:' +a['url'][5:]
    elif not a['url'].startswith('https:'):
        a['url'] = 'https:' +a['url']
    domain = a['url'][8:]
    LOG.debug(f"{len(list(a.keys()))} contact fields for {fp}")
    LOG.info(f"Downloading from {domain} for {fp}")
    try:
        l = lDownloadUrlFps(domain, timeout=20, host=host, port=port)
    except Exception as e:
        LOG.exception(f"Error downloading from {domain} for {fp} {e}")
        # should we put it's FPs from TRUST_DB on the ExcludeExitNodes?
        a['fps'] = []
    else:
        if not l:
            LOG.warn(f"Downloading from {domain} failed for {fp}")
            a['fps'] = []
        else:
            a['fps'] = l
    return a

def aParseContact(contact, fp):
    contact = str(contact, 'UTF-8')
    l = [line for line in contact.strip().replace('"', '').split(' ')
         if ':' in line]
    LOG.debug(f"{fp} {len(l)} fields")
    s = f'"{fp}":\n'
    s += '\n'.join([f"  {line}\"".replace(':',': \"', 1)
                    for line in l])
    oFd = StringIO(s)
    a = yaml.safe_load(oFd)
    return a

def oMainArgparser(_=None):
    # 'Mode: 0=chat 1=chat+audio 2=chat+audio+video default: 0'

    parser = argparse.ArgumentParser(add_help=True,
                                     epilog=__doc__)
    parser.add_argument('--proxy_host', '--proxy-host', type=str,
                        default='127.0.0.1',
                        help='proxy host')
    parser.add_argument('--proxy_port', '--proxy-port', default=9050, type=int,
                        help='proxy control port')
    parser.add_argument('--proxy_ctl', '--proxy-ctl',
                        default='/run/tor/control',
                        type=str,
                        help='control socket - or port')
    parser.add_argument('--timeout', default=20, type=int,
                        help='proxy download timeout')

    parser.add_argument('--good_nodes', type=str,
                        default='/etc/tor/torrc-goodnodes.yaml',
                        help="Yaml file of good nodes that should not be excluded")
    parser.add_argument('--bad_nodes', type=str,
                        default='/etc/tor/torrc-badnodes.yaml',
                        help="Yaml file of bad nodes that should also be excluded")
    parser.add_argument('--contact', type=str, default='Empty,NoEmail',
                        help="comma sep list of conditions - Empty,NoEmail")
    parser.add_argument('--wait_boot', type=int, default=120,
                        help="Seconds to wait for Tor to booststrap")
    parser.add_argument('--log_level', type=int, default=20,
                        help="10=debug 20=info 30=warn 40=error")
    parser.add_argument('--bad_sections', type=str,
                        default='Hetzner,BadExit',
                        help="sections of the badnodes.yaml to use, comma separated, '' defaults to all")
    parser.add_argument('--white_onions', type=str,
                        default='',
                        help="comma sep. list of onions to whitelist their introduction points - BROKEN")
    parser.add_argument('--bad_output', type=str, default='',
                        help="Write the torrc configuration to a file")
    parser.add_argument('--details_output', type=str, default='',
                        help="Write the lookup URLs of the excluded nodes to a file")
    parser.add_argument('--proof_output', type=str, default='',
                        help="Write the proof data of the included nodes to a YAML file")
    return parser

def iMain(lArgs):
    global     oTOX_OARGS
    global aTRUST_DB
    parser = oMainArgparser()
    oArgs = parser.parse_args(lArgs)

    aKw = dict(level=oArgs.log_level,
            format='%(name)s %(levelname)-4s %(message)s',
            stream=sys.stdout,
            force=True)
    logging.basicConfig(**aKw)
    logging.getLogger('stem').setLevel(oArgs.log_level)

    sFile = oArgs.proof_output
    if sFile and os.path.exists(sFile):
        with open(sFile, 'rt') as oFd:
            aTRUST_DB = yaml.safe_load(oFd)

    if oArgs.proxy_ctl.startswith('/') or os.path.exists(oArgs.proxy_ctl):
        controller = oMakeController(sSock=oArgs.proxy_ctl)
    else:
        port =int(oArgs.proxy_ctl)            
        controller = oMakeController(port=port)
    
    elt = controller.get_conf('UseMicrodescriptors')
    if elt != '0' :
        LOG.warn('"UseMicrodescriptors 0" is required in your /etc/tor/torrc. Exiting.')
        controller.set_conf('UseMicrodescriptors', 0)
        # does it work dynamically?
        # return 2
        
    percent = i = 0
    # You can call this while boostrapping
    while percent < 100 and i < oArgs.wait_boot:
        bootstrap_status = controller.get_info("status/bootstrap-phase")
        progress_percent = re.match('.* PROGRESS=([0-9]+).*', bootstrap_status)
        percent = int(progress_percent.group(1))
        LOG.info(f"Bootstrapping {percent}%")
        time.sleep(5)
        i += 5
    elt = controller.get_conf('ExcludeExitNodes')
    if elt and elt != '{??}':
        LOG.warn(f'ExcludeExitNodes is in use already')

    lGood = lYamlGoodNodes(oArgs.good_nodes)
    LOG.info(f'lYamlGoodNodes {len(lGood)}')

    if oArgs.white_onions:
        l = lIntroductionPoints(oArgs.white_onions.split(','))
        lGood += l

    relays = controller.get_server_descriptors()

    if oArgs.bad_sections:
        sections = oArgs.bad_sections.split(',') 
        exit_excludelist = lYamlBadNodes(lWanted=sections)
    else:
        exit_excludelist = lYamlBadNodes()
        
    LOG.info(f'lYamlBadNodes {len(exit_excludelist)}')

    if oArgs.details_output:
        oFd = open(oArgs.details_output, 'wt')
    else:
        oFd = None

    lProofUriFps = []
    aProofUri = {}
    lConds = oArgs.contact.split(',')
    for relay in relays:
        if not relay.exit_policy.is_exiting_allowed(): continue
        if not is_valid_fingerprint(relay.fingerprint):
            LOG.warn('Invalid Fingerprint: %s' % relay.fingerprint)
            continue
        
        if relay.fingerprint in aTRUST_DB:
            if aTRUST_DB[relay.fingerprint]['fps']:
                lProofUriFps += aTRUST_DB[relay.fingerprint]['fps']
                
        if relay.fingerprint in lProofUriFps:
            # we already have it.
            continue
        if relay.contact and b'proof:uri-rsa' in relay.contact.lower():
            a = aParseContact(relay.contact, relay.fingerprint)
            if not a: continue
            b = aVerifyContact(list(a.values())[0], relay.fingerprint,
                               timeout=oArgs.timeout,
                               host=oArgs.proxy_host,
                               port=oArgs.proxy_port)
            if not b:
                continue
            if 'fps' in b and b['fps'] and relay.fingerprint in b['fps']:
                lProofUriFps += b['fps']
                aProofUri[relay.fingerprint] = b
        if ('Empty' in lConds and not relay.contact) or \
           ('NoEmail' in lConds and relay.contact and not b'@' in relay.contact):
            exit_excludelist.append(relay.fingerprint)
            if oFd:
                oFd.write(sDETAILS_URL +relay.fingerprint +"\n")

    exit_excludelist = list(set(exit_excludelist).difference(set(lGood)))
    LOG.info(f'ExcludeExitNodes {len(exit_excludelist)} net bad exit nodes')
    controller.set_conf('ExcludeExitNodes', exit_excludelist)
    elt = controller.get_conf('ExcludeExitNodes')
    if oArgs.bad_output:
        with open(oArgs.bad_output, 'wt') as oFdE:
            oFdE.write(f"ExcludeExitNodes {','.join(exit_excludelist)}\n")
        LOG.info(f"Wrote tor configuration to {oArgs.bad_output}")
    if lProofUriFps:
        LOG.info(f'ExitNodes {len(lProofUriFps)} good exit nodes')
        controller.set_conf('ExitNodes', lProofUriFps)
        
    if oFd:
        LOG.info(f"Wrote details URLs to {oArgs.details_output}")
        oFd.close()
        
    if oArgs.proof_output:
        with open(oArgs.proof_output, 'wt') as oFdD:
            s = yaml.dump_all(aProofUri, indent=2, stream=None)
            oFdD.write(s +'\n')
            LOG.info(f"Wrote proof details to {oArgs.proof_output}")
            oFdD.close()
    
    logging.getLogger('stem').setLevel(40)
    for elt in controller._event_listeners:
        controller.remove_event_listener(elt)
    controller.close()
    
    return(0)

if __name__ == '__main__':
    try:
        i = iMain(sys.argv[1:])
    except IncorrectPassword as e:
        LOG.error(e)
        i = 1
    except Exception as e:
        LOG.exception(e)
        i = 1
    sys.exit(i)