add exclude_badExits.bash

exclude_badExits.bash

@@ -0,0 +1,41 @@
+#!/bin/bash
+# -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*-
+
+PROG=exclude_badExits.py
+SOCKS_PORT=9050
+CAFILE=/etc/ssl/certs/ca-certificates.crt
+
+# an example of running exclude_badExits with full debugging
+# expected to take an hour or so
+declare -a LARGS
+LARGS=(
+    --log_level 10
+    )
+# you may have a special python for installed packages
+EXE=`which python3.bash`
+LARGS+=(
+    --strict_nodes 0
+    --points_timeout 120
+    --proxy-host 127.0.0.1
+    --proxy-port $SOCKS_PORT
+    --https_cafile $CAFILE
+)
+
+if [ -f '/run/tor/control' ] ; then
+    LARGS+=(--proxy-ctl '/run/tor/control' )
+else
+    LARGS+=(--proxy-ctl 9051 )
+fi
+
+ddg=duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad
+# for example, whitelist the introduction points to DuckDuckGo
+LARGS+=( --white_onions $ddg )
+
+# you may need to be the tor user to read /run/tor/control
+grep -q ^debian-tor /etc/group && TORU=debian-tor || {
+    grep -q ^tor /etc/group && TORU=tor
+}
+sudo -u $TORU $EXE exclude_badExits.py "${LARGS[@]}" \
+   2>&1|tee exclude_badExits6.log
+
+# The DEBUG statements contain the detail of why the relay was considered bad.

exclude_badExits.py

@@ -65,7 +65,7 @@ not just exclude Exit.
 If the Contact info is good we add the list of fingerprints to add
 to ExitNodes, a whitelist of relays to use as exits.
 
-```--proof_output``` will write the contact info as a ciiss dictionary
+```--good_contacts``` will write the contact info as a ciiss dictionary
 to a YAML file. If the proof is uri-rsa, the well-known file of fingerprints
 is downloaded and the fingerprints are added on a 'fps' field we create
 of that fingerprint's entry of the YAML dictionary. This file is read at the
@@ -99,7 +99,8 @@ from stem.connection import IncorrectPassword
 from stem.util.tor_tools import is_valid_fingerprint
 try:
     from ruamel.yaml import YAML
-    yaml = YAML()
+    yaml = YAML(typ='rt')
+    yaml.indent(mapping=2, sequence=2)
     safe_load = yaml.load
 except:
     yaml = None
@@ -451,7 +452,7 @@ def oMainArgparser(_=None):
     parser.add_argument('--torrc_output', type=str,
                         default=os.path.join(ETC_DIR, 'torrc.new'),
                         help="Write the torrc configuration to a file")
-    parser.add_argument('--proof_output', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'),
+    parser.add_argument('--good_contacts', type=str, default=os.path.join(ETC_DIR, 'goodcontacts.yaml'),
                         help="Write the proof data of the included nodes to a YAML file")
     return parser
 
@@ -460,7 +461,7 @@ def vwrite_badnodes(oArgs, oBAD_NODES, slen):
         tmp = oArgs.bad_nodes +'.tmp'
         bak = oArgs.bad_nodes +'.bak'
         with open(tmp, 'wt') as oFYaml:
-            yaml.dump(oBAD_NODES, indent=2, stream=oFYaml)
+            yaml.dump(oBAD_NODES, oFYaml)
             LOG.info(f"Wrote {slen}  to {oArgs.bad_nodes}")
             oFYaml.close()
         if os.path.exists(oArgs.bad_nodes):
@@ -472,7 +473,7 @@ def vwrite_goodnodes(oArgs, oGOOD_NODES, ilen):
         tmp = oArgs.good_nodes +'.tmp'
         bak = oArgs.good_nodes +'.bak'
         with open(tmp, 'wt') as oFYaml:
-            yaml.dump(oGOOD_NODES, indent=2, stream=oFYaml)
+            yaml.dump(oGOOD_NODES, oFYaml)
             LOG.info(f"Wrote {ilen} good relays to {oArgs.good_nodes}")
             oFYaml.close()
         if os.path.exists(oArgs.good_nodes):
@@ -497,7 +498,7 @@ def iMain(lArgs):
         icheck_torrc(sFile, oArgs)
 
     twhitelist_set = set()
-    sFile = oArgs.proof_output
+    sFile = oArgs.good_contacts
     if sFile and os.path.exists(sFile):
         try:
             with open(sFile, 'rt') as oFd:
@@ -527,8 +528,8 @@ def iMain(lArgs):
 
     vwait_for_controller(controller, oArgs.wait_boot)
 
-    if oArgs.proof_output:
+    if oArgs.good_contacts:
-        proof_output_tmp = oArgs.proof_output + '.tmp'
+        good_contacts_tmp = oArgs.good_contacts + '.tmp'
 
     elt = controller.get_conf('UseMicrodescriptors')
     if elt != '0' :
@@ -711,10 +712,10 @@ def iMain(lArgs):
             aTRUST_DB[relay.fingerprint] = b
             for elt in b['fps']:
                 aTRUST_DB_INDEX[elt] = b
-            if oArgs.proof_output and oArgs.log_level <= 20:
+            if oArgs.good_contacts and oArgs.log_level <= 20:
                 # as we go along then clobber
-                with open(proof_output_tmp, 'wt') as oFYaml:
+                with open(good_contacts_tmp, 'wt') as oFYaml:
-                    yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
+                    yaml.dump(aTRUST_DB, oFYaml)
                     oFYaml.close()
 
     LOG.info(f"Filtered {len(twhitelist_set)} whitelisted relays")
@@ -723,16 +724,6 @@ def iMain(lArgs):
     texclude_set = texclude_set.difference(tdns_urls)
     LOG.info(f"{len(list(aTRUST_DB.keys()))} good contacts out of {iTotalContacts}")
 
-    if oArgs.proof_output and aTRUST_DB:
-        with open(proof_output_tmp, 'wt') as oFYaml:
-            yaml.dump(aTRUST_DB, indent=2, stream=oFYaml)
-            oFYaml.close()
-        if os.path.exists(oArgs.proof_output):
-            bak = oArgs.proof_output +'.bak'
-            os.rename(oArgs.proof_output, bak)
-        os.rename(proof_output_tmp, oArgs.proof_output)
-        LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.proof_output}")
-
     if oArgs.torrc_output and texclude_set:
         with open(oArgs.torrc_output, 'wt') as oFTorrc:
             oFTorrc.write(f"{sEXCLUDE_EXIT_KEY} {','.join(texclude_set)}\n")
@@ -744,9 +735,19 @@ def iMain(lArgs):
     if oArgs.bad_contacts and aBadContacts:
         # for later analysis
         with open(oArgs.bad_contacts, 'wt') as oFYaml:
-            yaml.dump(aBadContacts, indent=2, stream=oFYaml)
+            yaml.dump(aBadContacts, oFYaml)
             oFYaml.close()
 
+    if oArgs.good_contacts != '' and aTRUST_DB:
+        with open(good_contacts_tmp, 'wt') as oFYaml:
+            yaml.dump(aTRUST_DB, oFYaml)
+            oFYaml.close()
+        if os.path.exists(oArgs.good_contacts):
+            bak = oArgs.good_contacts +'.bak'
+            os.rename(oArgs.good_contacts, bak)
+        os.rename(good_contacts_tmp, oArgs.good_contacts)
+        LOG.info(f"Wrote {len(list(aTRUST_DB.keys()))} good contact details to {oArgs.good_contacts}")
+
     oBAD_NODES[oBAD_ROOT]['ExcludeNodes']['BadExit'] = list(texclude_set)
     oBAD_NODES[oBAD_ROOT]['ExcludeDomains'] = lKNOWN_NODNS
     vwrite_badnodes(oArgs, oBAD_NODES, str(len(texclude_set)))
@@ -754,6 +755,7 @@ def iMain(lArgs):
     oGOOD_NODES['GoodNodes']['Relays']['ExitNodes'] = list(aTRUST_DB_INDEX.keys())
     # GuardNodes are readonl
     vwrite_goodnodes(oArgs, oGOOD_NODES, len(aTRUST_DB_INDEX.keys()))
+    
     retval = 0
     try:
         logging.getLogger('stem').setLevel(30)
@@ -790,9 +792,6 @@ def iMain(lArgs):
             LOG.errro(f"Failed setting {sINCLUDE_EXIT_KEY} good exit nodes in Tor")
             retval += 1
 
-        sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
-        return retval
-
     except InvalidRequest as e:
         # Unacceptable option value: Invalid router list.
         LOG.error(str(e))
@@ -817,6 +816,9 @@ def iMain(lArgs):
         except Exception as e:
             LOG.warn(str(e))
             
+    sys.stdout.write("dns-rsa domains:\n" +'\n'.join(tdns_urls) +'\n')
+    return retval
+
 if __name__ == '__main__':
     try:
         i = iMain(sys.argv[1:])