This commit is contained in:
emdee@spm.plastiras.org 2024-02-24 07:32:50 +00:00
parent 79c9ac434a
commit c693937271
11 changed files with 73 additions and 51 deletions

View File

@ -1,12 +1,24 @@
PREFIX=/usr/local PREFIX=/usr/local
PYTHON_EXE_MSYS=${PREFIX}/bin/python3.sh PYTHON_EXE_MSYS=${PREFIX}/bin/python3.sh
PIP_EXE_MSYS=${PREFIX}/bin/pip3.sh PIP_EXE_MSYS=${PREFIX}/bin/pip3.sh
PYTHON_MINOR=`python3 --version 2>&1 | sed -e 's@^.* @@' -e 's@\.[0-9]*$$@@'`
LOCAL_DOCTEST=${PREFIX}/bin/toxcore_run_doctest3.bash LOCAL_DOCTEST=${PREFIX}/bin/toxcore_run_doctest3.bash
DOCTEST=${LOCAL_DOCTEST} DOCTEST=${LOCAL_DOCTEST}
MOD=exclude_badExits MOD=exclude_badExits
all:: README.md exclude_badExits.hlp
exclude_badExits.hlp:: src/exclude_badExits/__main__.py
PYTHONPATH=${PWD}/src \
${PYTHON_EXE_MSYS} -m exclude_badExits --help > exclude_badExits.hlp
README.md:: src/exclude_badExits/__main__.py
PYTHONPATH=${PWD}/src \
${PYTHON_EXE_MSYS} -c 'from exclude_badExits.__main__ import __doc__; print(__doc__)' \
> README.md
check:: check::
sh python3.sh -c "import ${MOD}" sh ${PYTHON_EXE_MSYS} -c "import ${MOD}"
lint:: lint::
sh .pylint.sh sh .pylint.sh
@ -16,6 +28,8 @@ install::
--no-deps \ --no-deps \
--target ${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/ \ --target ${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/ \
--upgrade . --upgrade .
sed -i -e "1s@/usr/bin/python${PYTHON_MINOR}@${PYTHON_EXE_MSYS}@" \
${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/bin/*
rsync:: rsync::
bash .rsync.sh bash .rsync.sh

View File

@ -42,8 +42,8 @@ Use the GoodNodes/Onions list to list onion services you want the
Introduction Points whitelisted - these points may change daily Introduction Points whitelisted - these points may change daily
Look in tor's notice.log for warnings of 'Every introduction point for service' Look in tor's notice.log for warnings of 'Every introduction point for service'
```--hs_dir``` ```default='/var/lib/tor'``` will make the program ```--hs_dir``` ```default='/var/lib/tor'``` will make the program
parse the files named ```hostname``` below this dir to find parse the files named ```hostname``` below this dir to find
Hidden Services to whitelist. Hidden Services to whitelist.
The Introduction Points can change during the day, so you may want to The Introduction Points can change during the day, so you may want to
@ -86,9 +86,12 @@ Use the GoodNodes/Onions list in goodnodes.yaml to list onion services
you want the Introduction Points whitelisted - these points may change daily. you want the Introduction Points whitelisted - these points may change daily.
Look in tor's notice.log for 'Every introduction point for service' Look in tor's notice.log for 'Every introduction point for service'
```notice_log``` will parse the notice log for warnings about relays and ```notice_log``` will parse the notice log for warnings about relays and
services that will then be whitelisted. services that will then be whitelisted.
```--torrc``` will read a file like /etc/tor/torrc and make some
suggestions based on what it finds; it will not edit or change the file.
```--torrc_output``` will write the torrc ExcludeNodes configuration to a file. ```--torrc_output``` will write the torrc ExcludeNodes configuration to a file.
```--good_contacts``` will write the contact info as a ciiss dictionary ```--good_contacts``` will write the contact info as a ciiss dictionary
@ -129,15 +132,10 @@ To be "good" the ContactInfo must:
```--wellknown_output``` will make the program write the well-known files ```--wellknown_output``` will make the program write the well-known files
(```/.well-known/tor-relay/rsa-fingerprint.txt```) to a directory. (```/.well-known/tor-relay/rsa-fingerprint.txt```) to a directory.
```--torrc_output``` will write a file of the commands that it sends to ```--relays_output write the download relays in json to a file. The relays
the Tor controller, so you can include it in a ```/etc/toc/torrc```.
```--relays_output``` write the download relays in json to a file. The relays
are downloaded from https://onionoo.torproject.org/details are downloaded from https://onionoo.torproject.org/details
For usage, do ```python3 exclude_badExits.py --help``` For usage, do ```python3 exclude_badExits.py --help`
See [exclude_badExits.txt](./exclude_badExits.txt)
See [exclude_badExits.hlp](./exclude_badExits.hlp)
or there's a doctest file in [exclude_badExits.txt](./exclude_badExits.txt)
Up-to-date code is on https://git.plastiras.org/emdee/exclude_badExits

0
exclude_badExits-installer.bash Normal file → Executable file
View File

0
exclude_badExits-pki.bash Normal file → Executable file
View File

12
exclude_badExits.bash Normal file → Executable file
View File

@ -6,16 +6,22 @@ SOCKS_PORT=9050
SOCKS_HOST=127.0.0.1 SOCKS_HOST=127.0.0.1
CAFILE=/etc/ssl/certs/ca-certificates.crt CAFILE=/etc/ssl/certs/ca-certificates.crt
# you may have a special python for installed packages # you may have a special python for installed packages
EXE=`which python3.bash` EXE=`which python3.sh`
MOD=exclude_badExits MOD=exclude_badExits
[ -f exclude_badExits.hlp -a exclude_badExits.hlp -nt src/exclude_badExits/exclude_badExits.py] || \ if [ ! -f exclude_badExits.hlp ] || \
[ exclude_badExits.hlp -nt src/exclude_badExits/exclude_badExits.py ] ; then
PYTHONPATH=$PWD/src \ PYTHONPATH=$PWD/src \
$EXE src/${MOD}/exclude_badExits.py --help > exclude_badExits.hlp $EXE src/${MOD}/exclude_badExits.py --help > exclude_badExits.hlp
[ -f README.md -a README.md -nt src/exclude_badExits/exclude_badExits.py] || \ fi
if [ ! -f README.md ] || \
[ README.md -nt src/exclude_badExits/exclude_badExits.py ] ; then
PYTHONPATH=$PWD/src \ PYTHONPATH=$PWD/src \
$EXE -c 'from exclude_badExits.exclude_badExits import __doc__; print(__doc__)' \ $EXE -c 'from exclude_badExits.exclude_badExits import __doc__; print(__doc__)' \
> README.md > README.md
fi
# an example of running exclude_badExits with full debugging # an example of running exclude_badExits with full debugging
# expected to 20 minutes or so # expected to 20 minutes or so
declare -a LARGS declare -a LARGS

View File

@ -20,7 +20,7 @@ classifiers = [
dynamic = ["version", "readme", "dependencies"] # cannot be dynamic ['license'] dynamic = ["version", "readme", "dependencies"] # cannot be dynamic ['license']
[project.scripts] [project.scripts]
exclude_badExits = "exclude_badExits.exclude_badExits:iMain" exclude_badExits = "exclude_badExits.__main__:iMain"
[tool.setuptools.dynamic] [tool.setuptools.dynamic]
version = {attr = "exclude_badExits.__version__"} version = {attr = "exclude_badExits.__version__"}

View File

@ -1,2 +1,8 @@
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
__version__ = "1.0.0" __version__ = "1.0.0"
__all__ = [
'exclude_utils',
'torcontactinfo',
'trustor_poc']

View File

@ -197,30 +197,29 @@ from toxygen_wrapper.tests.support_onions import (
bAreWeConnected, icheck_torrc, lIntroductionPoints, bAreWeConnected, icheck_torrc, lIntroductionPoints,
yKNOWN_NODNS, zResolveDomain) yKNOWN_NODNS, zResolveDomain)
from trustor_poc import TrustorError, idns_validate from exclude_badExits.trustor_poc import TrustorError, idns_validate
try: if False:
import xxxhttpx import httpx
import asyncio import asyncio
from trustor_poc import oDownloadUrlHttpx from exclude_badExits.trustor_poc import oDownloadUrlHttpx
except: else:
httpx = None httpx = None
from trustor_poc import oDownloadUrlUrllib3Socks as oDownloadUrl from exclude_badExits.trustor_poc import oDownloadUrlUrllib3Socks as oDownloadUrl
try: from exclude_badExits.torcontactinfo import TorContactInfoParser
from torcontactinfo import TorContactInfoParser
oPARSER = TorContactInfoParser()
except ImportError:
oPARSER = None
from exclude_utils import (aCleanContact, sCleanEmail, aParseContact, from exclude_badExits.exclude_utils import (
oStemController, oMainArgparser, aCleanContact, sCleanEmail, aParseContact,
vwrite_goodnodes, vwrite_badnodes, vwrite_good_contacts, oStemController, oMainArgparser,
vwritefinale, vsetup_logging) vwrite_goodnodes, vwrite_badnodes, vwrite_good_contacts,
vwritefinale, vsetup_logging )
warnings.filterwarnings('ignore') warnings.filterwarnings('ignore')
global LOG global LOG
LOG = logging.getLogger() LOG = logging.getLogger()
oPARSER = TorContactInfoParser()
aGOOD_CONTACTS_DB = {} aGOOD_CONTACTS_DB = {}
aGOOD_CONTACTS_FPS = {} aGOOD_CONTACTS_FPS = {}
aBAD_CONTACTS_DB = {} aBAD_CONTACTS_DB = {}
@ -555,10 +554,10 @@ def aContactFps(oargs, a, fp, o, domain):
if oargs.wellknown_output: if oargs.wellknown_output:
sdir = os.path.join(oargs.wellknown_output, domain, sdir = os.path.join(oargs.wellknown_output, domain,
'.well-known', 'tor-relay') '.well-known', 'tor-relay')
sfile = os.path.join(sdir, "rsa-fingerprint.txt")
try: try:
if not os.path.isdir(sdir): if not os.path.isdir(sdir):
os.makedirs(sdir) os.makedirs(sdir)
sfile = os.path.join(sdir, "rsa-fingerprint.txt")
with open(sfile, 'wt') as oFd: with open(sfile, 'wt') as oFd:
oFd.write(data) oFd.write(data)
except Exception as e: except Exception as e:
@ -665,6 +664,7 @@ def bProcessContact(b, texclude_set, aBadContacts, iFakeContact=0):
return True return True
lNOT_IN_RELAYS_DB = []
def bCheckFp(relay, sofar, lConds, texclude_set): def bCheckFp(relay, sofar, lConds, texclude_set):
global aGOOD_CONTACTS_DB global aGOOD_CONTACTS_DB
global aGOOD_CONTACTS_FPS global aGOOD_CONTACTS_FPS
@ -853,7 +853,6 @@ def iMain(lArgs):
iFakeContact = 0 iFakeContact = 0
iTotalContacts = 0 iTotalContacts = 0
aBadContacts = {} aBadContacts = {}
lNOT_IN_RELAYS_DB = []
iR = 0 iR = 0
relays = controller.get_server_descriptors() relays = controller.get_server_descriptors()
lqueue = [] lqueue = []

View File

@ -11,7 +11,7 @@ import logging
import warnings import warnings
global LOG global LOG
from toxygen_wrapper.tests.support_onions import (woGetStemController, from toxygen_wrapper.tests.support_onions import (oGetStemController,
vwait_for_controller,) vwait_for_controller,)
try: try:
@ -20,8 +20,6 @@ try:
yaml.indent(mapping=2, sequence=2) yaml.indent(mapping=2, sequence=2)
safe_load = yaml.load safe_load = yaml.load
except: except:
yaml = None
if yaml is None:
try: try:
import yaml import yaml
safe_load = yaml.safe_load safe_load = yaml.safe_load
@ -36,6 +34,15 @@ try:
except ImportError: except ImportError:
coloredlogs = False coloredlogs = False
lMORONS = ['hoster:Quintex Alliance Consulting ']
oCONTACT_RE = re.compile(r'([^:]*)(\s+)(email|url|proof|ciissversion|abuse|gpg):')
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
'sandbox', 'offlinemasterkey']
lEMAILS = ['abuse', 'email']
ETC_DIR = '/usr/local/etc/tor/yaml'
def aCleanContact(a, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> dict: def aCleanContact(a, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> dict:
# cleanups # cleanups
for elt in lINTS: for elt in lINTS:
@ -78,15 +85,6 @@ def sCleanEmail(s, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> str:
s = s.replace(elt, '?') s = s.replace(elt, '?')
return s return s
lMORONS = ['hoster:Quintex Alliance Consulting ']
oCONTACT_RE = re.compile(r'([^:]*)(\s+)(email|url|proof|ciissversion|abuse|gpg):')
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
'sandbox', 'offlinemasterkey']
lEMAILS = ['abuse', 'email']
ETC_DIR = '/usr/local/etc/tor/yaml'
def oStemController(oargs, sEXCLUDE_EXIT_GROUP): def oStemController(oargs, sEXCLUDE_EXIT_GROUP):
if os.path.exists(oargs.proxy_ctl): if os.path.exists(oargs.proxy_ctl):
controller = oGetStemController(log_level=oargs.log_level, controller = oGetStemController(log_level=oargs.log_level,
@ -261,7 +259,6 @@ def vsetup_logging(theLOG, log_level, logfile='', stream=sys.stdout) -> None:
kwargs = dict(level=log_level, kwargs = dict(level=log_level,
force=True, force=True,
format='%(levelname)s %(message)s') format='%(levelname)s %(message)s')
logging.basicConfig(**kwargs) logging.basicConfig(**kwargs)
if add and logfile: if add and logfile:
oHandler = logging.StreamHandler(stream) oHandler = logging.StreamHandler(stream)

View File

@ -29,7 +29,7 @@ import textwrap
# HAS_RICH = True # HAS_RICH = True
if True: if True:
def rprint(value='', *args, **kwargs): def rprint(value='', *args, **kwargs):
if value not in [None, False, True] and if value not in [None, False, True] and \
isinstance(value, (dict, list, set, tuple)): isinstance(value, (dict, list, set, tuple)):
value = json.dumps(value, indent=4) value = json.dumps(value, indent=4)
return LOG.debug(value, *args, **kwargs) return LOG.debug(value, *args, **kwargs)

View File

@ -9,7 +9,7 @@ import re
import sys import sys
import ipaddress import ipaddress
import warnings import warnings
import logging
import urllib3.util import urllib3.util
from urllib3.util import parse_url as urlparse from urllib3.util import parse_url as urlparse
@ -24,7 +24,6 @@ except:
ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
global LOG global LOG
import logging
warnings.filterwarnings('ignore') warnings.filterwarnings('ignore')
LOG = logging.getLogger() LOG = logging.getLogger()
@ -36,7 +35,7 @@ logging.getLogger("urllib3").setLevel(logging.INFO)
# https://github.com/erans/torcontactinfoparser # https://github.com/erans/torcontactinfoparser
# sys.path.append('/home/....') # sys.path.append('/home/....')
try: try:
from torcontactinfo import TorContactInfoParser from exclude_badExits.torcontactinfo import TorContactInfoParser
except: except:
TorContactInfoParser = None TorContactInfoParser = None
@ -266,7 +265,10 @@ def oDownloadUrlRequests(uri, sCAfile: str, timeout: int = 30, host: str = '127.
return oReqResp return oReqResp
# There's no point in using asyncio because of duplicate urls in the tasks # There's no point in using asyncio because of duplicate urls in the tasks
async def oDownloadUrlHttpx(uri:str, sCAfile:str, timeout;int = 30, host:str = '127.0.0.1', port:int = 9050, content_type:str = 'text/plain'): async def oDownloadUrlHttpx(uri:str, sCAfile:str, timeout:int = 30,
host:str = '127.0.0.1', port:int = 9050,
content_type:str = 'text/plain'):
import httpcore import httpcore
import asyncio import asyncio
import httpx import httpx