This commit is contained in:
emdee@spm.plastiras.org 2024-02-24 07:32:50 +00:00
parent 79c9ac434a
commit c693937271
11 changed files with 73 additions and 51 deletions

View File

@ -1,12 +1,24 @@
PREFIX=/usr/local
PYTHON_EXE_MSYS=${PREFIX}/bin/python3.sh
PIP_EXE_MSYS=${PREFIX}/bin/pip3.sh
PYTHON_MINOR=`python3 --version 2>&1 | sed -e 's@^.* @@' -e 's@\.[0-9]*$$@@'`
LOCAL_DOCTEST=${PREFIX}/bin/toxcore_run_doctest3.bash
DOCTEST=${LOCAL_DOCTEST}
MOD=exclude_badExits
all:: README.md exclude_badExits.hlp
exclude_badExits.hlp:: src/exclude_badExits/__main__.py
PYTHONPATH=${PWD}/src \
${PYTHON_EXE_MSYS} -m exclude_badExits --help > exclude_badExits.hlp
README.md:: src/exclude_badExits/__main__.py
PYTHONPATH=${PWD}/src \
${PYTHON_EXE_MSYS} -c 'from exclude_badExits.__main__ import __doc__; print(__doc__)' \
> README.md
check::
sh python3.sh -c "import ${MOD}"
sh ${PYTHON_EXE_MSYS} -c "import ${MOD}"
lint::
sh .pylint.sh
@ -16,6 +28,8 @@ install::
--no-deps \
--target ${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/ \
--upgrade .
sed -i -e "1s@/usr/bin/python${PYTHON_MINOR}@${PYTHON_EXE_MSYS}@" \
${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/bin/*
rsync::
bash .rsync.sh

View File

@ -42,8 +42,8 @@ Use the GoodNodes/Onions list to list onion services you want the
Introduction Points whitelisted - these points may change daily
Look in tor's notice.log for warnings of 'Every introduction point for service'
```--hs_dir``` ```default='/var/lib/tor'``` will make the program
parse the files named ```hostname``` below this dir to find
```--hs_dir``` ```default='/var/lib/tor'``` will make the program
parse the files named ```hostname``` below this dir to find
Hidden Services to whitelist.
The Introduction Points can change during the day, so you may want to
@ -86,9 +86,12 @@ Use the GoodNodes/Onions list in goodnodes.yaml to list onion services
you want the Introduction Points whitelisted - these points may change daily.
Look in tor's notice.log for 'Every introduction point for service'
```notice_log``` will parse the notice log for warnings about relays and
```notice_log``` will parse the notice log for warnings about relays and
services that will then be whitelisted.
```--torrc``` will read a file like /etc/tor/torrc and make some
suggestions based on what it finds; it will not edit or change the file.
```--torrc_output``` will write the torrc ExcludeNodes configuration to a file.
```--good_contacts``` will write the contact info as a ciiss dictionary
@ -129,15 +132,10 @@ To be "good" the ContactInfo must:
```--wellknown_output``` will make the program write the well-known files
(```/.well-known/tor-relay/rsa-fingerprint.txt```) to a directory.
```--torrc_output``` will write a file of the commands that it sends to
the Tor controller, so you can include it in a ```/etc/toc/torrc```.
```--relays_output``` write the download relays in json to a file. The relays
```--relays_output write the download relays in json to a file. The relays
are downloaded from https://onionoo.torproject.org/details
For usage, do ```python3 exclude_badExits.py --help```
For usage, do ```python3 exclude_badExits.py --help`
See [exclude_badExits.txt](./exclude_badExits.txt)
See [exclude_badExits.hlp](./exclude_badExits.hlp)
or there's a doctest file in [exclude_badExits.txt](./exclude_badExits.txt)
Up-to-date code is on https://git.plastiras.org/emdee/exclude_badExits

0
exclude_badExits-installer.bash Normal file → Executable file
View File

0
exclude_badExits-pki.bash Normal file → Executable file
View File

12
exclude_badExits.bash Normal file → Executable file
View File

@ -6,16 +6,22 @@ SOCKS_PORT=9050
SOCKS_HOST=127.0.0.1
CAFILE=/etc/ssl/certs/ca-certificates.crt
# you may have a special python for installed packages
EXE=`which python3.bash`
EXE=`which python3.sh`
MOD=exclude_badExits
[ -f exclude_badExits.hlp -a exclude_badExits.hlp -nt src/exclude_badExits/exclude_badExits.py] || \
if [ ! -f exclude_badExits.hlp ] || \
[ exclude_badExits.hlp -nt src/exclude_badExits/exclude_badExits.py ] ; then
PYTHONPATH=$PWD/src \
$EXE src/${MOD}/exclude_badExits.py --help > exclude_badExits.hlp
[ -f README.md -a README.md -nt src/exclude_badExits/exclude_badExits.py] || \
fi
if [ ! -f README.md ] || \
[ README.md -nt src/exclude_badExits/exclude_badExits.py ] ; then
PYTHONPATH=$PWD/src \
$EXE -c 'from exclude_badExits.exclude_badExits import __doc__; print(__doc__)' \
> README.md
fi
# an example of running exclude_badExits with full debugging
# expected to 20 minutes or so
declare -a LARGS

View File

@ -20,7 +20,7 @@ classifiers = [
dynamic = ["version", "readme", "dependencies"] # cannot be dynamic ['license']
[project.scripts]
exclude_badExits = "exclude_badExits.exclude_badExits:iMain"
exclude_badExits = "exclude_badExits.__main__:iMain"
[tool.setuptools.dynamic]
version = {attr = "exclude_badExits.__version__"}

View File

@ -1,2 +1,8 @@
# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*-
__version__ = "1.0.0"
__all__ = [
'exclude_utils',
'torcontactinfo',
'trustor_poc']

View File

@ -197,30 +197,29 @@ from toxygen_wrapper.tests.support_onions import (
bAreWeConnected, icheck_torrc, lIntroductionPoints,
yKNOWN_NODNS, zResolveDomain)
from trustor_poc import TrustorError, idns_validate
try:
import xxxhttpx
from exclude_badExits.trustor_poc import TrustorError, idns_validate
if False:
import httpx
import asyncio
from trustor_poc import oDownloadUrlHttpx
except:
from exclude_badExits.trustor_poc import oDownloadUrlHttpx
else:
httpx = None
from trustor_poc import oDownloadUrlUrllib3Socks as oDownloadUrl
from exclude_badExits.trustor_poc import oDownloadUrlUrllib3Socks as oDownloadUrl
try:
from torcontactinfo import TorContactInfoParser
oPARSER = TorContactInfoParser()
except ImportError:
oPARSER = None
from exclude_badExits.torcontactinfo import TorContactInfoParser
from exclude_utils import (aCleanContact, sCleanEmail, aParseContact,
oStemController, oMainArgparser,
vwrite_goodnodes, vwrite_badnodes, vwrite_good_contacts,
vwritefinale, vsetup_logging)
from exclude_badExits.exclude_utils import (
aCleanContact, sCleanEmail, aParseContact,
oStemController, oMainArgparser,
vwrite_goodnodes, vwrite_badnodes, vwrite_good_contacts,
vwritefinale, vsetup_logging )
warnings.filterwarnings('ignore')
global LOG
LOG = logging.getLogger()
oPARSER = TorContactInfoParser()
aGOOD_CONTACTS_DB = {}
aGOOD_CONTACTS_FPS = {}
aBAD_CONTACTS_DB = {}
@ -555,10 +554,10 @@ def aContactFps(oargs, a, fp, o, domain):
if oargs.wellknown_output:
sdir = os.path.join(oargs.wellknown_output, domain,
'.well-known', 'tor-relay')
sfile = os.path.join(sdir, "rsa-fingerprint.txt")
try:
if not os.path.isdir(sdir):
os.makedirs(sdir)
sfile = os.path.join(sdir, "rsa-fingerprint.txt")
with open(sfile, 'wt') as oFd:
oFd.write(data)
except Exception as e:
@ -665,6 +664,7 @@ def bProcessContact(b, texclude_set, aBadContacts, iFakeContact=0):
return True
lNOT_IN_RELAYS_DB = []
def bCheckFp(relay, sofar, lConds, texclude_set):
global aGOOD_CONTACTS_DB
global aGOOD_CONTACTS_FPS
@ -853,7 +853,6 @@ def iMain(lArgs):
iFakeContact = 0
iTotalContacts = 0
aBadContacts = {}
lNOT_IN_RELAYS_DB = []
iR = 0
relays = controller.get_server_descriptors()
lqueue = []

View File

@ -11,7 +11,7 @@ import logging
import warnings
global LOG
from toxygen_wrapper.tests.support_onions import (woGetStemController,
from toxygen_wrapper.tests.support_onions import (oGetStemController,
vwait_for_controller,)
try:
@ -20,8 +20,6 @@ try:
yaml.indent(mapping=2, sequence=2)
safe_load = yaml.load
except:
yaml = None
if yaml is None:
try:
import yaml
safe_load = yaml.safe_load
@ -36,6 +34,15 @@ try:
except ImportError:
coloredlogs = False
lMORONS = ['hoster:Quintex Alliance Consulting ']
oCONTACT_RE = re.compile(r'([^:]*)(\s+)(email|url|proof|ciissversion|abuse|gpg):')
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
'sandbox', 'offlinemasterkey']
lEMAILS = ['abuse', 'email']
ETC_DIR = '/usr/local/etc/tor/yaml'
def aCleanContact(a, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> dict:
# cleanups
for elt in lINTS:
@ -78,15 +85,6 @@ def sCleanEmail(s, lAT_REPS, lDOT_REPS, lNO_EMAIL) -> str:
s = s.replace(elt, '?')
return s
lMORONS = ['hoster:Quintex Alliance Consulting ']
oCONTACT_RE = re.compile(r'([^:]*)(\s+)(email|url|proof|ciissversion|abuse|gpg):')
lINTS = ['ciissversion', 'uplinkbw', 'signingkeylifetime', 'memory']
lBOOLS = ['dnssec', 'dnsqname', 'aesni', 'autoupdate', 'dnslocalrootzone',
'sandbox', 'offlinemasterkey']
lEMAILS = ['abuse', 'email']
ETC_DIR = '/usr/local/etc/tor/yaml'
def oStemController(oargs, sEXCLUDE_EXIT_GROUP):
if os.path.exists(oargs.proxy_ctl):
controller = oGetStemController(log_level=oargs.log_level,
@ -261,7 +259,6 @@ def vsetup_logging(theLOG, log_level, logfile='', stream=sys.stdout) -> None:
kwargs = dict(level=log_level,
force=True,
format='%(levelname)s %(message)s')
logging.basicConfig(**kwargs)
if add and logfile:
oHandler = logging.StreamHandler(stream)

View File

@ -29,7 +29,7 @@ import textwrap
# HAS_RICH = True
if True:
def rprint(value='', *args, **kwargs):
if value not in [None, False, True] and
if value not in [None, False, True] and \
isinstance(value, (dict, list, set, tuple)):
value = json.dumps(value, indent=4)
return LOG.debug(value, *args, **kwargs)

View File

@ -9,7 +9,7 @@ import re
import sys
import ipaddress
import warnings
import logging
import urllib3.util
from urllib3.util import parse_url as urlparse
@ -24,7 +24,6 @@ except:
ub_ctx = RR_TYPE_TXT = RR_CLASS_IN = None
global LOG
import logging
warnings.filterwarnings('ignore')
LOG = logging.getLogger()
@ -36,7 +35,7 @@ logging.getLogger("urllib3").setLevel(logging.INFO)
# https://github.com/erans/torcontactinfoparser
# sys.path.append('/home/....')
try:
from torcontactinfo import TorContactInfoParser
from exclude_badExits.torcontactinfo import TorContactInfoParser
except:
TorContactInfoParser = None
@ -266,7 +265,10 @@ def oDownloadUrlRequests(uri, sCAfile: str, timeout: int = 30, host: str = '127.
return oReqResp
# There's no point in using asyncio because of duplicate urls in the tasks
async def oDownloadUrlHttpx(uri:str, sCAfile:str, timeout;int = 30, host:str = '127.0.0.1', port:int = 9050, content_type:str = 'text/plain'):
async def oDownloadUrlHttpx(uri:str, sCAfile:str, timeout:int = 30,
host:str = '127.0.0.1', port:int = 9050,
content_type:str = 'text/plain'):
import httpcore
import asyncio
import httpx