From db070348b565a3829738d9a7de253c5342275e38 Mon Sep 17 00:00:00 2001 From: "emdee@spm.plastiras.org" Date: Mon, 5 Feb 2024 13:11:36 +0000 Subject: [PATCH] update --- LICENSE | 16 + Makefile | 10 +- docs/toktok.ltd/spec.html | 8947 +++++++++++++++++ pyproject.toml | 16 +- requirements.txt | 6 + setup.cfg | 14 +- src/toxygen_wrapper/__init__.py | 7 + src/toxygen_wrapper/libtox.py | 87 + src/toxygen_wrapper/tests/__init__.py | 0 src/toxygen_wrapper/tests/socks.py | 391 + src/toxygen_wrapper/tests/support_http.py | 163 + src/toxygen_wrapper/tests/support_onions.py | 573 ++ src/toxygen_wrapper/tests/support_testing.py | 1020 ++ src/toxygen_wrapper/tests/tests_wrapper.py | 2284 +++++ src/toxygen_wrapper/tox.py | 3380 +++++++ src/toxygen_wrapper/toxav.py | 409 + src/toxygen_wrapper/toxav_enums.py | 133 + .../toxcore_enums_and_consts.py | 982 ++ src/toxygen_wrapper/toxencryptsave.py | 91 + .../toxencryptsave_enums_and_consts.py | 29 + src/toxygen_wrapper/toxygen_echo.py | 458 + tox_profile.py | 1471 +++ tox_profile_examples.bash | 24 + tox_profile_test.bash | 337 + 24 files changed, 20823 insertions(+), 25 deletions(-) create mode 100644 LICENSE create mode 100644 docs/toktok.ltd/spec.html create mode 100644 requirements.txt create mode 100644 src/toxygen_wrapper/__init__.py create mode 100644 src/toxygen_wrapper/libtox.py create mode 100644 src/toxygen_wrapper/tests/__init__.py create mode 100644 src/toxygen_wrapper/tests/socks.py create mode 100644 src/toxygen_wrapper/tests/support_http.py create mode 100644 src/toxygen_wrapper/tests/support_onions.py create mode 100644 src/toxygen_wrapper/tests/support_testing.py create mode 100644 src/toxygen_wrapper/tests/tests_wrapper.py create mode 100644 src/toxygen_wrapper/tox.py create mode 100644 src/toxygen_wrapper/toxav.py create mode 100644 src/toxygen_wrapper/toxav_enums.py create mode 100644 src/toxygen_wrapper/toxcore_enums_and_consts.py create mode 100644 src/toxygen_wrapper/toxencryptsave.py create mode 100644 src/toxygen_wrapper/toxencryptsave_enums_and_consts.py create mode 100644 src/toxygen_wrapper/toxygen_echo.py create mode 100644 tox_profile.py create mode 100644 tox_profile_examples.bash create mode 100755 tox_profile_test.bash diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e06aa93 --- /dev/null +++ b/LICENSE @@ -0,0 +1,16 @@ +Copyright (c) year copyright holder. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. +Redistribution of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. +Redistribution in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. +Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +YOU ACKNOWLEDGE THAT THIS SOFTWARE IS NOT DESIGNED, LICENSED OR INTENDED FOR USE IN THE DESIGN, CONSTRUCTION, OPERATION OR MAINTENANCE OF ANY MILITARY FACILITY. diff --git a/Makefile b/Makefile index b6920eb..7b33ce7 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,10 @@ PREFIX=/usr/local -PYTHON=python3.sh -PIP=pip3.sh +PYTHON_EXE_MSYS=${PREFIX}/bin/python3.sh +PIP_EXE_MSYS=${PREFIX}/bin/pip3.sh + iTEST_TIMEOUT=60 fSOCKET_TIMEOUT=15.0 +PYTHON_MINOR=`python3 --version 2>&1 | sed -e 's@^.* @@' -e 's@\.[0-9]*$$@@'` prepare:: bash .pylint.sh @@ -13,7 +15,9 @@ check:: > .pyanal.out 2>&1 install:: - $(PIP) install --target $PREFIX/lib/python3.11/site-packages --upgrade . + ${PIP_EXE_MSYS} --python ${PYTHON_EXE_MSYS} install \ + --target ${PREFIX}/lib/python${PYTHON_MINOR}/site-packages/ \ + --upgrade . rsync:: bash .rsync.sh diff --git a/docs/toktok.ltd/spec.html b/docs/toktok.ltd/spec.html new file mode 100644 index 0000000..1952bbc --- /dev/null +++ b/docs/toktok.ltd/spec.html @@ -0,0 +1,8947 @@ + + + + + + + + The TokTok Project - Protocol + + + + + + + + + + + + + + + +
+
+ + + +

Introduction

+ +

This document is a textual specification of the Tox protocol and all the +supporting modules required to implement it. The goal of this document +is to give enough guidance to permit a complete and correct +implementation of the protocol.

+ +

Objectives

+ +

This section provides an overview of goals and non-goals of Tox. It +provides the reader with:

+ +
    +
  • +

    a basic understanding of what problems Tox intends to solve;

    +
    • +
  • +

    a means to validate whether those problems are indeed solved by the +protocol as specified;

    +
    • +
  • +

    the ability to make better tradeoffs and decisions in their own +reimplementation of the protocol.

    +
    • +
+ +

Goals

+ +
    +
  • +

    Authentication: Tox aims to provide authenticated communication. +This means that during a communication session, both parties can be +sure of the other party’s identity. Users are identified by their +public key. The initial key exchange is currently not in scope for +the Tox protocol. In the future, Tox may provide a means for initial +authentication using a challenge/response or shared secret based +exchange.

    + +

    If the secret key is compromised, the user’s identity is +compromised, and an attacker can impersonate that user. When this +happens, the user must create a new identity with a new public key.

    +
    • +
  • +

    End-to-end encryption: The Tox protocol establishes end-to-end +encrypted communication links. Shared keys are deterministically +derived using a Diffie-Hellman-like method, so keys are never +transferred over the network.

    +
    • +
  • +

    Forward secrecy: Session keys are re-negotiated when the peer +connection is established.

    +
    • +
  • +

    Privacy: When Tox establishes a communication link, it aims to +avoid leaking to any third party the identities of the parties +involved (i.e. their public keys).

    + +

    Furthermore, it aims to avoid allowing third parties to determine +the IP address of a given user.

    +
    • +
  • +

    Resilience:

    + +
      +
    • +

      Independence of infrastructure: Tox avoids relying on servers as +much as possible. Communications are not transmitted via or +stored on central servers. Joining a Tox network requires +connecting to a well-known node called a bootstrap node. Anyone +can run a bootstrap node, and users need not put any trust in +them.

      +
      • +
    • +

      Tox tries to establish communication paths in difficult network +situations. This includes connecting to peers behind a NAT or +firewall. Various techniques help achieve this, such as UDP +hole-punching, UPnP, NAT-PMP, other untrusted nodes acting as +relays, and DNS tunnels.

      +
      • +
    • +

      Resistance to basic denial of service attacks: short timeouts +make the network dynamic and resilient against poisoning +attempts.

      +
      • +
    +
    • +
  • +

    Minimum configuration: Tox aims to be nearly zero-conf. +User-friendliness is an important aspect to security. Tox aims to +make security easy to achieve for average users.

    +
    • +
+ +

Non-goals

+ +
    +
  • +

    Anonymity is not in scope for the Tox protocol itself, but it +provides an easy way to integrate with software providing anonymity, +such as Tor.

    + +

    By default, Tox tries to establish direct connections between peers; +as a consequence, each is aware of the other’s IP address, and third +parties may be able to determine that a connection has been +established between those IP addresses. One of the reasons for +making direct connections is that relaying real-time multimedia +conversations over anonymity networks is not feasible with the +current network infrastructure.

    +
    • +
+ +

Threat model

+ +

TODO(iphydf): Define one.

+ +

Data types

+ +

All data types are defined before their first use, and their binary +protocol representation is given. The protocol representations are +normative and must be implemented exactly as specified. For some types, +human-readable representations are suggested. An implementation may +choose to provide no such representation or a different one. The +implementation is free to choose any in-memory representation of the +specified types.

+ +

Binary formats are specified in tables with length, type, and content +descriptions. If applicable, specific enumeration types are used, so +types may be self-explanatory in some cases. The length can be either a +fixed number in bytes (e.g. 32), a number in bits (e.g. 7 bit), a +choice of lengths (e.g. 4 | 16), or an inclusive range (e.g. +[0, 100]). Open ranges are denoted [n,] to mean a minimum length of +n with no specified maximum length.

+ +

Integers

+ +

The protocol uses four bounded unsigned integer types. Bounded means +they have an upper bound beyond which incrementing is not defined. The +integer types support modular arithmetic, so overflow wraps around to +zero. Unsigned means their lower bound is 0. Signed integer types are +not used. The binary encoding of all integer types is a fixed-width byte +sequence with the integer encoded in Big +Endian unless stated +otherwise.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Type nameC typeLengthUpper bound
Word8uint8_t1255 (0xff)
Word16uint16_t265535 (0xffff)
Word32uint32_t44294967295 (0xffffffff)
Word64uint64_t818446744073709551615 (0xffffffffffffffff)
+ +

Strings

+ +

A String is a data structure used for human readable text. Strings are +sequences of glyphs. A glyph consists of one non-zero-width unicode code +point and zero or more zero-width unicode code points. The +human-readable representation of a String starts and ends with a +quotation mark (") and contains all human-readable glyphs verbatim. +Control characters are represented in an isomorphic human-readable way. +I.e. every control character has exactly one human-readable +representation, and a mapping exists from the human-readable +representation to the control character. Therefore, the use of Unicode +Control Characters (U+240x) is not permitted without additional marker.

+ +

Crypto

+ +

The Crypto module contains all the functions and data types related to +cryptography. This includes random number generation, encryption and +decryption, key generation, operations on nonces and generating random +nonces.

+ +

Key

+ +

A Crypto Number is a large fixed size unsigned (non-negative) integer. +Its binary encoding is as a Big Endian integer in exactly the encoded +byte size. Its human-readable encoding is as a base-16 number encoded as +String. The NaCl implementation +libsodium supplies the +functions sodium_bin2hex and sodium_hex2bin to aid in implementing +the human-readable encoding. The in-memory encoding of these crypto +numbers in NaCl already satisfies the binary encoding, so for +applications directly using those APIs, binary encoding and decoding is +the identity +function.

+ +

Tox uses four kinds of Crypto Numbers:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeBitsEncoded byte size
Public Key25632
Secret Key25632
Combined Key25632
Nonce19224
+ +

Key Pair

+ +

A Key Pair is a pair of Secret Key and Public Key. A new key pair is +generated using the crypto_box_keypair function of the NaCl crypto +library. Two separate calls to the key pair generation function must +return distinct key pairs. See the NaCl +documentation for details.

+ +

A Public Key can be computed from a Secret Key using the NaCl function +crypto_scalarmult_base, which computes the scalar product of a +standard group element and the Secret Key. See the NaCl +documentation for details.

+ +

Combined Key

+ +

A Combined Key is computed from a Secret Key and a Public Key using the +NaCl function crypto_box_beforenm. Given two Key Pairs KP1 (SK1, PK1) +and KP2 (SK2, PK2), the Combined Key computed from (SK1, PK2) equals the +one computed from (SK2, PK1). This allows for symmetric encryption, as +peers can derive the same shared key from their own secret key and their +peer’s public key.

+ +

In the Tox protocol, packets are encrypted using the public key of the +receiver and the secret key of the sender. The receiver decrypts the +packets using the receiver’s secret key and the sender’s public key.

+ +

The fact that the same key is used to encrypt and decrypt packets on +both sides means that packets being sent could be replayed back to the +sender if there is nothing to prevent it.

+ +

The shared key generation is the most resource intensive part of the +encryption/decryption which means that resource usage can be reduced +considerably by saving the shared keys and reusing them later as much as +possible.

+ +

Nonce

+ +

A random nonce is generated using the cryptographically secure random +number generator from the NaCl library randombytes.

+ +

A nonce is incremented by interpreting it as a Big Endian number and +adding 1. If the nonce has the maximum value, the value after the +increment is 0.

+ +

Most parts of the protocol use random nonces. This prevents new nonces +from being associated with previous nonces. If many different packets +could be tied together due to how the nonces were generated, it might +for example lead to tying DHT and onion announce packets together. This +would introduce a flaw in the system as non friends could tie some +people’s DHT keys and long term keys together.

+ +

Box

+ +

The Tox protocol differentiates between two types of text: Plain Text +and Cipher Text. Cipher Text may be transmitted over untrusted data +channels. Plain Text can be Sensitive or Non Sensitive. Sensitive Plain +Text must be transformed into Cipher Text using the encryption function +before it can be transmitted over untrusted data channels.

+ +

The encryption function takes a Combined Key, a Nonce, and a Plain Text, +and returns a Cipher Text. It uses crypto_box_afternm to perform the +encryption. The meaning of the sentence “encrypting with a secret key, a +public key, and a nonce” is: compute a combined key from the secret key +and the public key and then use the encryption function for the +transformation.

+ +

The decryption function takes a Combined Key, a Nonce, and a Cipher +Text, and returns either a Plain Text or an error. It uses +crypto_box_open_afternm from the NaCl library. Since the cipher is +symmetric, the encryption function can also perform decryption, but will +not perform message authentication, so the implementation must be +careful to use the correct functions.

+ +

crypto_box uses xsalsa20 symmetric encryption and poly1305 +authentication.

+ +

The create and handle request functions are the encrypt and decrypt +functions for a type of DHT packets used to send data directly to other +DHT nodes. To be honest they should probably be in the DHT module but +they seem to fit better here. TODO: What exactly are these functions?

+ +

Node Info

+ +

Transport Protocol

+ +

A Transport Protocol is a transport layer protocol directly below the +Tox protocol itself. Tox supports two transport protocols: UDP and TCP. +The binary representation of the Transport Protocol is a single bit: 0 +for UDP, 1 for TCP. If encoded as standalone value, the bit is stored in +the least significant bit of a byte. If followed by other bit-packed +data, it consumes exactly one bit.

+ +

The human-readable representation for UDP is UDP and for TCP is TCP.

+ +

Host Address

+ +

A Host Address is either an IPv4 or an IPv6 address. The binary +representation of an IPv4 address is a Big Endian 32 bit unsigned +integer (4 bytes). For an IPv6 address, it is a Big Endian 128 bit +unsigned integer (16 bytes). The binary representation of a Host Address +is a 7 bit unsigned integer specifying the address family (2 for IPv4, +10 for IPv6), followed by the address itself.

+ +

Thus, when packed together with the Transport Protocol, the first bit of +the packed byte is the protocol and the next 7 bits are the address +family.

+ +

Port Number

+ +

A Port Number is a 16 bit number. Its binary representation is a Big +Endian 16 bit unsigned integer (2 bytes).

+ +

Socket Address

+ +

A Socket Address is a pair of Host Address and Port Number. Together +with a Transport Protocol, it is sufficient information to address a +network port on any internet host.

+ +

Node Info (packed node format)

+ +

The Node Info data structure contains a Transport Protocol, a Socket +Address, and a Public Key. This is sufficient information to start +communicating with that node. The binary representation of a Node Info +is called the “packed node format”.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthTypeContents
1 bitTransport ProtocolUDP = 0, TCP = 1
7 bitAddress Family2 = IPv4, 10 = IPv6
4 \| 16IP address4 bytes for IPv4, 16 bytes for IPv6
2Port NumberPort number
32Public KeyNode ID
+ +

The packed node format is a way to store the node info in a small yet +easy to parse format. To store more than one node, simply append another +one to the previous one: [packed node 1][packed node 2][...].

+ +

In the packed node format, the first byte (high bit protocol, lower 7 +bits address family) are called the IP Type. The following table is +informative and can be used to simplify the implementation.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IP TypeTransport ProtocolAddress Family
2 (0x02)UDPIPv4
10 (0x0a)UDPIPv6
130 (0x82)TCPIPv4
138 (0x8a)TCPIPv6
+ +

The number 130 is used for an IPv4 TCP relay and 138 is used to +indicate an IPv6 TCP relay.

+ +

The reason for these numbers is that the numbers on Linux for IPv4 and +IPv6 (the AF_INET and AF_INET6 defines) are 2 and 10. The TCP +numbers are just the UDP numbers + 128.

+ +

Protocol Packet

+ +

A Protocol Packet is the top level Tox protocol element. All other +packet types are wrapped in Protocol Packets. It consists of a Packet +Kind and a payload. The binary representation of a Packet Kind is a +single byte (8 bits). The payload is an arbitrary sequence of bytes.

+ + + + + + + + + + + + + + + + + + + + + +
LengthTypeContents
1Packet KindThe packet kind identifier
[0,]BytesPayload
+ +

These top level packets can be transported in a number of ways, the most +common way being over the network using UDP or TCP. The protocol itself +does not prescribe transport methods, and an implementation is free to +implement additional transports such as WebRTC, IRC, or pipes.

+ +

In the remainder of the document, different kinds of Protocol Packet are +specified with their packet kind and payload. The packet kind is not +repeated in the payload description (TODO: actually it mostly is, but +later it won’t).

+ +

Inside Protocol Packets payload, other packet types can specify +additional packet kinds. E.g. inside a Crypto Data packet (0x1b), the +Messenger module defines its protocols for messaging, file +transfers, etc. Top level Protocol Packets are themselves not encrypted, +though their payload may be.

+ +

Packet Kind

+ +

The following is an exhaustive list of top level packet kind names and +their number. Their payload is specified in dedicated sections. Each +section is named after the Packet Kind it describes followed by the byte +value in parentheses, e.g. Ping Request (0x00).

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Byte valuePacket Kind
0x00Ping Request
0x01Ping Response
0x02Nodes Request
0x04Nodes Response
0x18Cookie Request
0x19Cookie Response
0x1aCrypto Handshake
0x1bCrypto Data
0x20DHT Request
0x21LAN Discovery
0x80Onion Request 0
0x81Onion Request 1
0x82Onion Request 2
0x83Announce Request
0x84Announce Response
0x85Onion Data Request
0x86Onion Data Response
0x8cOnion Response 3
0x8dOnion Response 2
0x8eOnion Response 1
0xf0Bootstrap Info
+ +

DHT

+ +

The DHT is a self-organizing swarm of all nodes in the Tox network. A +node in the Tox network is also called a “Tox node”. When we talk about +“peers”, we mean any node that is not the local node (the subject). This +module takes care of finding the IP and port of nodes and establishing a +route to them directly via UDP using hole punching if +necessary. The DHT only runs on UDP and so is only used if UDP works.

+ +

Every node in the Tox DHT has an ephemeral Key Pair called the DHT Key +Pair, consisting of the DHT Secret Key and the DHT Public Key. The DHT +Public Key acts as the node address. The DHT Key Pair is renewed every +time the Tox instance is closed or restarted. An implementation may +choose to renew the key more often, but doing so will disconnect all +peers.

+ +

The DHT public key of a friend is found using the onion +module. Once the DHT public key of a friend is known, the DHT is used to +find them and connect directly to them via UDP.

+ +

Distance

+ +

A Distance is a positive integer. Its human-readable representation is a +base-16 number. Distance (type) is an ordered +monoid with the +associative binary operator + and the identity element 0.

+ +

The DHT uses a +metric to +determine the distance between two nodes. The Distance type is the +co-domain of this metric. The metric currently used by the Tox DHT is +the XOR of the nodes’ public keys: distance(x, y) = x XOR y. For +this computation, public keys are interpreted as Big Endian integers +(see Crypto Numbers).

+ +

When we speak of a “close node”, we mean that its Distance to the node +under consideration is small compared to the Distance to other nodes.

+ +

An implementation is not required to provide a Distance type, so it has +no specified binary representation. For example, instead of computing a +distance and comparing it against another distance, the implementation +can choose to implement Distance as a pair of public keys and define an +ordering on Distance without computing the complete integral value. This +works, because as soon as an ordering decision can be made in the most +significant bits, further bits won’t influence that decision.

+ +

XOR is a valid metric, i.e. it satisfies the required conditions:

+ +
    +
  1. +

    Non-negativity distance(x, y) >= 0: Since public keys are Crypto +Numbers, which are by definition non-negative, their XOR is +necessarily non-negative.

    +
    2. +
  2. +

    Identity of indiscernibles distance(x, y) == 0 iff x == y: The +XOR of two integers is zero iff they are equal.

    +
    3. +
  3. +

    Symmetry distance(x, y) == distance(y, x): XOR is a symmetric +operation.

    +
    4. +
  4. +

    Subadditivity distance(x, z) <= distance(x, y) + distance(y, z): +follows from associativity, since x XOR z = x XOR (y XOR y) XOR z = +distance(x, y) XOR distance(y, z) which is not greater than +distance(x, y) + distance(y, z).

    +
    5. +
+ +

In addition, XOR has other useful properties:

+ +
    +
  • +

    Unidirectionality: given the key x and the distance d there +exist one and only one key y such that distance(x, y) = d.

    + +

    The implication is that repeated lookups are likely to pass along +the same way and thus caching makes sense.

    + +

    Source: +maymounkov-kademlia

    +
    • +
+ + + +

Example: Given three nodes with keys 2, 5, and 6:

+ +
    +
  • +

    2 XOR 5 = 7

    +
    • +
  • +

    2 XOR 6 = 4

    +
    • +
  • +

    5 XOR 2 = 7

    +
    • +
  • +

    5 XOR 6 = 3

    +
    • +
  • +

    6 XOR 2 = 4

    +
    • +
  • +

    6 XOR 5 = 3

    +
    • +
+ +

The closest node from both 2 and 5 is 6. The closest node from 6 is 5 +with distance 3. This example shows that a key that is close in terms of +integer addition may not necessarily be close in terms of XOR.

+ +

Client Lists

+ +

A Client List of maximum size k with a given public key as base +key is an ordered set of at most k nodes close to the base key. The +elements are sorted by distance from the base key. Thus, +the first (smallest) element of the set is the closest one to the base +key in that set, the last (greatest) element is the furthest away. The +maximum size and base key are constant throughout the lifetime of a +Client List.

+ +

A Client List is full when the number of nodes it contains is the +maximum size of the list.

+ +

A node is viable for entry if the Client List is not full or the +node’s public key has a lower distance from the base key than the +current entry with the greatest distance.

+ +

If a node is viable and the Client List is full, the entry with the +greatest distance from the base key is removed to keep the size below +the maximum configured size.

+ +

Adding a node whose key already exists will result in an update of the +Node Info in the Client List. Removing a node for which no Node Info +exists in the Client List has no effect. Thus, removing a node twice is +permitted and has the same effect as removing it once.

+ +

The iteration order of a Client List is in order of distance from the +base key. I.e. the first node seen in iteration is the closest, and the +last node is the furthest away in terms of the distance metric.

+ +

K-buckets

+ +

K-buckets is a data structure for efficiently storing a set of nodes +close to a certain key called the base key. The base key is constant +throughout the lifetime of a k-buckets instance.

+ +

A k-buckets is a map from small integers 0 <= n < 256 to Client Lists +of maximum size (k). Each Client List is called a (k-)bucket. A +k-buckets is equipped with a base key, and each bucket has this key as +its base key. k is called the bucket size. The default bucket size is

+
    +
  1. A large bucket size was chosen to increase the speed at which peers +are found.
    2. +
+ +

The above number n is the bucket index. It is a non-negative integer +with the range [0, 255], i.e. the range of an 8 bit unsigned integer.

+ +

Bucket Index

+ +

The index of the bucket can be computed using the following function: +bucketIndex(baseKey, nodeKey) = 255 - log_2(distance(baseKey, +nodeKey)). This function is not defined when baseKey == nodeKey, +meaning k-buckets will never contain a Node Info about the base node.

+ +

Thus, each k-bucket contains only Node Infos for whose keys the +following holds: if node with key nodeKey is in k-bucket with index +n, then bucketIndex(baseKey, nodeKey) == n. Thus, n’th k-bucket +consists of nodes for which distance to the base node lies in range +[2^n, 2^(n+1) - 1].

+ +

The bucket index can be efficiently computed by determining the first +bit at which the two keys differ, starting from the most significant +bit. So, if the local DHT key starts with e.g. 0x80 and the bucketed +node key starts with 0x40, then the bucket index for that node is 0. +If the second bit differs, the bucket index is 1. If the keys are almost +exactly equal and only the last bit differs, the bucket index is 255.

+ +

Manipulating k-buckets

+ +

TODO: this is different from kademlia’s least-recently-seen eviction +policy; why the existing solution was chosen, how does it affect +security, performance and resistance to poisoning? original paper claims +that preference of old live nodes results in better persistence and +resistance to basic DDoS attacks;

+ +

Any update or lookup operation on a k-buckets instance that involves a +single node requires us to first compute the bucket index for that node. +An update involving a Node Info with nodeKey == baseKey has no effect. +If the update results in an empty bucket, that bucket is removed from +the map.

+ +

Adding a node to, or removing a node from, a k-buckets consists of +performing the corresponding operation on the Client List bucket whose +index is that of the node’s public key, except that adding a new node to +a full bucket has no effect. A node is considered viable for entry if +the corresponding bucket is not full.

+ +

Iteration order of a k-buckets instance is in order of distance from the +base key. I.e. the first node seen in iteration is the closest, and the +last node is the furthest away in terms of the distance metric.

+ +

DHT node state

+ +

Every DHT node contains the following state:

+ +
    +
  • +

    DHT Key Pair: The Key Pair used to communicate with other DHT nodes. +It is immutable throughout the lifetime of the DHT node.

    +
    • +
  • +

    DHT Close List: A set of Node Infos of nodes that are close to the +DHT Public Key (public part of the DHT Key Pair). The Close List is +represented as a k-buckets data structure, with the +DHT Public Key as the Base Key.

    +
    • +
  • +

    DHT Search List: A list of Public Keys of nodes that the DHT node is +searching for, associated with a DHT Search Entry.

    +
    • +
+ + + +

A DHT node state is initialised using a Key Pair, which is stored in the +state as DHT Key Pair and as base key for the Close List. Both the Close +and Search Lists are initialised to be empty.

+ +

DHT Search Entry

+ +

A DHT Search Entry contains a Client List with base key the searched +node’s Public Key. Once the searched node is found, it is also stored in +the Search Entry.

+ +

The maximum size of the Client List is set to 8. (Must be the same or +smaller than the bucket size of the close list to make sure all the +closest peers found will know the node being searched (TODO(zugz): this +argument is unclear.)).

+ +

A DHT node state therefore contains one Client List for each bucket +index in the Close List, and one Client List for each DHT Search Entry. +These lists are not required to be disjoint - a node may be in multiple +Client Lists simultaneously.

+ +

A Search Entry is initialised with the searched-for Public Key. The +contained Client List is initialised to be empty.

+ +

Manipulating the DHT node state

+ +

Adding a search key to the DHT node state creates an empty entry in the +Search Nodes list. If a search entry for the public key already existed, +the “add” operation has no effect.

+ +

Removing a search key removes its search entry and all associated data +structures from memory.

+ +

The Close List and the Search Entries are termed the Node Lists of the +DHT State.

+ +

The iteration order over the DHT state is to first process the Close +List k-buckets, then the Search List entry Client Lists. Each of these +follows the iteration order in the corresponding specification.

+ +

A node info is considered to be contained in the DHT State if it is +contained in the Close List or in at least one of the Search Entries.

+ +

The size of the DHT state is defined to be the number of node infos it +contains, counted with multiplicity: node infos contained multiple +times, e.g. in the close list and in various search entries, are counted +as many times as they appear. Search keys do not directly count towards +the state size. So the size of the state is the sum of the sizes of the +Close List and the Search Entries.

+ +

The state size is relevant to later pruning algorithms that decide when +to remove a node info and when to request a ping from stale nodes. +Search keys, once added, are never automatically pruned.

+ +

Adding a Node Info to the state is done by adding the node to each Node +List in the state.

+ +

When adding a node info to the state, the search entry for the node’s +public key, if it exists, is updated to contain the new node info. All +k-buckets and Client Lists that already contain the node info will also +be updated. See the corresponding specifications for the update +algorithms. However, a node info will not be added to a search entry +when it is the node to which the search entry is associated (i.e. the +node being search for).

+ +

Removing a node info from the state removes it from all k-buckets. If a +search entry for the removed node’s public key existed, the node info in +that search entry is unset. The search entry itself is not removed.

+ +

Self-organisation

+ +

Self-organising in the DHT occurs through each DHT peer connecting to an +arbitrary number of peers closest to their own DHT public key and some +that are further away.

+ +

If each peer in the network knows the peers with the DHT public key +closest to its DHT public key, then to find a specific peer with public +key X a peer just needs to recursively ask peers in the DHT for known +peers that have the DHT public keys closest to X. Eventually the peer +will find the peers in the DHT that are the closest to that peer and, if +that peer is online, they will find them.

+ +

DHT Packet

+ +

The DHT Packet contains the sender’s DHT Public Key, an encryption +Nonce, and an encrypted payload. The payload is encrypted with the DHT +secret key of the sender, the DHT public key of the receiver, and the +nonce that is sent along with the packet. DHT Packets are sent inside +Protocol Packets with a varying Packet Kind.

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthTypeContents
32Public KeySender DHT Public Key
24NonceRandom nonce
[16,]BytesEncrypted payload
+ +

The encrypted payload is at least 16 bytes long, because the encryption +includes a +MAC of 16 +bytes. A 16 byte payload would thus be the empty message. The DHT +protocol never actually sends empty messages, so in reality the minimum +size is 27 bytes for the Ping Packet.

+ +

RPC Services

+ +

A DHT RPC Service consists of a Request packet and a Response packet. A +DHT RPC Packet contains a payload and a Request ID. This ID is a 64 bit +unsigned integer that helps identify the response for a given request.

+ +

Replies to RPC requests

+ +

A reply to a Request packet is a Response packet with the Request ID +in the Response packet set equal to the Request ID in the Request +packet. A response is accepted if and only if it is the first received +reply to a request which was sent sufficiently recently, according to a +time limit which depends on the service.

+ +

DHT RPC Packets are encrypted and transported within DHT Packets.

+ + + + + + + + + + + + + + + + + + + + + +
LengthTypeContents
[0,]BytesPayload
8uint64_tRequest ID
+ +

The minimum payload size is 0, but in reality the smallest sensible +payload size is 1. Since the same symmetric key is used in both +communication directions, an encrypted Request would be a valid +encrypted Response if they contained the same plaintext.

+ +

Parts of the protocol using RPC packets must take care to make Request +payloads not be valid Response payloads. For instance, Ping +Packets carry a boolean flag that indicate whether the +payload corresponds to a Request or a Response.

+ +

The Request ID provides some resistance against replay attacks. If there +were no Request ID, it would be easy for an attacker to replay old +responses and thus provide nodes with out-of-date information. A Request +ID should be randomly generated for each Request which is sent.

+ +

Ping Service

+ +

The Ping Service is used to check if a node is responsive.

+ +

A Ping Packet payload consists of just a boolean value saying whether it +is a request or a response.

+ +

The one byte boolean inside the encrypted payload is added to prevent +peers from creating a valid Ping Response from a Ping Request without +decrypting the packet and encrypting a new one. Since symmetric +encryption is used, the encrypted Ping Response would be byte-wise equal +to the Ping Request without the discriminator byte.

+ + + + + + + + + + + + + + + + +
LengthTypeContents
1BoolResponse flag: 0x00 for Request, 0x01 for Response
+ +
Ping Request (0x00)
+ +

A Ping Request is a Ping Packet with the response flag set to False. +When a Ping Request is received and successfully decrypted, a Ping +Response packet is created and sent back to the requestor.

+ +
Ping Response (0x01)
+ +

A Ping Response is a Ping Packet with the response flag set to True.

+ +

Nodes Service

+ +

The Nodes Service is used to query another DHT node for up to 4 nodes +they know that are the closest to a requested node.

+ +

The DHT Nodes RPC service uses the Packed Node Format.

+ +

Only the UDP Protocol (IP Type 2 and 10) is used in the DHT module +when sending nodes with the packed node format. This is because the TCP +Protocol is used to send TCP relay information and the DHT is UDP only.

+ +
Nodes Request (0x02)
+ + + + + + + + + + + + + + + + +
LengthTypeContents
32Public KeyRequested DHT Public Key
+ +

The DHT Public Key sent in the request is the one the sender is +searching for.

+ +
Nodes Response (0x04)
+ + + + + + + + + + + + + + + + + + + + + +
LengthTypeContents
1IntNumber of nodes in the response (maximum 4)
[39, 204]Node InfosNodes in Packed Node Format
+ +

An IPv4 node is 39 bytes, an IPv6 node is 51 bytes, so the maximum size +of the packed Node Infos is 51 * 4 = 204 bytes.

+ +

Nodes responses should contain the 4 closest nodes that the sender of +the response has in their lists of known nodes.

+ +

DHT Operation

+ +

DHT Initialisation

+ +

A new DHT node is initialised with a DHT State with a fresh random key +pair, an empty close list, and a search list containing 2 empty search +entries searching for the public keys of fresh random key pairs.

+ +

Periodic sending of Nodes Requests

+ +

For each Nodes List in the DHT State, every 20 seconds we send a Nodes +Request to a random node on the list, searching for the base key of the +list.

+ +

When a Nodes List first becomes populated with nodes, we send 5 such +random Nodes Requests in quick succession.

+ +

Random nodes are chosen since being able to predict which node a node +will send a request to next could make some attacks that disrupt the +network easier, as it adds a possible attack vector.

+ +

Furthermore, we periodically check every node for responsiveness by +sending it a Nodes Request: for each Nodes List in the DHT State, we +send each node on the list a Nodes Request every 60 seconds, searching +for the base key of the list. We remove from the DHT State any node from +which we persistently fail to receive Nodes Responses.

+ +

c-toxcore’s implementation of checking and timeouts: A Last Checked time +is maintained for each node in each list. When a node is added to a +list, if doing so evicts a node from the list then the Last Checked time +is set to that of the evicted node, and otherwise it is set to 0. This +includes updating an already present node. Nodes from which we have not +received a Nodes Response for 122 seconds are considered Bad; they +remain in the DHT State, but are preferentially overwritten when adding +to the DHT State, and are ignored for all operations except the +once-per-60s checking described above. If we have not received a Nodes +Response for 182 seconds, the node is not even checked. So one check is +sent after the node becomes Bad. In the special case that every node in +the Close List is Bad, they are all checked once more.)

+ +

hs-toxcore implementation of checking and timeouts: We maintain a Last +Checked timestamp and a Checks Counter on each node on each Nodes List +in the Dht State. When a node is added to a list, these are set +respectively to the current time and to 0. This includes updating an +already present node. We periodically pass through the nodes on the +lists, and for each which is due a check, we: check it, update the +timestamp, increment the counter, and, if the counter is then 2, remove +the node from the list. This is pretty close to the behaviour of +c-toxcore, but much simpler. TODO: currently hs-toxcore doesn’t do +anything to try to recover if the Close List becomes empty. We could +maintain a separate list of the most recently heard from nodes, and +repopulate the Close List with that if the Close List becomes empty.

+ +

Handling Nodes Response packets

+ +

When we receive a valid Nodes Response packet, we first check that it is +a reply to a Nodes Request which we sent within the last 60 seconds to +the node from which we received the response, and that no previous reply +has been received. If this check fails, the packet is ignored. If the +check succeeds, first we add to the DHT State the node from which the +response was sent. Then, for each node listed in the response and for +each Nodes List in the DHT State which does not currently contain the +node and to which the node is viable for entry, we send a Nodes Request +to the node with the requested public key being the base key of the +Nodes List.

+ +

An implementation may choose not to send every such Nodes Request. +(c-toxcore only sends so many per list (8 for the Close List, 4 for a +Search Entry) per 50ms, prioritising the closest to the base key).

+ +

Handling Nodes Request packets

+ +

When we receive a Nodes Request packet from another node, we reply with +a Nodes Response packet containing the 4 nodes in the DHT State which +are the closest to the public key in the packet. If there are fewer than +4 nodes in the state, we reply with all the nodes in the state. If there +are no nodes in the state, no reply is sent.

+ +

We also send a Ping Request when this is appropriate; see below.

+ +

Handling Ping Request packets

+ +

When a valid Ping Request packet is received, we reply with a Ping +Response.

+ +

We also send a Ping Request when this is appropriate; see below.

+ +

Handling Ping Response packets

+ +

When we receive a valid Ping Response packet, we first check that it is +a reply to a Ping Request which we sent within the last 5 seconds to the +node from which we received the response, and that no previous reply has +been received. If this check fails, the packet is ignored. If the check +succeeds, we add to the DHT State the node from which the response was +sent.

+ +

Sending Ping Requests

+ +

When we receive a Nodes Request or a Ping Request, in addition to the +handling described above, we sometimes send a Ping Request. Namely, we +send a Ping Request to the node which sent the packet if the node is +viable for entry to the Close List and is not already in the Close List. +An implementation may (TODO: should?) choose not to send every such Ping +Request. (c-toxcore sends at most 32 every 2 seconds, preferring closer +nodes.)

+ +

DHT Request Packets

+ +

DHT Request packets are used to route encrypted data from a sender to +another node, referred to as the addressee of the packet, via a third +node.

+ +

A DHT Request Packet is sent as the payload of a Protocol Packet with +the corresponding Packet Kind. It contains the DHT Public Key of an +addressee, and a DHT Packet which is to be received by the addressee.

+ + + + + + + + + + + + + + + + + + + + + +
LengthTypeContents
32Public KeyAddressee DHT Public Key
[72,]DHT PacketDHT Packet
+ +

Handling DHT Request packets

+ +

A DHT node that receives a DHT request packet checks whether the +addressee public key is their DHT public key. If it is, they will +decrypt and handle the packet. Otherwise, they will check whether the +addressee DHT public key is the DHT public key of one of the nodes in +their Close List. If it isn’t, they will drop the packet. If it is they +will resend the packet, unaltered, to that DHT node.

+ +

DHT request packets are used for DHT public key packets (see +onion) and NAT ping packets.

+ +

NAT ping packets

+ +

A NAT ping packet is sent as the payload of a DHT request packet.

+ +

We use NAT ping packets to see if a friend we are not connected to +directly is online and ready to do the hole punching.

+ +
NAT ping request
+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0xfe)
1uint8_t (0x00)
8uint64_t random number
+ +
NAT ping response
+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0xfe)
1uint8_t (0x01)
8uint64_t random number (the same that was received in request)
+ +

TODO: handling these packets.

+ +

Effects of chosen constants on performance

+ +

If the bucket size of the k-buckets were increased, it would increase +the amount of packets needed to check if each node is still alive, which +would increase the bandwidth usage, but reliability would go up. If the +number of nodes were decreased, reliability would go down along with +bandwidth usage. The reason for this relationship between reliability +and number of nodes is that if we assume that not every node has its UDP +ports open or is behind a cone NAT it means that each of these nodes +must be able to store a certain number of nodes behind restrictive NATs +in order for others to be able to find those nodes behind restrictive +NATs. For example if 7/8 nodes were behind restrictive NATs, using 8 +nodes would not be enough because the chances of some of these nodes +being impossible to find in the network would be too high.

+ +

TODO(zugz): this seems a rather wasteful solution to this problem.

+ +

If the ping timeouts and delays between pings were higher it would +decrease the bandwidth usage but increase the amount of disconnected +nodes that are still being stored in the lists. Decreasing these delays +would do the opposite.

+ +

If the maximum size 8 of the DHT Search Entry Client Lists were +increased would increase the bandwidth usage, might increase hole +punching efficiency on symmetric NATs (more ports to guess from, see +Hole punching) and might increase the reliability. Lowering this number +would have the opposite effect.

+ +

The timeouts and number of nodes in lists for toxcore were picked by +feeling alone and are probably not the best values. This also applies to +the behavior which is simple and should be improved in order to make the +network resist better to sybil attacks.

+ +

TODO: consider giving min and max values for the constants.

+ +

NATs

+ +

We assume that peers are either directly accessible or are behind one of +3 types of NAT:

+ +

Cone NATs: Assign one whole port to each UDP socket behind the NAT; any +packet from any IP/port sent to that assigned port from the internet +will be forwarded to the socket behind it.

+ +

Restricted Cone NATs: Assign one whole port to each UDP socket behind +the NAT. However, it will only forward packets from IPs that the UDP +socket has sent a packet to.

+ +

Symmetric NATs: The worst kind of NAT, they assign a new port for each +IP/port a packet is sent to. They treat each new peer you send a UDP +packet to as a ’connection’ and will only forward packets from the +IP/port of that ’connection’.

+ +

Hole punching

+ +

Holepunching on normal cone NATs is achieved simply through the way in +which the DHT functions.

+ +

If more than half of the 8 peers closest to the friend in the DHT return +an IP/port for the friend and we send a ping request to each of the +returned IP/ports but get no response. If we have sent 4 ping requests +to 4 IP/ports that supposedly belong to the friend and get no response, +then this is enough for toxcore to start the hole punching. The numbers +8 and 4 are used in toxcore and were chosen based on feel alone and so +may not be the best numbers.

+ +

Before starting the hole punching, the peer will send a NAT ping packet +to the friend via the peers that say they know the friend. If a NAT ping +response with the same random number is received the hole punching will +start.

+ +

If a NAT ping request is received, we will first check if it is from a +friend. If it is not from a friend it will be dropped. If it is from a +friend, a response with the same 8 byte number as in the request will be +sent back via the nodes that know the friend sending the request. If no +nodes from the friend are known, the packet will be dropped.

+ +

Receiving a NAT ping response therefore means that the friend is both +online and actively searching for us, as that is the only way they would +know nodes that know us. This is important because hole punching will +work only if the friend is actively trying to connect to us.

+ +

NAT ping requests are sent every 3 seconds in toxcore, if no response is +received for 6 seconds, the hole punching will stop. Sending them in +longer intervals might increase the possibility of the other node going +offline and ping packets sent in the hole punching being sent to a dead +peer but decrease bandwidth usage. Decreasing the intervals will have +the opposite effect.

+ +

There are 2 cases that toxcore handles for the hole punching. The first +case is if each 4+ peers returned the same IP and port. The second is if +the 4+ peers returned same IPs but different ports.

+ +

A third case that may occur is the peers returning different IPs and +ports. This can only happen if the friend is behind a very restrictive +NAT that cannot be hole punched or if the peer recently connected to +another internet connection and some peers still have the old one +stored. Since there is nothing we can do for the first option it is +recommended to just use the most common IP returned by the peers and to +ignore the other IP/ports.

+ +

In the case where the peers return the same IP and port it means that +the other friend is on a restricted cone NAT. These kinds of NATs can be +hole punched by getting the friend to send a packet to our public +IP/port. This means that hole punching can be achieved easily and that +we should just continue sending DHT ping packets regularly to that +IP/port until we get a ping response. This will work because the friend +is searching for us in the DHT and will find us and will send us a +packet to our public IP/port (or try to with the hole punching), thereby +establishing a connection.

+ +

For the case where peers do not return the same ports, this means that +the other peer is on a symmetric NAT. Some symmetric NATs open ports in +sequences so the ports returned by the other peers might be something +like: 1345, 1347, 1389, 1395. The method to hole punch these NATs is to +try to guess which ports are more likely to be used by the other peer +when they try sending us ping requests and send some ping requests to +these ports. Toxcore just tries all the ports beside each returned port +(ex: for the 4 ports previously it would try: 1345, 1347, 1389, 1395, +1346, 1348, 1390, 1396, 1344, 1346…) getting gradually further and +further away and, although this works, the method could be improved. +When using this method toxcore will try up to 48 ports every 3 seconds +until both connect. After 5 tries toxcore doubles this and starts trying +ports from 1024 (48 each time) along with the previous port guessing. +This is because I have noticed that this seemed to fix it for some +symmetric NATs, most likely because a lot of them restart their count at +1024.

+ +

Increasing the amount of ports tried per second would make the hole +punching go faster but might DoS NATs due to the large number of packets +being sent to different IPs in a short amount of time. Decreasing it +would make the hole punching slower.

+ +

This works in cases where both peers have different NATs. For example, +if A and B are trying to connect to each other: A has a symmetric NAT +and B a restricted cone NAT. A will detect that B has a restricted cone +NAT and keep sending ping packets to his one IP/port. B will detect that +A has a symmetric NAT and will send packets to it to try guessing his +ports. If B manages to guess the port A is sending packets from they +will connect together.

+ +

DHT Bootstrap Info (0xf0)

+ +

Bootstrap nodes are regular Tox nodes with a stable DHT public key. This +means the DHT public key does not change across restarts. DHT bootstrap +nodes have one additional request kind: Bootstrap Info. The request is +simply a packet of length 78 bytes where the first byte is 0xf0. The +other bytes are ignored.

+ +

The response format is as follows:

+ + + + + + + + + + + + + + + + + + + + + +
LengthTypeContents
4Word32Bootstrap node version
256BytesMessage of the day
+ +

LAN discovery

+ +

LAN discovery is a way to discover Tox peers that are on a local +network. If two Tox friends are on a local network, the most efficient +way for them to communicate together is to use the local network. If a +Tox client is opened on a local network in which another Tox client +exists then good behavior would be to bootstrap to the network using the +Tox client on the local network. This is what LAN discovery aims to +accomplish.

+ +

LAN discovery works by sending a UDP packet through the toxcore UDP +socket to the interface broadcast address on IPv4, the global broadcast +address (255.255.255.255) and the multicast address on IPv6 (FF02::1) on +the default Tox UDP port (33445).

+ +

The LAN Discovery packet:

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (33)
32DHT public key
+ +

LAN Discovery packets contain the DHT public key of the sender. When a +LAN Discovery packet is received, a DHT get nodes packet will be sent to +the sender of the packet. This means that the DHT instance will +bootstrap itself to every peer from which it receives one of these +packets. Through this mechanism, Tox clients will bootstrap themselves +automatically from other Tox clients running on the local network.

+ +

When enabled, toxcore sends these packets every 10 seconds to keep +delays low. The packets could be sent up to every 60 seconds but this +would make peer finding over the network 6 times slower.

+ +

LAN discovery enables two friends on a local network to find each other +as the DHT prioritizes LAN addresses over non LAN addresses for DHT +peers. Sending a get node request/bootstrapping from a peer successfully +should also add them to the list of DHT peers if we are searching for +them. The peer must not be immediately added if a LAN discovery packet +with a DHT public key that we are searching for is received as there is +no cryptographic proof that this packet is legitimate and not +maliciously crafted. This means that a DHT get node or ping packet must +be sent, and a valid response must be received, before we can say that +this peer has been found.

+ +

LAN discovery is how Tox handles and makes everything work well on LAN.

+ +

Messenger

+ +

Messenger is the module at the top of all the other modules. It sits on +top of friend_connection in the hierarchy of toxcore.

+ +

Messenger takes care of sending and receiving messages using the +connection provided by friend_connection. The module provides a way +for friends to connect and makes it usable as an instant messenger. For +example, Messenger lets users set a nickname and status message which it +then transmits to friends when they are online. It also allows users to +send messages to friends and builds an instant messenging system on top +of the lower level friend_connection module.

+ +

Messenger offers two methods to add a friend. The first way is to add a +friend with only their long term public key, this is used when a friend +needs to be added but for some reason a friend request should not be +sent. The friend should only be added. This method is most commonly used +to accept friend requests but could also be used in other ways. If two +friends add each other using this function they will connect to each +other. Adding a friend using this method just adds the friend to +friend_connection and creates a new friend entry in Messenger for the +friend.

+ +

The Tox ID is used to identify peers so that they can be added as +friends in Tox. In order to add a friend, a Tox user must have the +friend’s Tox ID. The Tox ID contains the long term public key of the +peer (32 bytes) followed by the 4 byte nospam (see: friend_requests) +value and a 2 byte XOR checksum. The method of sending the Tox ID to +others is up to the user and the client but the recommended way is to +encode it in hexadecimal format and have the user manually send it to +the friend using another program.

+ +

Tox ID:

+ +

Tox ID

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
32long term public key
4nospam
2checksum
+ +

The checksum is calculated by XORing the first two bytes of the ID with +the next two bytes, then the next two bytes until all the 36 bytes have +been XORed together. The result is then appended to the end to form the +Tox ID.

+ +

The user must make sure the Tox ID is not intercepted and replaced in +transit by a different Tox ID, which would mean the friend would connect +to a malicious person instead of the user, though taking reasonable +precautions as this is outside the scope of Tox. Tox assumes that the +user has ensured that they are using the correct Tox ID, belonging to +the intended person, to add a friend.

+ +

The second method to add a friend is by using their Tox ID and a message +to be sent in a friend request. This way of adding friends will try to +send a friend request, with the set message, to the peer whose Tox ID +was added. The method is similar to the first one, except that a friend +request is crafted and sent to the other peer.

+ +

When a friend connection associated to a Messenger friend goes online, a +ONLINE packet will be sent to them. Friends are only set as online if an +ONLINE packet is received.

+ +

As soon as a friend goes online, Messenger will stop sending friend +requests to that friend, if it was sending them, as they are redundant +for this friend.

+ +

Friends will be set as offline if either the friend connection +associated to them goes offline or if an OFFLINE packet is received from +the friend.

+ +

Messenger packets are sent to the friend using the online friend +connection to the friend.

+ +

Should Messenger need to check whether any of the non lossy packets in +the following list were received by the friend, for example to implement +receipts for text messages, net_crypto can be used. The net_crypto +packet number, used to send the packets, should be noted and then +net_crypto checked later to see if the bottom of the send array is +after this packet number. If it is, then the friend has received them. +Note that net_crypto packet numbers could overflow after a long time, +so checks should happen within 2**32 net_crypto packets sent with +the same friend connection.

+ +

Message receipts for action messages and normal text messages are +implemented by adding the net_crypto packet number of each message, +along with the receipt number, to the top of a linked list that each +friend has as they are sent. Every Messenger loop, the entries are read +from the bottom and entries are removed and passed to the client until +an entry that refers to a packet not yet received by the other is +reached, when this happens it stops.

+ +

List of Messenger packets:

+ +

ONLINE

+ +

length: 1 byte

+ + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x18)
+ +

Sent to a friend when a connection is established to tell them to mark +us as online in their friends list. This packet and the OFFLINE packet +are necessary as friend_connections can be established with +non-friends who are part of a groupchat. The two packets are used to +differentiate between these peers, connected to the user through +groupchats, and actual friends who ought to be marked as online in the +friendlist.

+ +

On receiving this packet, Messenger will show the peer as being online.

+ +

OFFLINE

+ +

length: 1 byte

+ + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x19)
+ +

Sent to a friend when deleting the friend. Prevents a deleted friend +from seeing us as online if we are connected to them because of a group +chat.

+ +

On receiving this packet, Messenger will show this peer as offline.

+ +

NICKNAME

+ +

length: 1 byte to 129 bytes.

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x30)
[0, 128]Nickname as a UTF8 byte string
+ +

Used to send the nickname of the peer to others. This packet should be +sent every time to each friend every time they come online and each time +the nickname is changed.

+ +

STATUSMESSAGE

+ +

length: 1 byte to 1008 bytes.

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x31)
[0, 1007]Status message as a UTF8 byte string
+ +

Used to send the status message of the peer to others. This packet +should be sent every time to each friend every time they come online and +each time the status message is changed.

+ +

USERSTATUS

+ +

length: 2 bytes

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x32)
1uint8_t status (0 = online, 1 = away, 2 = busy)
+ +

Used to send the user status of the peer to others. This packet should +be sent every time to each friend every time they come online and each +time the user status is changed.

+ +

TYPING

+ +

length: 2 bytes

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x33)
1uint8_t typing status (0 = not typing, 1 = typing)
+ +

Used to tell a friend whether the user is currently typing or not.

+ +

MESSAGE

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x40)
[0, 1372]Message as a UTF8 byte string
+ +

Used to send a normal text message to the friend.

+ +

ACTION

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x41)
[0, 1372]Action message as a UTF8 byte string
+ +

Used to send an action message (like an IRC action) to the friend.

+ +

MSI

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x45)
?data
+ +

Reserved for Tox AV usage.

+ + + +

FILE_SENDREQUEST

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x50)
1uint8_t file number
4uint32_t file type
8uint64_t file size
32file id (32 bytes)
[0, 255]filename as a UTF8 byte string
+ +

Note that file type and file size are sent in big endian/network byte +format.

+ +

FILE_CONTROL

+ +

length: 4 bytes if control_type isn’t seek. 8 bytes if control_type +is seek.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x51)
1uint8_t send_receive
1uint8_t file number
1uint8_t control_type
8uint64_t seek parameter
+ +

send_receive is 0 if the control targets a file being sent (by the +peer sending the file control), and 1 if it targets a file being +received.

+ +

control_type can be one of: 0 = accept, 1 = pause, 2 = kill, 3 = seek.

+ +

The seek parameter is only included when control_type is seek (3).

+ +

Note that if it is included the seek parameter will be sent in big +endian/network byte format.

+ +

FILE_DATA

+ +

length: 2 to 1373 bytes.

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x52)
1uint8_t file number
[0, 1371]file data piece
+ +

Files are transferred in Tox using File transfers.

+ +

To initiate a file transfer, the friend creates and sends a +FILE_SENDREQUEST packet to the friend it wants to initiate a file +transfer to.

+ +

The first part of the FILE_SENDREQUEST packet is the file number. The +file number is the number used to identify this file transfer. As the +file number is represented by a 1 byte number, the maximum amount of +concurrent files Tox can send to a friend is 256. 256 file transfers per +friend is enough that clients can use tricks like queueing files if +there are more files needing to be sent.

+ +

256 outgoing files per friend means that there is a maximum of 512 +concurrent file transfers, between two users, if both incoming and +outgoing file transfers are counted together.

+ +

As file numbers are used to identify the file transfer, the Tox instance +must make sure to use a file number that isn’t used for another outgoing +file transfer to that same friend when creating a new outgoing file +transfer. File numbers are chosen by the file sender and stay unchanged +for the entire duration of the file transfer. The file number is used by +both FILE_CONTROL and FILE_DATA packets to identify which file +transfer these packets are for.

+ +

The second part of the file transfer request is the file type. This is +simply a number that identifies the type of file. for example, tox.h +defines the file type 0 as being a normal file and type 1 as being an +avatar meaning the Tox client should use that file as an avatar. The +file type does not effect in any way how the file is transfered or the +behavior of the file transfer. It is set by the Tox client that creates +the file transfers and send to the friend untouched.

+ +

The file size indicates the total size of the file that will be +transfered. A file size of UINT64_MAX (maximum value in a uint64_t) +means that the size of the file is undetermined or unknown. For example +if someone wanted to use Tox file transfers to stream data they would +set the file size to UINT64_MAX. A file size of 0 is valid and behaves +exactly like a normal file transfer.

+ +

The file id is 32 bytes that can be used to uniquely identify the file +transfer. For example, avatar transfers use it as the hash of the avatar +so that the receiver can check if they already have the avatar for a +friend which saves bandwidth. It is also used to identify broken file +transfers across toxcore restarts (for more info see the file transfer +section of tox.h). The file transfer implementation does not care about +what the file id is, as it is only used by things above it.

+ +

The last part of the file transfer is the optional file name which is +used to tell the receiver the name of the file.

+ +

When a FILE_SENDREQUEST packet is received, the implementation +validates and sends the info to the Tox client which decides whether +they should accept the file transfer or not.

+ +

To refuse or cancel a file transfer, they will send a FILE_CONTROL +packet with control_type 2 (kill).

+ +

FILE_CONTROL packets are used to control the file transfer. +FILE_CONTROL packets are used to accept/unpause, pause, kill/cancel +and seek file transfers. The control_type parameter denotes what the +file control packet does.

+ +

The send_receive and file number are used to identify a specific file +transfer. Since file numbers for outgoing and incoming files are not +related to each other, the send_receive parameter is used to identify +if the file number belongs to files being sent or files being received. +If send_receive is 0, the file number corresponds to a file being sent +by the user sending the file control packet. If send_receive is 1, it +corresponds to a file being received by the user sending the file +control packet.

+ +

control_type indicates the purpose of the FILE_CONTROL packet. +control_type of 0 means that the FILE_CONTROL packet is used to tell +the friend that the file transfer is accepted or that we are unpausing a +previously paused (by us) file transfer. control_type of 1 is used to +tell the other to pause the file transfer.

+ +

If one party pauses a file transfer, that party must be the one to +unpause it. Should both sides pause a file transfer, both sides must +unpause it before the file can be resumed. For example, if the sender +pauses the file transfer, the receiver must not be able to unpause it. +To unpause a file transfer, control_type 0 is used. Files can only be +paused when they are in progress and have been accepted.

+ +

control_type 2 is used to kill, cancel or refuse a file transfer. When +a FILE_CONTROL is received, the targeted file transfer is considered +dead, will immediately be wiped and its file number can be reused. The +peer sending the FILE_CONTROL must also wipe the targeted file +transfer from their side. This control type can be used by both sides of +the transfer at any time.

+ +

control_type 3, the seek control type is used to tell the sender of +the file to start sending from a different index in the file than 0. It +can only be used right after receiving a FILE_SENDREQUEST packet and +before accepting the file by sending a FILE_CONTROL with +control_type 0. When this control_type is used, an extra 8 byte +number in big endian format is appended to the FILE_CONTROL that is +not present with other control types. This number indicates the index in +bytes from the beginning of the file at which the file sender should +start sending the file. The goal of this control type is to ensure that +files can be resumed across core restarts. Tox clients can know if they +have received a part of a file by using the file id and then using this +packet to tell the other side to start sending from the last received +byte. If the seek position is bigger or equal to the size of the file, +the seek packet is invalid and the one receiving it will discard it.

+ +

To accept a file Tox will therefore send a seek packet, if it is needed, +and then send a FILE_CONTROL packet with control_type 0 (accept) to +tell the file sender that the file was accepted.

+ +

Once the file transfer is accepted, the file sender will start sending +file data in sequential chunks from the beginning of the file (or the +position from the FILE_CONTROL seek packet if one was received).

+ +

File data is sent using FILE_DATA packets. The file number corresponds +to the file transfer that the file chunks belong to. The receiver +assumes that the file transfer is over as soon as a chunk with the file +data size not equal to the maximum size (1371 bytes) is received. This +is how the sender tells the receiver that the file transfer is complete +in file transfers where the size of the file is unknown (set to +UINT64_MAX). The receiver also assumes that if the amount of received +data equals to the file size received in the FILE_SENDREQUEST, the +file sending is finished and has been successfully received. Immediately +after this occurs, the receiver frees up the file number so that a new +incoming file transfer can use that file number. The implementation +should discard any extra data received which is larger than the file +size received at the beginning.

+ +

In 0 filesize file transfers, the sender will send one FILE_DATA +packet with a file data size of 0.

+ +

The sender will know if the receiver has received the file successfully +by checking if the friend has received the last FILE_DATA packet sent +(containing the last chunk of the file). net_crypto can be used to +check whether packets sent through it have been received by storing the +packet number of the sent packet and verifying later in net_crypto to +see whether it was received or not. As soon as net_crypto says the +other received the packet, the file transfer is considered successful, +wiped and the file number can be reused to send new files.

+ +

FILE_DATA packets should be sent as fast as the net_crypto +connection can handle it respecting its congestion control.

+ +

If the friend goes offline, all file transfers are cleared in toxcore. +This makes it simpler for toxcore as it does not have to deal with +resuming file transfers. It also makes it simpler for clients as the +method for resuming file transfers remains the same, even if the client +is restarted or toxcore loses the connection to the friend because of a +bad internet connection.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Packet IDPacket Name
0x60INVITE_GROUPCHAT
0x61ONLINE_PACKET
0x62DIRECT_GROUPCHAT
0x63MESSAGE_GROUPCHAT
0xC7LOSSY_GROUPCHAT
+ +

Messenger also takes care of saving the friends list and other friend +information so that it’s possible to close and start toxcore while +keeping all your friends, your long term key and the information +necessary to reconnect to the network.

+ +

Important information messenger stores includes: the long term private +key, our current nospam value, our friends’ public keys and any friend +requests the user is currently sending. The network DHT nodes, TCP +relays and some onion nodes are stored to aid reconnection.

+ +

In addition to this, a lot of optional data can be stored such as the +usernames of friends, our current username, status messages of friends, +our status message, etc… can be stored. The exact format of the +toxcore save is explained later.

+ +

The TCP server is run from the toxcore messenger module if the client +has enabled it. TCP server is usually run independently as part of the +bootstrap node package but it can be enabled in clients. If it is +enabled in toxcore, Messenger will add the running TCP server to the TCP +relay.

+ +

Messenger is the module that transforms code that can connect to friends +based on public key into a real instant messenger.

+ +

TCP client

+ +

TCP client is the client for the TCP server. It establishes and keeps +a connection to the TCP server open.

+ +

All the packet formats are explained in detail in TCP server so this +section will only cover TCP client specific details which are not +covered in the TCP server documentation.

+ +

TCP clients can choose to connect to TCP servers through a proxy. Most +common types of proxies (SOCKS, HTTP) work by establishing a connection +through a proxy using the protocol of that specific type of proxy. After +the connection through that proxy to a TCP server is established, the +socket behaves from the point of view of the application exactly like a +TCP socket that connects directly to a TCP server instance. This means +supporting proxies is easy.

+ +

TCP client first establishes a TCP connection, either through a proxy +or directly to a TCP server. It uses the DHT public key as its long term +key when connecting to the TCP server.

+ +

It establishes a secure connection to the TCP server. After establishing +a connection to the TCP server, and when the handshake response has been +received from the TCP server, the toxcore implementation immediately +sends a ping packet. Ideally the first packets sent would be routing +request packets but this solution aids code simplicity and allows the +server to confirm the connection.

+ +

Ping packets, like all other data packets, are sent as encrypted +packets.

+ +

Ping packets are sent by the toxcore TCP client every 30 seconds with a +timeout of 10 seconds, the same interval and timeout as toxcore TCP +server ping packets. They are the same because they accomplish the same +thing.

+ +

TCP client must have a mechanism to make sure important packets +(routing requests, disconnection notifications, ping packets, ping +response packets) don’t get dropped because the TCP socket is full. +Should this happen, the TCP client must save these packets and +prioritize sending them, in order, when the TCP socket on the server +becomes available for writing again. TCP client must also take into +account that packets might be bigger than the number of bytes it can +currently write to the socket. In this case, it must save the bytes of +the packet that it didn’t write to the socket and write them to the +socket as soon as the socket allows so that the connection does not get +broken. It must also assume that it may receive only part of an +encrypted packet. If this occurs it must save the part of the packet it +has received and wait for the rest of the packet to arrive before +handling it.

+ +

TCP client can be used to open up a route to friends who are connected +to the TCP server. This is done by sending a routing request to the TCP +server with the DHT public key of the friend. This tells the server to +register a connection_id to the DHT public key sent in the packet. The +server will then respond with a routing response packet. If the +connection was accepted, the TCP client will store the connection id +for this connection. The TCP client will make sure that routing +response packets are responses to a routing packet that it sent by +storing that it sent a routing packet to that public key and checking +the response against it. This prevents the possibility of a bad TCP +server exploiting the client.

+ +

The TCP client will handle connection notifications and disconnection +notifications by alerting the module using it that the connection to the +peer is up or down.

+ +

TCP client will send a disconnection notification to kill a connection +to a friend. It must send a disconnection notification packet regardless +of whether the peer was online or offline so that the TCP server will +unregister the connection.

+ +

Data to friends can be sent through the TCP relay using OOB (out of +band) packets and connected connections. To send an OOB packet, the DHT +public key of the friend must be known. OOB packets are sent in blind +and there is no way to query the TCP relay to see if the friend is +connected before sending one. OOB packets should be sent when the +connection to the friend via the TCP relay isn’t in an connected state +but it is known that the friend is connected to that relay. If the +friend is connected via the TCP relay, then normal data packets must be +sent as they are smaller than OOB packets.

+ +

OOB recv and data packets must be handled and passed to the module using +it.

+ +

TCP connections

+ +

TCP_connections takes care of handling multiple TCP client instances +to establish a reliable connection via TCP relays to a friend. +Connecting to a friend with only one relay would not be very reliable, +so TCP_connections provides the level of abstraction needed to manage +multiple relays. For example, it ensures that if a relay goes down, the +connection to the peer will not be impacted. This is done by connecting +to the other peer with more than one relay.

+ +

TCP_connections is above TCP client and below +net_crypto.

+ +

A TCP connection in TCP_connections is defined as a connection to a +peer though one or more TCP relays. To connect to another peer with +TCP_connections, a connection in TCP_connections to the peer with +DHT public key X will be created. Some TCP relays which we know the peer +is connected to will then be associated with that peer. If the peer +isn’t connected directly yet, these relays will be the ones that the +peer has sent to us via the onion module. The peer will also send some +relays it is directly connected to once a connection is established, +however, this is done by another module.

+ +

TCP_connections has a list of all relays it is connected to. It tries +to keep the number of relays it is connected to as small as possible in +order to minimize load on relays and lower bandwidth usage for the +client. The desired number of TCP relay connections per peer is set to 3 +in toxcore with the maximum number set to 6. The reason for these +numbers is that 1 would mean no backup relays and 2 would mean only 1 +backup. To be sure that the connection is reliable 3 seems to be a +reasonable lower bound. The maximum number of 6 is the maximum number of +relays that can be tied to each peer. If 2 peers are connected each to +the same 6+ relays and they both need to be connected to that amount of +relays because of other friends this is where this maximum comes into +play. There is no reason why this number is 6 but in toxcore it has to +be at least double than the desired number (3) because the code assumes +this.

+ +

If necessary, TCP_connections will connect to TCP relays to use them +to send onion packets. This is only done if there is no UDP connection +to the network. When there is a UDP connection, packets are sent with +UDP only because sending them with TCP relays can be less reliable. It +is also important that we are connected at all times to some relays as +these relays will be used by TCP only peers to initiate a connection to +us.

+ +

In toxcore, each client is connected to 3 relays even if there are no +TCP peers and the onion is not needed. It might be optimal to only +connect to these relays when toxcore is initializing as this is the only +time when peers will connect to us via TCP relays we are connected to. +Due to how the onion works, after the initialization phase, where each +peer is searched in the onion and then if they are found the info +required to connect back (DHT pk, TCP relays) is sent to them, there +should be no more peers connecting to us via TCP relays. This may be a +way to further reduce load on TCP relays, however, more research is +needed before it is implemented.

+ +

TCP_connections picks one relay and uses only it for sending data to +the other peer. The reason for not picking a random connected relay for +each packet is that it severely deteriorates the quality of the link +between two peers and makes performance of lossy video and audio +transmissions really poor. For this reason, one relay is picked and used +to send all data. If for any reason no more data can be sent through +that relay, the next relay is used. This may happen if the TCP socket is +full and so the relay should not necessarily be dropped if this occurs. +Relays are only dropped if they time out or if they become useless (if +the relay is one too many or is no longer being used to relay data to +any peers).

+ +

TCP_connections in toxcore also contains a mechanism to make +connections go to sleep. TCP connections to other peers may be put to +sleep if the connection to the peer establishes itself with UDP after +the connection is established with TCP. UDP is the method preferred by +net_crypto to communicate with other peers. In order to keep track of +the relays which were used to connect with the other peer in case the +UDP connection fails, they are saved by TCP_connections when the +connection is put to sleep. Any relays which were only used by this +redundant connection are saved then disconnected from. If the connection +is awakened, the relays are reconnected to and the connection is +reestablished. Putting a connection to sleep is the same as saving all +the relays used by the connection and removing the connection. Awakening +the connection is the same as creating a new connection with the same +parameters and restoring all the relays.

+ +

A method to detect potentially dysfunctional relays that try to disrupt +the network by lying that they are connecting to a peer when they are +not or that maliciously drop all packets should be considered. Toxcore +doesn’t currently implement such a system and adding one requires more +research and likely also requires extending the protocol.

+ +

When TCP connections connects to a relay it will create a new +TCP_client instance for that relay. At any time if the +TCP_client instance reports that it has disconnected, the TCP relay +will be dropped. Once the TCP relay reports that it is connected, +TCP_connections will find all the connections that are associated to +the relay and announce to the relay that it wants to connect to each of +them with routing requests. If the relay reports that the peer for a +connection is online, the connection number and relay will be used to +send data in that connection with data packets. If the peer isn’t +reported as online but the relay is associated to a connection, TCP OOB +(out of band) packets will be used to send data instead of data packets. +TCP OOB packets are used in this case since the relay most likely has +the peer connected but it has not sent a routing request to connect to +us.

+ +

TCP_connections is used as the bridge between individual TCP_client +instances and net_crypto, or the bridge between individual connections +and something that requires an interface that looks like one connection.

+ +

TCP server

+ +

The TCP server in tox has the goal of acting like a TCP relay between +clients who cannot connect directly to each other or who for some reason +are limited to using the TCP protocol to connect to each other. +TCP_server is typically run only on actual server machines but any Tox +client could host one as the api to run one is exposed through the tox.h +api.

+ +

To connect to a hosted TCP server toxcore uses the TCP client module.

+ +

The TCP server implementation in toxcore can currently either work on +epoll on linux or using unoptimized but portable socket polling.

+ +

TCP connections between the TCP client and the server are encrypted to +prevent an outsider from knowing information like who is connecting to +whom just be looking at someones connection to a TCP server. This is +useful when someone connects though something like Tor for example. It +also prevents someone from injecting data in the stream and makes it so +we can assume that any data received was not tampered with and is +exactly what was sent by the client.

+ +

When a client first connects to a TCP server he opens up a TCP +connection to the ip and port the TCP server is listening on. Once the +connection is established he then sends a handshake packet, the server +then responds with his own and a secure connection is established. The +connection is then said to be unconfirmed and the client must then send +some encrypted data to the server before the server can mark the +connection as confirmed. The reason it works like this is to prevent a +type of attack where a peer would send a handshake packet and then time +out right away. To prevent this the server must wait a few seconds for a +sign that the client received his handshake packet before confirming the +connection. The both can then communicate with each other using the +encrypted connection.

+ +

The TCP server essentially acts as just a relay between 2 peers. When a +TCP client connects to the server he tells the server which clients he +wants the server to connect him to. The server will only let two clients +connect to each other if both have indicated to the server that they +want to connect to each other. This is to prevent non friends from +checking if someone is connected to a TCP server. The TCP server +supports sending packets blindly through it to clients with a client +with public key X (OOB packets) however the TCP server does not give any +feedback or anything to say if the packet arrived or not and as such it +is only useful to send data to friends who may not know that we are +connected to the current TCP server while we know they are. This occurs +when one peer discovers the TCP relay and DHT public key of the other +peer before the other peer discovers its DHT public key. In that case +OOB packets would be used until the other peer knows that the peer is +connected to the relay and establishes a connection through it.

+ +

In order to make toxcore work on TCP only the TCP server supports +relaying onion packets from TCP clients and sending any responses from +them to TCP clients.

+ +

To establish a secure connection with a TCP server send the following +128 bytes of data or handshake packet to the server:

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
32DHT public key of client
24Nonce for the encrypted data
72Payload (plus MAC)
+ +

Payload is encrypted with the DHT private key of the client and public +key of the server and the nonce:

+ + + + + + + + + + + + + + + + + + +
LengthContents
32Public key
24Base nonce
+ +

The base nonce is the one TCP client wants the TCP server to use to +decrypt the packets received from the TCP client.

+ +

The first 32 bytes are the public key (DHT public key) that the TCP +client is announcing itself to the server with. The next 24 bytes are a +nonce which the TCP client uses along with the secret key associated +with the public key in the first 32 bytes of the packet to encrypt the +rest of this ’packet’. The encrypted part of this packet contains a +temporary public key that will be used for encryption during the +connection and will be discarded after. It also contains a base nonce +which will be used later for decrypting packets received from the TCP +client.

+ +

If the server decrypts successfully the encrypted data in the handshake +packet and responds with the following handshake response of length 96 +bytes:

+ + + + + + + + + + + + + + + + + + +
LengthContents
24Nonce for the encrypted data
72Payload (plus MAC)
+ +

Payload is encrypted with the private key of the server and the DHT +public key of the client and the nonce:

+ + + + + + + + + + + + + + + + + + +
LengthContents
32Public key
24Base nonce
+ +

The base nonce is the one the TCP server wants the TCP client to use to +decrypt the packets received from the TCP server.

+ +

The client already knows the long term public key of the server so it is +omitted in the response, instead only a nonce is present in the +unencrypted part. The encrypted part of the response has the same +elements as the encrypted part of the request: a temporary public key +tied to this connection and a base nonce which will be used later when +decrypting packets received from the TCP client both unique for the +connection.

+ +

In toxcore the base nonce is generated randomly like all the other +nonces, it must be randomly generated to prevent nonce reuse. For +example if the nonce used was 0 for both sides since both sides use the +same keys to encrypt packets they send to each other, two packets would +be encrypted with the same nonce. These packets could then be possibly +replayed back to the sender which would cause issues. A similar +mechanism is used in net_crypto.

+ +

After this the client will know the connection temporary public key and +base nonce of the server and the server will know the connection base +nonce and temporary public key of the client.

+ +

The client will then send an encrypted packet to the server, the +contents of the packet do not matter and it must be handled normally by +the server (ex: if it was a ping send a pong response. The first packet +must be any valid encrypted data packet), the only thing that does +matter is that the packet was encrypted correctly by the client because +it means that the client has correctly received the handshake response +the server sent to it and that the handshake the client sent to the +server really came from the client and not from an attacker replaying +packets. The server must prevent resource consuming attacks by timing +out clients if they do not send any encrypted packets so the server to +prove to the server that the connection was established correctly.

+ +

Toxcore does not have a timeout for clients, instead it stores +connecting clients in large circular lists and times them out if their +entry in the list gets replaced by a newer connection. The reasoning +behind this is that it prevents TCP flood attacks from having a negative +impact on the currently connected nodes. There are however much better +ways to do this and the only reason toxcore does it this way is because +writing it was very simple. When connections are confirmed they are +moved somewhere else.

+ +

When the server confirms the connection he must look in the list of +connected peers to see if he is already connected to a client with the +same announced public key. If this is the case the server must kill the +previous connection because this means that the client previously timed +out and is reconnecting. Because of Toxcore design it is very unlikely +to happen that two legitimate different peers will have the same public +key so this is the correct behavior.

+ +

Encrypted data packets look like this to outsiders:

+ + + + + + + + + + + + + + + + + + +
LengthContents
2uint16_t length of data
variableencrypted data
+ +

In a TCP stream they would look like: +[[length][data]][[length][data]][[length][data]]....

+ +

Both the client and server use the following (temp public and private +(client and server) connection keys) which are each generated for the +connection and then sent to the other in the handshake and sent to the +other. They are then used like the next diagram shows to generate a +shared key which is equal on both sides.

+ +
Client:                                     Server:
+generate_shared_key(                        generate_shared_key(
+[temp connection public key of server],     [temp connection public key of client],
+[temp connection private key of client])    [temp connection private key of server])
+=                                           =
+[shared key]                                [shared key]
+
+ +

The generated shared key is equal on both sides and is used to encrypt +and decrypt the encrypted data packets.

+ +

each encrypted data packet sent to the client will be encrypted with the +shared key and with a nonce equal to: (client base nonce + number of +packets sent so for the first packet it is (starting at 0) nonce + 0, +the second is nonce + 1 and so on. Note that nonces like all other +numbers sent over the network in toxcore are numbers in big endian +format so when increasing them by 1 the least significant byte is the +last one)

+ +

each packet received from the client will be decrypted with the shared +key and with a nonce equal to: (server base nonce + number of packets +sent so for the first packet it is (starting at 0) nonce + 0, the second +is nonce + 1 and so on. Note that nonces like all other numbers sent +over the network in toxcore are numbers in big endian format so when +increasing them by 1 the least significant byte is the last one)

+ +

Encrypted data packets have a hard maximum size of 2 + 2048 bytes in the +toxcore TCP server implementation, 2048 bytes is big enough to make sure +that all toxcore packets can go through and leaves some extra space just +in case the protocol needs to be changed in the future. The 2 bytes +represents the size of the data length and the 2048 bytes the max size +of the encrypted part. This means the maximum size is 2050 bytes. In +current toxcore, the largest encrypted data packets sent will be of size +2 + 1417 which is 1419 total.

+ +

The logic behind the format of the handshake is that we:

+ +
    +
  1. +

    need to prove to the server that we own the private key related to +the public key we are announcing ourselves with.

    +
    2. +
  2. +

    need to establish a secure connection that has perfect forward +secrecy

    +
    3. +
  3. +

    prevent any replay, impersonation or other attacks

    +
    4. +
+ +

How it accomplishes each of those points:

+ +
    +
  1. +

    If the client does not own the private key related to the public key +they will not be able to create the handshake packet.

    +
    2. +
  2. +

    Temporary session keys generated by the client and server in the +encrypted part of the handshake packets are used to encrypt/decrypt +packets during the session.

    +
    3. +
  3. +

    The following attacks are prevented:

    + +
      +
    • +

      Attacker modifies any byte of the handshake packets: Decryption +fail, no attacks possible.

      +
      • +
    • +

      Attacker captures the handshake packet from the client and +replays it later to the server: Attacker will never get the +server to confirm the connection (no effect).

      +
      • +
    • +

      Attacker captures a server response and sends it to the client +next time they try to connect to the server: Client will never +confirm the connection. (See: TCP_client)

      +
      • +
    • +

      Attacker tries to impersonate a server: They won’t be able to +decrypt the handshake and won’t be able to respond.

      +
      • +
    • +

      Attacker tries to impersonate a client: Server won’t be able to +decrypt the handshake.

      +
      • +
    +
    4. +
+ +

The logic behind the format of the encrypted packets is that:

+ +
    +
  1. +

    TCP is a stream protocol, we need packets.

    +
    2. +
  2. +

    Any attacks must be prevented

    +
    3. +
+ +

How it accomplishes each of those points:

+ +
    +
  1. +

    2 bytes before each packet of encrypted data denote the length. We +assume a functioning TCP will deliver bytes in order which makes it +work. If the TCP doesn’t it most likely means it is under attack and +for that see the next point.

    +
    2. +
  2. +

    The following attacks are prevented:

    + +
      +
    • +

      Modifying the length bytes will either make the connection time +out and/or decryption fail.

      +
      • +
    • +

      Modifying any encrypted bytes will make decryption fail.

      +
      • +
    • +

      Injecting any bytes will make decryption fail.

      +
      • +
    • +

      Trying to re order the packets will make decryption fail because +of the ordered nonce.

      +
      • +
    • +

      Removing any packets from the stream will make decryption fail +because of the ordered nonce.

      +
      • +
    +
    3. +
+ +

Encrypted payload types

+ +

The folowing represents the various types of data that can be sent +inside encrypted data packets.

+ +

Routing request (0x00)

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x00)
32Public key
+ +

Routing request response (0x01)

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x01)
1uint8_t rpid
32Public key
+ +

rpid is invalid connection_id (0) if refused, connection_id if +accepted.

+ +

Connect notification (0x02)

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x02)
1uint8_t connection_id of connection that got connected
+ +

Disconnect notification (0x03)

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x03)
1uint8_t connection_id of connection that got disconnected
+ +

Ping packet (0x04)

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x04)
8uint64_t ping_id (0 is invalid)
+ +

Ping response (pong) (0x05)

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x05)
8uint64_t ping_id (0 is invalid)
+ +

OOB send (0x06)

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x06)
32Destination public key
variableData
+ +

OOB recv (0x07)

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x07)
32Sender public key
variableData
+ +

Onion packet (0x08)

+ +

Same format as initial onion packet but packet id is 0x08 instead of +0x80.

+ +

Onion packet response (0x09)

+ +

Same format as onion packet but packet id is 0x09 instead of 0x8e.

+ +

Data (0x10 and up)

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t packet id
1uint8_t connection id
variabledata
+ +

The TCP server is set up in a way to minimize waste while relaying the +many packets that might go between two tox peers hence clients must +create connections to other clients on the relay. The connection number +is a uint8_t and must be equal or greater to 16 in order to be valid. +Because a uint8_t has a maximum value of 256 it means that the maximum +number of different connections to other clients that each connection +can have is 240. The reason valid connection_ids are bigger than 16 is +because they are the first byte of data packets. Currently only number 0 +to 9 are taken however we keep a few extras in case we need to extend +the protocol without breaking it completely.

+ +

Routing request (Sent by client to server): Send a routing request to +the server that we want to connect to peer with public key where the +public key is the public the peer announced themselves as. The server +must respond to this with a Routing response.

+ +

Routing response (Sent by server to client): The response to the routing +request, tell the client if the routing request succeeded (valid +connection_id) and if it did, tell them the id of the connection +(connection_id). The public key sent in the routing request is also +sent in the response so that the client can send many requests at the +same time to the server without having code to track which response +belongs to which public key.

+ +

The only reason a routing request should fail is if the connection has +reached the maximum number of simultaneous connections. In case the +routing request fails the public key in the response will be the public +key in the failed request.

+ +

Connect notification (Sent by server to client): Tell the client that +connection_id is now connected meaning the other is online and data +can be sent using this connection_id.

+ +

Disconnect notification (Sent by client to server): Sent when client +wants the server to forget about the connection related to the +connection_id in the notification. Server must remove this connection +and must be able to reuse the connection_id for another connection. If +the connection was connected the server must send a disconnect +notification to the other client. The other client must think that this +client has simply disconnected from the TCP server.

+ +

Disconnect notification (Sent by server to client): Sent by the server +to the client to tell them that the connection with connection_id that +was connected is now disconnected. It is sent either when the other +client of the connection disconnect or when they tell the server to kill +the connection (see above).

+ +

Ping and Pong packets (can be sent by both client and server, both will +respond): ping packets are used to know if the other side of the +connection is still live. TCP when established doesn’t have any sane +timeouts (1 week isn’t sane) so we are obliged to have our own way to +check if the other side is still live. Ping ids can be anything except +0, this is because of how toxcore sets the variable storing the +ping_id that was sent to 0 when it receives a pong response which +means 0 is invalid.

+ +

The server should send ping packets every X seconds (toxcore +TCP_server sends them every 30 seconds and times out the peer if it +doesn’t get a response in 10). The server should respond immediately to +ping packets with pong packets.

+ +

The server should respond to ping packets with pong packets with the +same ping_id as was in the ping packet. The server should check that +each pong packet contains the same ping_id as was in the ping, if not +the pong packet must be ignored.

+ +

OOB send (Sent by client to server): If a peer with private key equal to +the key they announced themselves with is connected, the data in the OOB +send packet will be sent to that peer as an OOB recv packet. If no such +peer is connected, the packet is discarded. The toxcore TCP_server +implementation has a hard maximum OOB data length of 1024. 1024 was +picked because it is big enough for the net_crypto packets related to +the handshake and is large enough that any changes to the protocol would +not require breaking TCP server. It is however not large enough for the +biggest net_crypto packets sent with an established net_crypto +connection to prevent sending those via OOB packets.

+ +

OOB recv (Sent by server to client): OOB recv are sent with the +announced public key of the peer that sent the OOB send packet and the +exact data.

+ +

OOB packets can be used just like normal data packets however the extra +size makes sending data only through them less efficient than data +packets.

+ +

Data: Data packets can only be sent and received if the corresponding +connection_id is connection (a Connect notification has been received +from it) if the server receives a Data packet for a non connected or +existent connection it will discard it.

+ +

Why did I use different packet ids for all packets when some are only +sent by the client and some only by the server? It’s less confusing.

+ +

Friend connection

+ +

friend_connection is the module that sits on top of the DHT, onion and +net_crypto modules and takes care of linking the 3 together.

+ +

Friends in friend_connection are represented by their real public key. +When a friend is added in friend_connection, an onion search entry is +created for that friend. This means that the onion module will start +looking for this friend and send that friend their DHT public key, and +the TCP relays it is connected to, in case a connection is only possible +with TCP.

+ +

Once the onion returns the DHT public key of the peer, the DHT public +key is saved, added to the DHT friends list and a new net_crypto +connection is created. Any TCP relays returned by the onion for this +friend are passed to the net_crypto connection.

+ +

If the DHT establishes a direct UDP connection with the friend, +friend_connection will pass the IP/port of the friend to net_crypto +and also save it to be used to reconnect to the friend if they +disconnect.

+ +

If net_crypto finds that the friend has a different DHT public key, +which can happen if the friend restarted their client, net_crypto will +pass the new DHT public key to the onion module and will remove the DHT +entry for the old DHT public key and replace it with the new one. The +current net_crypto connection will also be killed and a new one with +the correct DHT public key will be created.

+ +

When the net_crypto connection for a friend goes online, +friend_connection will tell the onion module that the friend is online +so that it can stop spending resources looking for the friend. When the +friend connection goes offline, friend_connection will tell the onion +module so that it can start looking for the friend again.

+ +

There are 2 types of data packets sent to friends with the net_crypto +connection handled at the level of friend_connection, Alive packets +and TCP relay packets. Alive packets are packets with the packet id or +first byte of data (only byte in this packet) being 16. They are used in +order to check if the other friend is still online. net_crypto does +not have any timeout when the connection is established so timeouts are +caught using this packet. In toxcore, this packet is sent every 8 +seconds. If none of these packets are received for 32 seconds, the +connection is timed out and killed. These numbers seem to cause the +least issues and 32 seconds is not too long so that, if a friend times +out, toxcore won’t falsely see them online for too long. Usually when a +friend goes offline they have time to send a disconnect packet in the +net_crypto connection which makes them appear offline almost +instantly.

+ +

The timeout for when to stop retrying to connect to a friend by creating +new net_crypto connections when the old one times out in toxcore is +the same as the timeout for DHT peers (122 seconds). However, it is +calculated from the last time a DHT public key was received for the +friend or time the friend’s net_crypto connection went offline after +being online. The highest time is used to calculate when the timeout is. +net_crypto connections will be recreated (if the connection fails) +until this timeout.

+ +

friend_connection sends a list of 3 relays (the same number as the +target number of TCP relay connections in TCP_connections) to each +connected friend every 5 minutes in toxcore. Immediately before sending +the relays, they are associated to the current +net_crypto->TCP_connections connection. This facilitates connecting +the two friends together using the relays as the friend who receives the +packet will associate the sent relays to the net_crypto connection +they received it from. When both sides do this they will be able to +connect to each other using the relays. The packet id or first byte of +the packet of share relay packets is 0x11. This is then followed by some +TCP relays stored in packed node format.

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x11)
variableTCP relays in packed node format (see DHT)
+ +

If local IPs are received as part of the packet, the local IP will be +replaced with the IP of the peer that sent the relay. This is because we +assume this is the best way to attempt to connect to the TCP relay. If +the peer that sent the relay is using a local IP, then the sent local IP +should be used to connect to the relay.

+ +

For all other data packets, are passed by friend_connection up to the +upper Messenger module. It also separates lossy and lossless packets +from net_crypto.

+ +

Friend connection takes care of establishing the connection to the +friend and gives the upper messenger layer a simple interface to receive +and send messages, add and remove friends and know if a friend is +connected (online) or not connected (offline).

+ +

Friend requests

+ +

When a Tox user adds someone with Tox, toxcore will try sending a friend +request to that person. A friend request contains the long term public +key of the sender, a nospam number and a message.

+ +

Transmitting the long term public key is the primary goal of the friend +request as it is what the peer needs to find and establish a connection +to the sender. The long term public key is what the receiver adds to his +friends list if he accepts the friend request.

+ +

The nospam is a number used to prevent someone from spamming the network +with valid friend requests. It makes sure that the only people who have +seen the Tox ID of a peer are capable of sending them a friend request. +The nospam is one of the components of the Tox ID.

+ +

The nospam is a number or a list of numbers set by the peer, only +received friend requests that contain a nospam that was set by the peer +are sent to the client to be accepted or refused by the user. The nospam +prevents random peers in the network from sending friend requests to non +friends. The nospam is not long enough to be secure meaning an extremely +resilient attacker could manage to send a spam friend request to +someone. 4 bytes is large enough to prevent spam from random peers in +the network. The nospam could also allow Tox users to issue different +Tox IDs and even change Tox IDs if someone finds a Tox ID and decides to +send it hundreds of spam friend requests. Changing the nospam would stop +the incoming wave of spam friend requests without any negative effects +to the users friends list. For example if users would have to change +their public key to prevent them from receiving friend requests it would +mean they would have to essentially abandon all their current friends as +friends are tied to the public key. The nospam is not used at all once +the friends have each other added which means changing it won’t have any +negative effects.

+ +

Friend + request:

+ +
[uint32_t nospam][Message (UTF8) 1 to ONION_CLIENT_MAX_DATA_SIZE bytes]
+
+ +

Friend request packet when sent as an onion data packet:

+ +
[uint8_t (32)][Friend request]
+
+ +

Friend request packet when sent as a net_crypto data packet (If we are +directly connected to the peer because of a group chat but are not +friends with them):

+ +
[uint8_t (18)][Friend request]
+
+ +

When a friend is added to toxcore with their Tox ID and a message, the +friend is added in friend_connection and then toxcore tries to send +friend requests.

+ +

When sending a friend request, toxcore will check if the peer which a +friend request is being sent to is already connected to using a +net_crypto connection which can happen if both are in the same group +chat. If this is the case the friend request will be sent as a +net_crypto packet using that connection. If not, it will be sent as an +onion data packet.

+ +

Onion data packets contain the real public key of the sender and if a +net_crypto connection is established it means the peer knows our real +public key. This is why the friend request does not need to contain the +real public key of the peer.

+ +

Friend requests are sent with exponentially increasing interval of 2 +seconds, 4 seconds, 8 seconds, etc… in toxcore. This is so friend +requests get resent but eventually get resent in intervals that are so +big that they essentially expire. The sender has no way of knowing if a +peer refuses a friend requests which is why friend requests need to +expire in some way. Note that the interval is the minimum timeout, if +toxcore cannot send that friend request it will try again until it +manages to send it. One reason for not being able to send the friend +request would be that the onion has not found the friend in the onion +and so cannot send an onion data packet to them.

+ +

Received friend requests are passed to the client, the client is +expected to show the message from the friend request to the user and ask +the user if they want to accept the friend request or not. Friend +requests are accepted by adding the peer sending the friend request as a +friend and refused by simply ignoring it.

+ +

Friend requests are sent multiple times meaning that in order to prevent +the same friend request from being sent to the client multiple times +toxcore keeps a list of the last real public keys it received friend +requests from and discards any received friend requests that are from a +real public key that is in that list. In toxcore this list is a simple +circular list. There are many ways this could be improved and made more +efficient as a circular list isn’t very efficient however it has worked +well in toxcore so far.

+ +

Friend requests from public keys that are already added to the friends +list should also be discarded.

+ +

Group

+ +

Group chats in Tox work by temporarily adding some peers present in the +group chat as temporary friend_connection friends, that are deleted +when the group chat is exited.

+ +

Each peer in the group chat is identified by their real long term public +key. Peers also transmit their DHT public keys to each other via the +group chat in order to speed up the connection by making it unnecessary +for the peers to find each other’s DHT public keys with the onion, as +would happen had they added each other as normal friends.

+ +

The upside of using friend_connection is that group chats do not have +to deal with things like hole punching, peers only on TCP or other low +level networking things. The downside however is that every single peer +knows each other’s real long term public key and DHT public key, meaning +that these group chats should only be used between friends.

+ +

Each peer adds a friend_connection for each of up to 4 other peers in +the group. If the group chat has 5 participants or fewer, each of the +peers will therefore have each of the others added to their list of +friend connections, and a peer wishing to send a message to the group +may communicate it directly to the other peers. When there are more than +5 peers, messages are relayed along friend connections.

+ +

Since the maximum number of peers per groupchat that will be connected +to with friend connections is 4, if the peers in the groupchat are +arranged in a circle and each peer connects to the 2 peers that are +closest to the right of them and the 2 peers that are closest to the +left of them, then the peers should form a well-connected circle of +peers.

+ +

Group chats in toxcore do this by subtracting the real long term public +key of the peer with all the others in the group (our PK - other peer +PK), using modular arithmetic, and finding the two peers for which the +result of this operation is the smallest. The operation is then inversed +(other peer PK - our PK) and this operation is done again with all the +public keys of the peers in the group. The 2 peers for which the result +is again the smallest are picked.

+ +

This gives 4 peers that are then added as a friend connection and +associated to the group. If every peer in the group does this, they will +form a circle of perfectly connected peers.

+ +

Once the peers are connected to each other in a circle they relay each +other’s messages. Every time a peer leaves the group or a new peer +joins, each member of the chat will recalculate the peers they should +connect to.

+ +

To join a group chat, a peer must first be invited to it by their +friend. To make a groupchat the peer will first create a groupchat and +then invite people to this group chat. Once their friends are in the +group chat, those friends can invite their other friends to the chat, +and so on.

+ +

To create a group chat, a peer generates a random 32 byte id that is +used to uniquely identify the group chat. 32 bytes is enough so that +when randomly generated with a secure random number generator every +groupchat ever created will have a different id. The goal of this 32 +byte id is so that peers have a way of identifying each group chat, so +that they can prevent themselves from joining a groupchat twice for +example.

+ +

The groupchat will also have an unsigned 1 byte type. This type +indicates what kind of groupchat the groupchat is. The current types +are:

+ + + + + + + + + + + + + + + + + + +
Type numberType
0text
1audio
+ +

Text groupchats are text only, while audio indicates that the groupchat +supports sending audio to it as well as text.

+ +

The groupchat will also be identified by a unique unsigned 2 byte +integer, which in toxcore corresponds to the index of the groupchat in +the array it is being stored in. Every groupchat in the current instance +must have a different number. This number is used by groupchat peers +that are directly connected to us to tell us which packets are for which +groupchat. Every groupchat packet contains a 2 byte groupchat number. +Putting a 32 byte groupchat id in each packet would increase bandwidth +waste by a lot, and this is the reason why groupchat numbers are used +instead.

+ +

Using the group number as the index of the array used to store the +groupchat instances is recommended, because this kind of access is +usually most efficient and it ensures that each groupchat has a unique +group number.

+ +

When creating a new groupchat, the peer will add themselves as a +groupchat peer with a peer number of 0 and their own long term public +key and DHT public key.

+ +

Invite packets:

+ +

Invite packet:

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x60)
1uint8_t (0x00)
2uint16_t group number
33Group chat identifier
+ +

Accept Invite packet:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x60)
1uint8_t (0x01)
2uint16_t group number (local)
2uint16_t group number to join
33Group chat identifier
+ +

Member Information packet:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x60)
1uint8_t (0x02)
2uint16_t group number (local)
2uint16_t group number to join
33Group chat identifier
2uint16_t peer number
+ +

A group chat identifier consists of a 1-byte type and a 32-byte id +concatenated:

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t type
32uint8_t groupchat id
+ +

To invite a friend to a group chat, an invite packet is sent to the +friend. These packets are sent using Messenger (if you look at the +Messenger packet id section, all the groupchat packet ids are in there). +Note that all numbers here, like all numbers sent using Tox packets, are +sent in big endian format.

+ +

The group chat number is as explained above, the number used to uniquely +identify the groupchat instance from all the other groupchat instances +the peer has. It is sent in the invite packet because it is needed by +the friend in order to send back groupchat related packets.

+ +

What follows is the 33 byte group chat identifier.

+ +

To refuse the invite, the friend receiving it will simply ignore and +discard it.

+ +

To accept the invite, the friend will create their own groupchat +instance with the 1 byte type and 32 byte groupchat id sent in the +request, and send an invite accept packet back. The friend will also add +the peer who sent the invite as a groupchat connection, and mark the +connection as introducing the friend.

+ +

If the friend being invited is already in the group, they will respond +with a member information packet, add the peer who sent the invite as a +groupchat connection, and mark the connection as introducing both the +friend and the peer who sent the invite.

+ +

The first group number in the invite accept packet is the group number +of the groupchat the invited friend just created. The second group +number is the group number that was sent in the invite request. What +follows is the 33 byte group chat identifier which was sent in the +invite request. The member information packet is the same, but includes +also the current peer number of the invited friend.

+ +

When a peer receives an invite accept packet they will check if the +group identifier sent back corresponds to the group identifier of the +groupchat with the group number also sent back. If so, a new peer number +will be generated for the peer that sent the invite accept packet. Then +the peer with their generated peer number, their long term public key +and their DHT public key will be added to the peer list of the +groupchat. A new peer message packet will also be sent to tell everyone +in the group chat about the new peer. The peer will also be added as a +groupchat connection, and the connection will be marked as introducing +the peer.

+ +

When a peer receives a member information packet they proceed as with an +invite accept packet, but use the peer number in the packet rather than +generating a new one, and mark the new connection as also introducing +the peer receiving the member information packet.

+ +

Peer numbers are used to uniquely identify each peer in the group chat. +They are used in groupchat message packets so that peers receiving them +can know who or which groupchat peer sent them. As groupchat message +packets are relayed, they must contain something that is used by others +to identify the sender. Since putting a 32 byte public key in each +packet would be wasteful, a 2 byte peer number is used instead. Each +peer in the groupchat has a unique peer number. Toxcore generates each +peer number randomly but makes sure newly generated peer numbers are not +equal to current ones already used by other peers in the group chat. If +two peers join the groupchat from two different endpoints there is a +small possibility that both will be given the same peer number, but the +probability of this occurring is low enough in practice that it is not +an issue.

+ +

Peer online packet:

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x61)
2uint16_t group number (local)
33Group chat identifier
+ +

Peer introduced packet:

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x62)
2uint16_t group number (local)
1uint8_t (0x01)
+ +

For a groupchat connection to work, both peers in the groupchat must be +attempting to connect directly to each other.

+ +

Groupchat connections are established when both peers who want to +connect to each other either create a new friend connection to connect +to each other or reuse an existing friend connection that connects them +together (if they are friends or already are connected together because +of another group chat).

+ +

As soon as the connection to the other peer is opened, a peer online +packet is sent to the peer. The goal of the online packet is to tell the +peer that we want to establish the groupchat connection with them and +tell them the groupchat number of our groupchat instance. The peer +online packet contains the group number and the 33 byte group chat +identifier. The group number is the group number the peer has for the +group with the group id sent in the packet.

+ +

When both sides send an online packet to the other peer, a connection is +established.

+ +

When an online packet is received from a peer, if the connection to the +peer is already established (an online packet has been already +received), or if there is no group connection to that peer being +established, the packet is dropped. Otherwise, the group number to +communicate with the group via the peer is saved, the connection is +considered established, and an online packet is sent back to the peer. A +ping message is sent to the group. If this is the first group connection +to that group we establish, or the connection is marked as introducing +us, we send a peer query packet back to the peer. This is so we can get +the list of peers from the group. If the connection is marked as +introducing the peer, we send a new peer message to the group announcing +the peer, and a name message reannouncing our name.

+ +

A groupchat connection can be marked as introducing one or both of the +peers it connects, to indicate that the connection should be maintained +until that peer is well connected to the group. A peer maintains a +groupchat connection to a second peer as long as the second peer is one +of the four closest peers in the groupchat to the first, or the +connection is marked as introducing a peer who still requires the +connection. A peer requires a groupchat connection to a second peer +which introduces the first peer until the first peer has more than 4 +groupchat connections and receives a message from the second peer via a +different groupchat connection. The first peer then sends a peer +introduced packet to the second peer to indicate that they no longer +require the connection.

+ +

Peer query packet:

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x62)
2uint16_t group number
1uint8_t (0x08)
+ +

Peer response packet:

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x62)
2uint16_t group number
1uint8_t (0x09)
variableRepeated times number of peers: Peer info
+ +

The Peer info structure is as follows:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
2uint16_t peer number
32Long term public key
32DHT public key
1uint8_t Name length
[0, 255]Name
+ +

Title response packet:

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x62)
2uint16_t group number
1uint8_t (0x0a)
variableTitle
+ +

Message packets:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x63)
2uint16_t group number
2uint16_t peer number
4uint32_t message number
1uint8_t with a value representing id of message
variableData
+ +

Lossy Message packets:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0xc7)
2uint16_t group number
2uint16_t peer number
2uint16_t message number
1uint8_t with a value representing id of message
variableData
+ +

If a peer query packet is received, the receiver takes their list of +peers and creates a peer response packet which is then sent to the other +peer. If there are too many peers in the group chat and the peer +response packet would be larger than the maximum size of friend +connection packets (1373 bytes), more than one peer response packet is +sent back. A Title response packet is also sent back. This is how the +peer that joins a group chat finds out the list of peers in the group +chat and the title of the group chat right after joining.

+ +

Peer response packets are straightforward and contain the information +for each peer (peer number, real public key, DHT public key, name) +appended to each other. The title response is also straightforward.

+ +

Both the maximum length of groupchat peer names and the groupchat title +is 128 bytes. This is the same maximum length as names in all of +toxcore.

+ +

When a peer receives a peer response packet, they will add each of the +received peers to their groupchat peer list, find the 4 closest peers to +them and create groupchat connections to them as was explained +previously. The DHT public key of an already known peer is updated to +one given in the response packet if the peer is frozen, or if it has +been frozen since its DHT public key was last updated.

+ +

When a peer receives a title response packet, they update the title for +the groupchat accordingly if the title has not already been set, or if +since it was last set there has been a time at which all peers were +frozen.

+ +

If the peer does not yet know their own peer number, as is the case if +they have just accepted an invitation, the peer will find themselves in +the list of received peers and use the peer number assigned to them as +their own. They are then able to send messages and invite other peers to +the groupchat. They immediately send a name message to announce their +name to the group.

+ +

Message packets are used to send messages to all peers in the groupchat. +To send a message packet, a peer will first take their peer number and +the message they want to send. Each message packet sent will have a +message number that is equal to the last message number sent + 1. Like +all other numbers (group chat number, peer number) in the packet, the +message number in the packet will be in big endian format.

+ +

When a Message packet is received, the peer receiving it will first +check that the peer number of the sender is in their peer list. If not, +the peer ignores the message but sends a peer query packet to the peer +the packet was directly received from. That peer should have the message +sender in their peer list, and so will send the sender’s peer info back +in a peer response.

+ +

If the sender is in the receiver’s peer list, the receiver now checks +whether they have already seen a message with the same sender and +message number. This is achieved by storing the 8 greatest message +numbers received from a given sender peer number. If the message has +lesser message number than any of those 8, it is assumed to have been +received. If the message has already been received according to this +check, or if it is a name or title message and another message of the +same type from the same sender with a greater message number has been +received, then the packet is discarded. Otherwise, the message is +processed as described below, and a Message packet with the message is +sent (relayed) to all current group connections except the one that it +was received from, and also to that one if that peer is the original +sender of the message. The only thing that should change in the Message +packet as it is relayed is the group number.

+ +

Lossy message packets are used to send audio packets to others in audio +group chats. Lossy packets work the same way as normal relayed groupchat +messages in that they are relayed to everyone in the group chat until +everyone has them, but there are a few differences. Firstly, the message +number is only a 2 byte integer. When receiving a lossy packet from a +peer the receiving peer will first check if a message with that message +number was already received from that peer. If it wasn’t, the packet +will be added to the list of received packets and then the packet will +be passed to its handler and then sent to the 2 closest connected +groupchat peers that are not the sender. The reason for it to be 2 +instead of 4 (or 3 if we are not the original sender) as for lossless +message packets is that it reduces bandwidth usage without lowering the +quality of the received audio stream via lossy packets, at the cost of +reduced robustness against connections failing. To check if a packet was +already received, the last 256 message numbers received from each peer +are stored. If video was added meaning a much higher number of packets +would be sent, this number would be increased. If the packet number is +in this list then it was received.

+ +

Message ids

+ +

ping (0x00)

+ +

Sent approximately every 20 seconds by every peer. Contains no data.

+ +

new_peer (0x10)

+ +

Tell everyone about a new peer in the chat.

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
2uint16_t Peer number
32Long term public key
32DHT public key
+ +

kill_peer (0x11)

+ + + + + + + + + + + + + + +
LengthContents
2uint16_t Peer number
+ +

freeze_peer (0x12)

+ + + + + + + + + + + + + + +
LengthContents
2uint16_t Peer number
+ +

Name change (0x30)

+ + + + + + + + + + + + + + +
LengthContents
variableName (namelen)
+ +

Groupchat title change (0x31)

+ + + + + + + + + + + + + + +
LengthContents
variableTitle (titlelen)
+ +

Chat message (0x40)

+ + + + + + + + + + + + + + +
LengthContents
variableMessage (messagelen)
+ +

Action (/me) (0x41)

+ + + + + + + + + + + + + + +
LengthContents
variableMessage (messagelen)
+ +

Ping messages are sent every 20 seconds by every peer. This is how other +peers know that the peers are still alive.

+ +

When a new peer joins, the peer which invited the joining peer will send +a new peer message to warn everyone that there is a new peer in the +chat. When a new peer message is received, the peer in the message must +be added to the peer list if it is not there already, and its DHT public +key must be set to that in the message.

+ +

Kill peer messages are used to indicate that a peer has quit the group +chat permanently. Freeze peer messages are similar, but indicate that +the quitting peer may later return to the group. Each is sent by the one +quitting the group chat right before they quit it.

+ +

Name change messages are used to change or set the name of the peer +sending it. They are also sent by a joining peer right after receiving +the list of peers in order to tell others what their name is.

+ +

Title change packets are used to change the title of the group chat and +can be sent by anyone in the group chat.

+ +

Chat and action messages are used by the group chat peers to send +messages to others in the group chat.

+ +

Timeouts and reconnection

+ +

Groupchat connections may go down, and this may lead to a peer becoming +disconnected from the group or the group otherwise splitting into +multiple connected components. To ensure the group becomes fully +connected again once suitable connections are re-established, peers keep +track of peers who are no longer visible in the group (“frozen” peers), +and try to re-integrate them into the group via any suitable friend +connections which may come to be available. The rejoin packet is used +for this.

+ +

Rejoin packet:

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x64)
33Group chat identifier
+ +

A peer in a groupchat is considered to be active when a group message or +rejoin packet is received from it, or a new peer message is received for +it. A peer which remains inactive for 60 seconds is set as frozen; this +means it is removed from the peer list and added to a separate list of +frozen peers. Frozen peers are disregarded for all purposes except those +discussed below.

+ +

If a frozen peer becomes active, we unfreeze it, meaning that we move it +from the frozen peers list to the peer list, and we send a name message +to the group.

+ +

Whenever we make a new friend connection to a peer, we check whether the +public key of the peer is that of any frozen peer. If so, we send a +rejoin packet to the peer along the friend connection, and create a +groupchat connection to the peer, marked as introducing us, and send a +peer online packet to the peer.

+ +

If we receive a rejoin packet from a peer along a friend connection, +then, after unfreezing the peer if it was frozen, we update the peer’s +DHT public key in the groupchat peer list to the key in the friend +connection, and create a groupchat connection for the peer, marked as +introducing the peer, and send a peer online packet to the peer.

+ +

When a peer is added to the peer list, any existing peer in the peer +list or frozen peers list with the same public key is first removed.

+ +

DHT Group Chats

+ +

This document details the groupchat implementation, giving a high level +overview of all the important features and aspects, as well as some +important low level implementation details. This documentation reflects +what is currently implemented at the time of writing; it is not +speculative. For detailed API docs see the groupchats section of the +tox.h header file.

+ +

Features

+ +
    +
  • +

    Plain and action messages (/me)

    +
    • +
  • +

    Private messages

    +
    • +
  • +

    Public groups (peers may join via a public key of group)

    +
    • +
  • +

    Private groups (require a friend invite for join)

    +
    • +
  • +

    Permanence (a group cannot ’die’ as long as at least one peer +retains their group credentials)

    +
    • +
  • +

    Persistence across client restarts

    +
    • +
  • +

    Ability to set peer limits

    +
    • +
  • +

    Group roles (founder, moderators, users, observers)

    +
    • +
  • +

    Moderation (kicking, silencing, controlling which roles may speak)

    +
    • +
  • +

    Permanent group names (set on creation)

    +
    • +
  • +

    Topics (permission to modify is set by founder)

    +
    • +
  • +

    Password protection

    +
    • +
  • +

    Self-repairing (auto-rejoin on disconnect, group split protection, +state syncing)

    +
    • +
  • +

    Identity separation from the Tox ID

    +
    • +
  • +

    Ability to ignore peers

    +
    • +
  • +

    Nicknames can be set on a per-group basis

    +
    • +
  • +

    Peer statuses (online, away, busy) which can be set on a per-group +basis

    +
    • +
  • +

    Sending group name in invites

    +
    • +
  • +

    Ability to disconnect from group and join later with the same +credentials

    +
    • +
+ +

Group roles

+ +

There are four distinct roles which are hierarchical in nature (higher +roles have all the privileges of lower roles).

+ +
    +
  • +

    Founder - The group’s creator. May set the role of all other +peers to anything except founder. May modify the shared state +(password, privacy state, topic lock, peer limit).

    +
    • +
  • +

    Moderator - Promoted by the founder. May kick peers below this +role, as well as set peers with the user role to observer, and vice +versa. May also set the topic when the topic lock is enabled.

    +
    • +
  • +

    User - Default non-founder role. May communicate with other +peers normally. May set the topic when the topic lock is disabled.

    +
    • +
  • +

    Observer - Demoted by moderators and the founder. May observe +the group and ignore peers; may not communicate with other peers or +with the group.

    +
    • +
+ +

Group types

+ +

Groups can have two types: private and public. The type can be set on +creation, and may also be toggled by the group founder at any point +after creation. (Note: password protection is independent of the group +type)

+ +

Public

+ +

Anyone may join the group using the Chat ID. If the group is public, +information about peers inside the group, including their IP addresses +and group public keys (but not their Tox ID’s) is visible to anyone with +access to a node storing their DHT announcement.

+ +

Private

+ +

The only way to join a private group is by having someone in your friend +list send you an invite. If the group is private, no peer/group +information (mentioned in the Public section) is present in the DHT; the +DHT is not used for any purpose at all. If a public group is set to +private, all DHT information related to the group will expire within a +few minutes.

+ +

Voice state

+ +

The voice state, which may only be set by the founder, determines which +group roles have permission to speak. There are three voice states:

+ +
    +
  • +

    Founder - Only the founder may speak.

    +
    • +
  • +

    Moderator - The founder and moderators may speak.

    +
    • +
  • +

    All - Everyone except observers may speak.

    +
    • +
+ +

The voice state does not affect topic setting or private messages, and +is set to All by default.

+ +

Cryptography

+ +

Groupchats use the NaCl/libsodium cryptography +library for all +cryptography related operations. All group communication is end-to-end +encrypted. Message confidentiality, integrity, and repudability are +guaranteed via authenticated +encryption, and +perfect forward secrecy +is also provided.

+ +

One of the most important security improvements from the old groupchat +implementation is the removal of a message-relay mechanism that uses a +group-wide shared key. Instead, connections are 1-to-1 (a complete +graph), meaning an outbound message is sent once per peer, and +encrypted/decrypted using a session key unique to each pair of peers. +This prevents MITM attacks that were previously possible. This +additionally ensures that private messages are truly private.

+ +

Groups make use of 11 unique keys in total: Two permanent keypairs +(encryption and signature), two group keypairs (encryption and +signature), one session keypair (encryption), and one shared symmetric +key (encryption).

+ +

The Tox ID/Tox public key is not used for any purpose. As such, neither +peers in a given group nor in the group DHT can be matched with their +Tox ID. In other words, there is no way of identifying a peer aside from +their IP address, nickname, and group public key. (Note: group +nicknames can be different from the client’s main nickname that their +friends see).

+ +

Permanent keypairs

+ +

When a peer creates or joins a group they generate two permanent +keypairs: an encryption keypair and a signature keypair, both of which +are unique to the group. The two public keys are the only guaranteed way +to identify a peer, and both keypairs will persist for as long as a peer +remains in the group (even across client restarts). If a peer exits the +group these keypairs will be lost forever.

+ +

This encryption keypair is not used for any encryption operations except +for the initial handshake when connecting to another peer. For usage +details on the signature key, see the Moderation +section.

+ +

Session keypair/shared symmetric key

+ +

When two peers establish a connection they each generate an ephemeral +session encryption keypair and share one another’s resulting public key. +With their own session secret key and the other’s session public key, +they will both generate the same symmetric encryption key. This +symmetric key, which must not be exposed to anyone else, will be used +for all further encryption and decryption operations between the two +peers for the duration of the session.

+ +

The purpose of this extra key exchange is to prevent an adversary from +decrypting messages from previous sessions in event that a secret +encryption key becomes compromised. This is known as forward secrecy.

+ +

Session keys are periodically rotated to further reduce the potential +damage in the event of a security breach, as well as to mitigate certain +types of data-based cryptography attacks.

+ +

Group keypairs

+ +

The group founder generates two additional permanent keypairs when the +group is created: an encryption keypair, and a signature keypair. The +public signature key is considered the Chat ID and is used as the +group’s permanent identifier, allowing other peers to join public groups +via the DHT. Every peer in the group holds a copy of the group’s public +encryption key along with the public signature key/Chat ID.

+ +

The group secret keys are similar to the permanent keypairs in that they +will persist across client restarts, but will be lost forever if the +founder exits the group. This is particularly important as +administration related functionality will not work without these keys.

+ +

See the Founders section for usage details.

+ +

Founders

+ +

The peer who creates the group is the group’s founder. Founders have a +set of admin privileges, including:

+ +
    +
  • +

    Promoting and demoting moderators

    +
    • +
  • +

    The ability to kick moderators along-side non-moderators

    +
    • +
  • +

    Setting the peer limit

    +
    • +
  • +

    Setting the group’s privacy state

    +
    • +
  • +

    Setting group passwords

    +
    • +
  • +

    Toggling the topic lock

    +
    • +
  • +

    Setting the voice state

    +
    • +
+ +

Shared state

+ +

Groups contain a data structure called the shared state which is +given to every peer who joins the group. Within this structure resides +all data pertaining to the group that may only be modified by the group +founder. This includes the group name, the group type, the peer limit, +the topic lock, the password, and the voice state. The shared state +holds a copy of the group founder’s public encryption and signature +keys, which is how other peers in the group are able to verify the +identity of the group founder. It also contains a hash of the moderator +list.

+ +

The shared state is signed by the founder using the group secret +signature key. As the founder is the only peer who holds this secret +key, the shared state can be shared with new peers and cryptographically +verified even in the absence of the founder.

+ +

When the founder modifies the shared state, he increments the shared +state version, signs the new shared state data with the group secret +signature key, and broadcasts the new shared state data along with its +signature to the entire group. When a peer receives this broadcast, he +uses the group public signature key to verify that the data was signed +with the group secret signature key, and also verifies that the new +version is not older than the current version.

+ +

Moderation

+ +

The founder has the ability to promote other peers to the moderator +role. Moderators have all the privileges of normal users. In addition, +they have the power to kick peers whose role is below moderator, as well +as set their roles to anything below moderator. Moderators may also +modify the group topic when it is locked. Moderators have no power over +one another; only the founder can kick or change the role of a +moderator.

+ +

Kicks

+ +

When a peer is kicked from the group, he will be disconnected from all +group peers, his role will be set to user, and his chat instance will be +left in a disconnected state. His public key will not be lost; he will +be able to reconnect to the group with the same identity.

+ +

Moderator list

+ +

Each peer holds a copy of the moderator list, which is an array of +public signature keys of peers who currently have the moderator role +(including those who are offline). A hash (sha256) of this list called +the mod_list_hash is stored in the shared state, which is itself +signed by the founder using the group secret signature key. This allows +the moderator list to be shared between untrusted peers, even in the +absence of the founder, while maintaining moderator verifiability.

+ +

When the founder modifies the moderator list, he updates the +mod_list_hash, increments the shared state version, signs the new +shared state, broadcasts the new shared state data along with its +signature to the entire group, then broadcasts the new moderator list to +the entire group. When a peer receives this moderator list (having +already verified the new shared state), he creates a hash of the new +list and verifies that it is identical to the mod_list_hash.

+ +

Sanctions list

+ +

Each peer holds a copy of the sanctions list. This list is +comprised of peers who have been demoted to the observer role.

+ +

Entries contain the public key of the sanctioned peer, a timestamp of +the time the entry was made, the public signature key of the peer who +set the sanction, and a signature of the entry’s data, which is signed +by the peer who created the entry using their secret signature key. +Individual entries are verified by ensuring that the entry’s public +signature key belongs to the founder or is present in the moderator +list, and then verifying that the entry’s data was signed by the owner +of that key.

+ +

Although each individual entry can be verified, we still need a way to +verify that the list as a whole is complete, and identical for every +peer, otherwise any peer would be able to remove entries arbitrarily, or +replace the list with an older version. Therefore each peer holds a copy +of the sanctions list credentials. This is a data structure that +holds a version number, a hash (sha256) of all combined sanctions list +entries, a 16-bit checksum of the hash, the public signature key of the +last peer to have modified the list, and a signature of the hash, which +is signed by the private signature key associated with the +aforementioned public signature key.

+ +

When a moderator or founder modifies the sanctions list, he will +increment the version, create a new hash of the list, make a checksum of +the hash, sign the hash+version with his secret signature key, and +replace the old public signature key with his own. He will then +broadcast the new changes (not the entire list) to the entire group +along with the new credentials. When a peer receives this broadcast, he +will verify that the new credentials version is not older than the +current version, validate the hash and checksum, and verify that the +changes were made by a moderator or the founder. If adding an entry, he +will verify that the entry was signed by the signature key of the +entry’s creator.

+ +

If a peer receives sanctions credentials with a version equal to their +own but with a different checksum, they will ignore the changes if the +new checksum is a smaller value than the checksum for their current +sanctions credentials.

+ +

When the founder kicks or demotes a moderator, he will first go through +the sanctions list and re-sign each entry made by that moderator using +the founder key, then re-broadcast the sanctions list to the entire +group. This is necessary to guarantee that all sanctions list entries +and its credentials are signed by a current moderator or the founder at +all times.

+ +

Note: The sanctions list is not saved to the Tox save file, meaning +that if the group ever becomes empty, the sanctions list will be reset. +This is in contrast to the shared state and moderator list, which are +both saved and will persist even if the group becomes empty.

+ +

Topics

+ +

The topic is an arbitrary string of characters with a maximum length of +512 bytes. The topic has two states: locked and unlocked. When locked, +only moderators and the founder may modify it. When unlocked, all peers +except observers may modify it. The integrity of the topic is maintained +in a similar manner as sanctions entries, using a data structure called +topic_info. This is a struct which contains the topic, a version, +a 16-bit checksum of the topic, and the public key of the peer who last +set the topic.

+ +

The topic lock state is kept track of in the shared state, and may only +be modified by the founder. If the topic lock is set to zero, this +indicates that the lock is enabled. If non-zero, the value is set to the +topic version of the last topic set when the lock was enabled. This +allows peers to ensure that the topic version is not modified while the +lock is disabled.

+ +

When the topic lock is enabled, the topic setter will create a new +checksum of the topic and increment the topic version. They will then +sign the topic and version with their secret signature key, replace the +public key with their own, and broadcast the new topic_info data along +with the signature to the entire group. When a peer receives this +broadcast they will check if the public signature key of the topic +setter either belongs to the founder or is in the moderator list, and +ensure that the version is not older than the current topic version. +They will then verify the signature using the setter’s public signature +key and validate the checksum. If the received topic has the same +version as their own but a different checksum, they will ignore the new +topic if its checksum value is smaller than the checksum value for their +current topic.

+ +

When the topic lock is disabled, the topic setter will create a +new checksum of the topic and leave the version unchanged. They will +then sign the topic and version with their secret signature key, replace +the public key with their own, and broadcast the new topic_info data +along with the signature to the entire group. When a peer receives this +broadcast they will ensure that the topic setter is not in the sanctions +list, and ensure that the version is equal to the value that the topic +lock is set to, unless the setter has the Founder role, in which case +they will ignore the version. They will then verify the signature using +the setter’s public signature key and validate the checksum.

+ +

If the peer who set the current topic is kicked or demoted, or if the +topic lock is enabled, the peer who initiated the action will re-sign +the topic using his own signature key and rebroadcast it to the entire +group.

+ +

If the peer who set the current topic is kicked or demoted, or if the +topic lock is enabled, the peer who initiated the action will re-sign +the topic using his own signature key and rebroadcast it to the entire +group.

+ +

State syncing

+ +

Peers send four unsigned 16-bit integers and three unsigned 32-bit +integers along with their ping packets: Their peer count[1], a +checksum of their peer list, their shared state version, their sanctions +credentials version, their peer roles checksum[2], their topic +version, and their topic checksum. If a peer receives a ping in which +any of the versions are greater than their own, or if their peer list +checksum does not match and their peer count is not greater than the +peer count received, this indicates that they may be out of sync with +the rest of the group. In this case they will send a sync request to the +respective peer, with the appropriate sync flags set to indicate what +group information they need.

+ +

In certain scenarios a peer may receive a topic version or sanctions +credentials version that is equal to their own, but with a different +checksum. This may occur if two or more peers in the group initiate an +action at the exact same time. If such a conflict occurs, the peer will +make the appropriate sync request if their checksum is a smaller value +than the one they received.

+ +

Peers that are connected to the DHT also occasionally append their IP +and port number to their ping packets for peers with which they do not +have a direct UDP connection established. This gives priority to direct +connections and ensures that TCP relays are used only as a fall-back, or +when a peer explicitly forces a TCP connection.

+ +

DHT Announcements

+ +

Public groupchats leverage the Tox DHT network in order to allow for +groups that can be joined by anyone who possesses the Chat ID. +Group announcements have the same underlying functionality as normal Tox +friend announcements (including onion routing).

+ +

DHT Group Chats Packet Protocols

+ +

All packet fields are considered mandatory unless flagged as +[optional]. The minimum size of an encrypted packet is 83 bytes +for lossless and 75 bytes for lossy. The maximum size of an encrypted +packet is 1400 bytes.

+ +

Full Packet Structure

+ +

Plaintext header

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1Packet Kind
32Sender’s Public Encryption Key
32Receiver’s Public Encryption Key [optional]
24Nonce
+ +

Encrypted header

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
0-8Padding
1Group Packet Identifier
8Message Id [optional]
+ +

Encrypted payload

+ + + + + + + + + + + + + + +
LengthContents
VariablePayload
+ +

The plaintext header contains a Toxcore Network Packet Kind which +identifies the toxcore networking level packet type. These types are:

+ + + + + + + + + + + + + + + + + + + + + + +
TypeNet Packet ID
NET_PACKET_GC_HANDSHAKE0x5a
NET_PACKET_GC_LOSSLESS0x5b
NET_PACKET_GC_LOSSY0x5c
+ +

The sender’s public encryption key is used to identify the peer who sent +the packet, as well as to identify the group instance for which the +packet is intended for all NET_PACKET_GC_LOSSLESS and +NET_PACKET_GC_LOSSY packets. It is also used to establish a secure +connection with the sender during the handshake protocol.

+ +

The receiver’s public encryption key is only sent in +NET_PACKET_GC_HANDSHAKE packets, and is used to identify the group +instance for which the packet is intended.

+ +

The encrypted header for lossless and lossy packets contains between 0 +and 8 bytes of empty padding. The Group Packet Identifier is used +to identify the type of group packet, and the Message ID is a +unique packet identifier which is used for the lossless UDP +implementation.

+ +

The encrypted payload contains arbitrary data specific to the respective +group packet identifier. The length may range from zero to the maximum +packet size (minus the headers). These payloads will be the focus of the +remainder of this document.

+ +

Handshake packet payloads

+ +

Handshake packet payloads are structured as follows:

+ +

HANDSHAKE_REQUEST (0x00) and HANDSHAKE_RESPONSE (0x01)

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
32Public Session Key
32Public Signature Key
1Request Type
variable1 Packed TCP Relay
+ +

This packet type is used to initiate a secure connection with a peer.

+ +

The Public Session Key is a temporary key unique to this peer +which, along with its secret counterpart, will be used to create a +shared session encryption key. This keypair is used for all further +communication for the current session. It must only be used for a single +peer, and must be discarded of once the connection with the peer is +severed.

+ +

The Public Signature Key is our own permanent signature key for +this group chat.

+ +

The Request Type is an identifier for the type of handshake being +initiated, defined as an enumerator starting at zero as follows:

+ + + + + + + + + + + + + + + + + + +
TypeID
HANDSHAKE_INVITE_REQUEST0x00
HANDSHAKE_PEER_INFO_EXCHANGE0x01
+ +

If the request type is an invite request, the receiving peer must +respond with a INVITE_REQUEST packet. If the request type is a +peer info exchange, the receiving peer must respond with a +PEER_INFO_RESPONSE packet followed immediately by a +PEER_INFO_REQUEST packet.

+ +

The packed TCP relay contains a TCP relay that the sender may be +connected through by the receiver.

+ +

Lossy Packet Payloads

+ +

PING (0x01)

+ +

A ping packet payload is structured as follows:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
2Peerlist Checksum
2Confirmed Peer Count
4Shared State Version
4Sanctions Credentials Version
2Peer Roles Checksum
4Topic Version
2Topic Checksum
VariablePacked IP Address and Port
+ +

Ping packets are periodically sent to every confirmed peer in order to +maintain peer connections, and to ensure the group state between peers +are in sync. A peer is considered to be disconnected from the group +after a ping packet has not been receieved over a period of time.

+ +

MESSAGE_ACK (0x02)

+ +

Message ack packets are structured as follows:

+ + + + + + + + + + + + + + + + + + +
LengthContents
8Message ID
1Type
+ +

This packet ensures that all lossless packets are successfully received +and processed in sequential order as they were sent.

+ +

Message ack types are defined by an enumerator beginning at zero as +follows:

+ + + + + + + + + + + + + + + + + + +
TypeID
GR_ACK_RECV0x00
GR_ACK_REQ0x01
+ +

If the type is GR_ACK_RECV, this indicates that the packet with +the given id has been received and successfully processed. If the type +is GR_ACK_REQ, this indicates that the message with the given id +should be sent again.

+ +

INVITE_RESPONSE_REJECT (0x03)

+ +

An invite response reject payload is structured as follows:

+ + + + + + + + + + + + + + +
LengthContents
1Type
+ +

This packet alerts a peer that their invite request has been rejected. +The reason for the rejection is specified by the type field.

+ +

Rejection types are defined by an enumerator beginning at zero as +follows:

+ + + + + + + + + + + + + + + + + + + + + + +
TypeID
GROUP_FULL0x00
INVALID_PASSWORD0x01
INVITE_FAILED0x02
+ +

Lossless Packet Payloads

+ +

FRAGMENT (0xef)

+ +

Fragment packets are structured as follows:

+ + + + + + + + + + + + + + + + + + +
LengthContents
1Lossless Packet Type [First chunk only]
VariableArbitrary data
+ +

Represents a segment in a sequence of packet fragments that comprise one +full lossless packet payload which exceeds the maximum allowed packet +chunk size (500 bytes). The first byte in the first chunk must be a +lossless packet type. Each chunk in the sequence must be sent in +succession.

+ +

The end of the sequence is signaled by a fragment packet with a length +of zero.

+ +

A fully assembled packet must be no greater than 50,000 bytes.

+ +

KEY_ROTATIONS (0xf0)

+ +

Key rotation packets are structured as follows:

+ + + + + + + + + + + + + + + + + + +
LengthContents
1is_response
32Public Encryption Key
+ +

Key rotation packets are used to rotate session encryption keys with a +peer. If is_response is false, the packet initiates a public key +exchange. Otherwise the packet is a response to a previously initiated +exchange.

+ +

The public encryption key must be a newly generated key which takes the +place of the previously used session key. The resulting shared session +key is generated using the same protocol as the initial handshake, and +must be kept secret.

+ +

Request packets should only be sent by the peer whose permanent public +encryption key for the given group is closer to the group Chat ID +according to the Distance metric.

+ +

TCP_RELAYS (0xf1)

+ +

A TCP relay packet payload is structured as follows:

+ + + + + + + + + + + + + + +
LengthContents
VariablePacked TCP Relays
+ +

The purpose of this packet is to share a list of TCP relays with a +confirmed peer. Used to maintain a list of mutual TCP relays with other +peers, which are used to maintain TCP connections when direct +connections cannot be established.

+ +

This packet is sent to every confirmed peer whenever a new TCP relay is +added to our list, or periodically when we presently have no shared TCP +relays with a given peer.

+ +

CUSTOM_PACKETS (0xf2)

+ +

A custom packet payload is structured as follows:

+ + + + + + + + + + + + + + +
LengthContents
VariableArbitrary Data
+ +

This packet is used to to send arbitrary data to another peer. It may be +used for client-side features.

+ +

BROADCAST (0xf3)

+ +

A broadcast packet payload is structured as follows:

+ + + + + + + + + + + + + + + + + + +
LengthContents
1Type
VariablePayload
+ +

This packet broadcasts a message to all confirmed peers in a group (with +the exception of PRIVATE_MESSAGE). The type of broadcast is +specificed by the type field.

+ +

Broadcast types are defined and structured as follows:

+ +
STATUS (0x00)
+ + + + + + + + + + + + + + +
LengthContents
1User status
+ +

Indicates that a peer has changed their status. Statuses must be of type +USERSTATUS.

+ +
NICK (0x01)
+ + + + + + + + + + + + + + +
LengthContents
VariableName
+ +

Indicates that a peer has changed their nickname. A nick must be greater +than 0 bytes, and may not exceed TOX_MAX_NAME_LENGTH bytes in +length.

+ +
PLAIN_MESSAGE (0x02)
+ + + + + + + + + + + + + + +
LengthContents
VariableArbitrary data
+ +

Contains an arbitrary message. A plain message must be greater than 0 +bytes, and may not exceed TOX_MAX_MESSAGE_LENGTH bytes.

+ +
ACTION_MESSAGE (0x03)
+ + + + + + + + + + + + + + +
LengthContents
VariableArbitrary data
+ +

Contains an arbitrary message. An action message must be greater than 0 +bytes, and may not exceed TOX_MAX_MESSAGE_LENGTH bytes.

+ +
PRIVATE_MESSAGE (0x04)
+ + + + + + + + + + + + + + +
LengthContents
VariableArbitrary data
+ +

Contains an arbitrary message which is only sent to the intended peer. A +private message must be greater than 0 bytes, and may not exceed +TOX_MAX_MESSAGE_LENGTH bytes.

+ +
PEER_EXIT (0x05)
+ + + + + + + + + + + + + + +
LengthContents
VariableArbitrary data [optional]
+ +

Indicates that a peer is leaving the group. Contains an optional parting +message which may not exceed TOX_GROUP_MAX_PART_LENGTH.

+ +
PEER_KICK (0x06)
+ + + + + + + + + + + + + + +
LengthContents
32Public Encryption Key
+ +

Indicates that the peer associated with the public encryption key has +been kicked from the group by a moderator or the founder. This peer must +be removed from the peer list.

+ +
SET_MOD (0x07)
+ + + + + + + + + + + + + + + + + + +
LengthContents
1Flag
32Public Signature Key
+ +

Indicates that the peer associated with the public signature key has +either been promoted to or demoted from the Moderator role by the +group founder. If flag is non-zero, the peer should be promoted +and added to the moderator list. Otherwise they should be demoted to the +User role and removed from the moderator list.

+ +
SET_OBSERVER (0x08)
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1Flag
32Public Encryption Key
32Public Signature Key
137Sanctions List Entry [optional]
132Packed Sanctions List Credentials
+ +

Indicates that the peer associated with the given public keys has either +been demoted to or promoted from the Observer role by the group +founder or a moderator. If flag is non-zero, the peer should be +demoted and added to the sanctions list. Otherwise they should be +promoted to the User role and removed from the sanctions list.

+ +

PEER_INFO_REQUEST (0xf4)

+ +

A peer info request packet contains an empty payload. Its purpose is to +request a peer to send us information about themselves.

+ +

PEER_INFO_RESPONSE (0xf5)

+ +

A peer info response packet payload is structured as follows:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
2Group Password Length [optional]
32Group Password [optional]
2Name Length
128Name
1Status
+ +

This packet supplies information about ourselves to a peer. It is sent +as a response to a PEER_INFO_REQUEST or +HS_PEER_INFO_EXCHANGE packet as part of the handshake protocol. A +password and length of password must be included in the packet if the +group is password protected.

+ +

INVITE_REQUEST (0xf6)

+ +

An invite request packet payload is structured as follows:

+ + + + + + + + + + + + + + + + + + +
LengthContents
2Group Password Length [optional]
32Group Password [optional]
+ +

This packet requests an invite to the group. A password and length of +password must be included in the packet if the group is password +protected.

+ +

INVITE_RESPONSE (0xf7)

+ +

An invite response packet has an empty payload.

+ +

This packet alerts a peer who sent us an INVITE_REQUEST packet +that their request has been validated, which informs them that they may +continue to the next step in the handshake protocol.

+ +

Before sending this packet we first attempt to validate the invite +request. If validation fails, we instead send a packet of type +INVITE_RESPONSE_REJECT in response, and remove the peer from our +peer list.

+ +

SYNC_REQUEST (0xf8)

+ +

A sync request packet payload is structured as follows:

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
2Sync_Flags
2Group Password Length [optional]
32Group Password [optional]
+ +

This packet asks a peer to send us state information about the group +chat. The specific information being requested is specified via the +Sync_Flags field. A password and length of password must be +included in the packet if the group is password protected.

+ +

Sync_Flags is a bitfield defined as a 16-bit unsigned integer +which may have the bits set for the respective values depending on what +information is being requested:

+ + + + + + + + + + + + + + + + + + + + + + +
TypeSet Bits
PEER_LIST0x01
TOPIC0x02
STATE0x04
+ +

SYNC_RESPONSE (0xf9)

+ +

A sync response packet payload is structured as follows:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
32Public Encryption Key
1IP_Port_Is_Set
1TCP Relays Count
VariablePacked IP_Port [optional]
VariablePacked TCP Relays [optional]
+ +

This packet is sent as a response to a peer who made a sync request via +the SYNC_REQUEST packet. It contains a single packed peer +announce, which is a data structure that contains all of the information +about a peer needed to initiate the handshake protocol via TCP relays, a +direct connection, or both.

+ +

If the IP_Port_Is_Set flag is non-zero, the packet will contain a +packed IP_Port of the peer associated with the given public key. +If TCP Relays Count is greater than 0, the packet will contain a +list of tcp relays that the peer associated with the given public key is +connected to.

+ +

When responding to a sync request, one separate sync response will be +sent for each peer in the peer list. All other requested group +information is sent via its respective packet.

+ +

TOPIC (0xfa)

+ +

A topic packet payload is structured as follows:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
64Topic Signature
4Topic Version
2Topic Checksum
2Topic Length
Topic LengthTopic
32Public Signature Key
+ +

This packet contains a topic as well as information used to validate the +topic. Sent when the topic changes, or in response to a +SYNC_REQUEST in which the TOPIC flag is set. A topic may not +exceed TOX_GROUP_MAX_TOPIC_LENGTH bytes in length.

+ +

SHARED_STATE (0xfb)

+ +

A shared state packet payload is structured as follows:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
64Shared State Signature
4Shared State Version
64Founder Extended Public Key
2Peer Limit
2Group Name Length
48Group Name
1Privacy State
2Group Password Length
32Group Password
32Moderator List Hash (Sha256)
4Topic Lock State
1Voice State
+ +

This packet contains information about the group shared state. Sent to +all peers by the group founder whenever the shared state has changed. +Also sent in response to a SYNC_REQUEST in which the STATE +flag is set.

+ +

MOD_LIST (0xfc)

+ +

A moderation list packet payload is structured as follows:

+ + + + + + + + + + + + + + + + + + +
LengthContents
2Moderator Count
VariableModerator List
+ +

This packet contains information about the moderator list, including the +number of moderators, and a list of public signature keys of all current +moderators. Sent to all peers by the group founder after the moderator +list has been modified. Also sent in response to a SYNC_REQUEST in +which the STATE flag is set.

+ +

The moderator list is comprised of one or more 32 byte public signature +keys.

+ +

This packet must always be sent after a SHARED_STATE packet, as +the moderator list is validated using data contained within the shared +state.

+ +

SANCTIONS_LIST (0xfd)

+ +

A sanctions list packet payload is structured as follows:

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
2Sanctions List Count
VariableSanctions List
132Packed Sanctions List Credentials
+ +
Sanctions List Entry
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1Type
32Public Signature Key
8Unix Timestamp
32Public Encryption Key
64Signature
+ +
Sanctions Credentials
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
4Version
32Hash (Sha256)
2Checksum
32Public Signature Key
64Signature
+ +

This packet contains information about the sanctions list, including the +number of entries, the sanctions list, and the credentials needed to +validate the sanctions list.

+ +

Sanctions types are defined as an enumerator beginning at zero as +follows:

+ + + + + + + + + + + + + + +
TypeID
OBSERVER0x00
+ +

During a sync response, this packet must be sent after a MOD_LIST +packet, as the sanctions list is validated using the moderator list.

+ +

FRIEND_INVITE (0xfe)

+ +

A friend invite packet payload is structured as follows:

+ + + + + + + + + + + + + + +
LengthContents
1Type
+ +

Used to initiate or respond to a group invite to or from an existing +friend. The invite action is specified by the type field.

+ +

Invite types are defined as an enumerator beginning at zero as follows:

+ + + + + + + + + + + + + + + + + + + + + + +
TypeID
GROUP_INVITE0x00
GROUP_INVITE_ACCEPTED0x01
GROUP_INVITE_CONFIRM0x02
+ +

HS_RESPONSE_ACK (0xff)

+ +

A handshake response ack packet has an empty payload. This packet is +used to send acknowledgement that a lower level toxcore +NET_PACKET_GC_HANDSHAKE packet has been received, which is the +first step in the group handshake protocol. This packet will initiate an +invite request via the INVITE_REQUEST packet.

+ +

Net crypto

+ +

The Tox transport protocol is what Tox uses to establish and send data +securely to friends and provides encryption, ordered delivery, and +perfect forward secrecy. It is a UDP protocol but it is also used when 2 +friends connect over TCP relays.

+ +

The reason the protocol for connections to friends over TCP relays and +direct UDP is the same is for simplicity and so the connection can +switch between both without the peers needing to disconnect and +reconnect. For example two Tox friends might first connect over TCP and +a few seconds later switch to UDP when a direct UDP connection becomes +possible. The opening up of the UDP route or ’hole punching’ is done by +the DHT module and the opening up of a relayed TCP connection is done by +the TCP_connection module. The Tox transport protocol has the job of +connecting two peers (tox friends) safely once a route or communications +link between both is found. Direct UDP is preferred over TCP because it +is direct and isn’t limited by possibly congested TCP relays. Also, a +peer can only connect to another using the Tox transport protocol if +they know the real public key and DHT public key of the peer they want +to connect to. However, both the DHT and TCP connection modules require +this information in order to find and open the route to the peer which +means we assume this information is known by toxcore and has been passed +to net_crypto when the net_crypto connection was created.

+ +

Because this protocol has to work over UDP it must account for possible +packet loss, packets arriving in the wrong order and has to implement +some kind of congestion control. This is implemented above the level at +which the packets are encrypted. This prevents a malicious TCP relay +from disrupting the connection by modifying the packets that go through +it. The packet loss prevention makes it work very well on TCP relays +that we assume may go down at any time as the connection will stay +strong even if there is need to switch to another TCP relay which will +cause some packet loss.

+ +

Before sending the actual handshake packet the peer must obtain a +cookie. This cookie step serves as a way for the receiving peer to +confirm that the peer initiating the connection can receive the +responses in order to prevent certain types of DoS attacks.

+ +

The peer receiving a cookie request packet must not allocate any +resources to the connection. They will simply respond to the packet with +a cookie response packet containing the cookie that the requesting peer +must then use in the handshake to initiate the actual connection.

+ +

The cookie response must be sent back using the exact same link the +cookie request packet was sent from. The reason for this is that if it +is sent back using another link, the other link might not work and the +peer will not be expecting responses from another link. For example, if +a request is sent from UDP with ip port X, it must be sent back by UDP +to ip port X. If it was received from a TCP OOB packet it must be sent +back by a TCP OOB packet via the same relay with the destination being +the peer who sent the request. If it was received from an established +TCP relay connection it must be sent back via that same exact +connection.

+ +

When a cookie request is received, the peer must not use the information +in the request packet for anything, he must not store it, he must only +create a cookie and cookie response from it, then send the created +cookie response packet and forget them. The reason for this is to +prevent possible attacks. For example if a peer would allocate long term +memory for each cookie request packet received then a simple packet +flood would be enough to achieve an effective denial of service attack +by making the program run out of memory.

+ +

cookie request packet (145 bytes):

+ +
[uint8_t 24]
+[Sender's DHT Public key (32 bytes)]
+[Random nonce (24 bytes)]
+[Encrypted message containing:
+    [Sender's real public key (32 bytes)]
+    [padding (32 bytes)]
+    [uint64_t echo id (must be sent back untouched in cookie response)]
+]
+
+ +

Encrypted message is encrypted with sender’s DHT private key, receiver’s +DHT public key and the nonce.

+ +

The packet id for cookie request packets is 24. The request contains the +DHT public key of the sender which is the key used (The DHT private key) +(along with the DHT public key of the receiver) to encrypt the encrypted +part of the cookie packet and a nonce also used to encrypt the encrypted +part of the packet. Padding is used to maintain backwards-compatibility +with previous versions of the protocol. The echo id in the cookie +request must be sent back untouched in the cookie response. This echo id +is how the peer sending the request can be sure that the response +received was a response to the packet that he sent.

+ +

The reason for sending the DHT public key and real public key in the +cookie request is that both are contained in the cookie sent back in the +response.

+ +

Toxcore currently sends 1 cookie request packet every second 8 times +before it kills the connection if there are no responses.

+ +

cookie response packet (161 bytes):

+ +
[uint8_t 25]
+[Random nonce (24 bytes)]
+[Encrypted message containing:
+    [Cookie]
+    [uint64_t echo id (that was sent in the request)]
+]
+
+ +

Encrypted message is encrypted with the exact same symmetric key as the +cookie request packet it responds to but with a different nonce.

+ +

The packet id for cookie request packets is 25. The response contains a +nonce and an encrypted part encrypted with the nonce. The encrypted part +is encrypted with the same key used to decrypt the encrypted part of the +request meaning the expensive shared key generation needs to be called +only once in order to handle and respond to a cookie request packet with +a cookie response.

+ +

The Cookie (see below) and the echo id that was sent in the request are +the contents of the encrypted part.

+ +

The Cookie should be (112 bytes):

+ +
[nonce]
+[encrypted data:
+    [uint64_t time]
+    [Sender's real public key (32 bytes)]
+    [Sender's DHT public key (32 bytes)]
+]
+
+ +

The cookie is a 112 byte piece of data that is created and sent to the +requester as part of the cookie response packet. A peer who wants to +connect to another must obtain a cookie packet from the peer they are +trying to connect to. The only way to send a valid handshake packet to +another peer is to first obtain a cookie from them.

+ +

The cookie contains information that will both prove to the receiver of +the handshake that the peer has received a cookie response and contains +encrypted info that tell the receiver of the handshake packet enough +info to both decrypt and validate the handshake packet and accept the +connection.

+ +

When toxcore is started it generates a symmetric encryption key that it +uses to encrypt and decrypt all cookie packets (using NaCl authenticated +encryption exactly like encryption everywhere else in toxcore). Only the +instance of toxcore that create the packets knows the encryption key +meaning any cookie it successfully decrypts and validates were created +by it.

+ +

The time variable in the cookie is used to prevent cookie packets that +are too old from being used. Toxcore has a time out of 15 seconds for +cookie packets. If a cookie packet is used more than 15 seconds after it +is created toxcore will see it as invalid.

+ +

When responding to a cookie request packet the sender’s real public key +is the known key sent by the peer in the encrypted part of the cookie +request packet and the senders DHT public key is the key used to encrypt +the encrypted part of the cookie request packet.

+ +

When generating a cookie to put inside the encrypted part of the +handshake: One of the requirements to connect successfully to someone +else is that we know their DHT public key and their real long term +public key meaning there is enough information to construct the cookie.

+ +

Handshake packet:

+ +
[uint8_t 26]
+[Cookie]
+[nonce (24 bytes)]
+[Encrypted message containing:
+    [24 bytes base nonce]
+    [session public key of the peer (32 bytes)]
+    [sha512 hash of the entire Cookie sitting outside the encrypted part]
+    [Other Cookie (used by the other to respond to the handshake packet)]
+]
+
+ +

The packet id for handshake packets is 26. The cookie is a cookie +obtained by sending a cookie request packet to the peer and getting a +cookie response packet with a cookie in it. It may also be obtained in +the handshake packet by a peer receiving a handshake packet (Other +Cookie).

+ +

The nonce is a nonce used to encrypt the encrypted part of the handshake +packet. The encrypted part of the handshake packet is encrypted with the +long term keys of both peers. This is to prevent impersonation.

+ +

Inside the encrypted part of the handshake packet there is a ’base +nonce’ and a session public key. The ’base nonce’ is a nonce that the +other should use to encrypt each data packet, adding + 1 to it for each +data packet sent. (first packet is ’base nonce’ + 0, next is ’base +nonce’ + 1, etc. Note that for mathematical operations the nonce is +considered to be a 24 byte number in big endian format). The session key +is the temporary connection public key that the peer has generated for +this connection and it sending to the other. This session key is used so +that the connection has perfect forward secrecy. It is important to save +the private key counterpart of the session public key sent in the +handshake, the public key received by the other and both the received +and sent base nonces as they are used to encrypt/decrypt the data +packets.

+ +

The hash of the cookie in the encrypted part is used to make sure that +an attacker has not taken an older valid handshake packet and then +replaced the cookie packet inside with a newer one which would be bad as +they could replay it and might be able to make a mess.

+ +

The ’Other Cookie’ is a valid cookie that we put in the handshake so +that the other can respond with a valid handshake without having to make +a cookie request to obtain one.

+ +

The handshake packet is sent by both sides of the connection. If a peer +receives a handshake it will check if the cookie is valid, if the +encrypted section decrypts and validates, if the cookie hash is valid, +if long term public key belongs to a known friend. If all these are true +then the connection is considered ’Accepted’ but not ’Confirmed’.

+ +

If there is no existing connection to the peer identified by the long +term public key to set to ’Accepted’, one will be created with that +status. If a connection to such peer with a not yet ’Accepted’ status to +exists, this connection is set to accepted. If a connection with a +’Confirmed’ status exists for this peer, the handshake packet will be +ignored and discarded (The reason for discarding it is that we do not +want slightly late handshake packets to kill the connection) except if +the DHT public key in the cookie contained in the handshake packet is +different from the known DHT public key of the peer. If this happens the +connection will be immediately killed because it means it is no longer +valid and a new connection will be created immediately with the +’Accepted’ status.

+ +

Sometimes toxcore might receive the DHT public key of the peer first +with a handshake packet so it is important that this case is handled and +that the implementation passes the DHT public key to the other modules +(DHT, TCP_connection) because this does happen.

+ +

Handshake packets must be created only once during the connection but +must be sent in intervals until we are sure the other received them. +This happens when a valid encrypted data packet is received and +decrypted.

+ +

The states of a connection:

+ +
    +
  1. +

    Not accepted: Send handshake packets.

    +
    2. +
  2. +

    Accepted: A handshake packet has been received from the other peer +but no encrypted packets: continue (or start) sending handshake +packets because the peer can’t know if the other has received them.

    +
    3. +
  3. +

    Confirmed: A valid encrypted packet has been received from the other +peer: Connection is fully established: stop sending handshake +packets.

    +
    4. +
+ +

Toxcore sends handshake packets every second 8 times and times out the +connection if the connection does not get confirmed (no encrypted packet +is received) within this time.

+ +

Perfect handshake scenario:

+ +
Peer 1                Peer 2
+Cookie request   ->
+                      <- Cookie response
+Handshake packet ->
+                      * accepts connection
+                      <- Handshake packet
+*accepts connection
+Encrypted packet ->   <- Encrypted packet
+*confirms connection  *confirms connection
+       Connection successful.
+Encrypted packets -> <- Encrypted packets
+
+More realistic handshake scenario:
+Peer 1                Peer 2
+Cookie request   ->   *packet lost*
+Cookie request   ->
+                      <- Cookie response
+                      *Peer 2 randomly starts new connection to peer 1
+                      <- Cookie request
+Cookie response  ->
+Handshake packet ->   <- Handshake packet
+*accepts connection   * accepts connection
+Encrypted packet ->   <- Encrypted packet
+*confirms connection  *confirms connection
+       Connection successful.
+Encrypted packets -> <- Encrypted packets
+
+ +

The reason why the handshake is like this is because of certain design +requirements:

+ +
    +
  1. +

    The handshake must not leak the long term public keys of the peers +to a possible attacker who would be looking at the packets but each +peer must know for sure that they are connecting to the right peer +and not an impostor.

    +
    2. +
  2. +

    A connection must be able of being established if only one of the +peers has the information necessary to initiate a connection (DHT +public key of the peer and a link to the peer).

    +
    3. +
  3. +

    If both peers initiate a connection to each other at the same time +the connection must succeed without issues.

    +
    4. +
  4. +

    There must be perfect forward secrecy.

    +
    5. +
  5. +

    Must be resistant to any possible attacks.

    +
    6. +
+ +

Due to how it is designed only one connection is possible at a time +between 2 peers.

+ +

Encrypted +packets:

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x1b)
2uint16_t The last 2 bytes of the nonce used to encrypt this
variablePayload
+ +

The payload is encrypted with the session key and ’base nonce’ set by +the receiver in their handshake + packet number (starting at 0, big +endian math).

+ +

The packet id for encrypted packets is 27. Encrypted packets are the +packets used to send data to the other peer in the connection. Since +these packets can be sent over UDP the implementation must assume that +they can arrive out of order or even not arrive at all.

+ +

To get the key used to encrypt/decrypt each packet in the connection a +peer takes the session public key received in the handshake and the +private key counterpart of the key it sent it the handshake and +generates a shared key from it. This shared key will be identical for +both peers. It is important to note that connection keys must be wiped +when the connection is killed.

+ +

To create an encrypted packet to be sent to the other peer, the data is +encrypted with the shared key for this connection and the base nonce +that the other peer sent in the handshake packet with the total number +of encrypted packets sent in the connection added to it (’base nonce’ + +0 for the first encrypted data packet sent, ’base nonce’ + 1 for the +second, etc. Note that the nonce is treated as a big endian number for +mathematical operations like additions). The 2 byte (uint16_t) number +at the beginning of the encrypted packet is the last 2 bytes of this 24 +byte nonce.

+ +

To decrypt a received encrypted packet, the nonce the packet was +encrypted with is calculated using the base nonce that the peer sent to +the other and the 2 byte number at the beginning of the packet. First we +assume that packets will most likely arrive out of order and that some +will be lost but that packet loss and out of orderness will never be +enough to make the 2 byte number need an extra byte. The packet is +decrypted using the shared key for the connection and the calculated +nonce.

+ +

Toxcore uses the following method to calculate the nonce for each +packet:

+ +
    +
  1. +

    diff = (2 byte number on the packet) - (last 2 bytes of the +current saved base nonce) NOTE: treat the 3 variables as 16 bit +unsigned ints, the result is expected to sometimes roll over.

    +
    2. +
  2. +

    copy saved_base_nonce to temp_nonce.

    +
    3. +
  3. +

    temp_nonce = temp_nonce + diff. temp_nonce is the correct nonce +that can be used to decrypt the packet.

    +
    4. +
  4. +

    DATA_NUM_THRESHOLD = (1/3 of the maximum number that can be stored +in an unsigned 2 bit integer)

    +
    5. +
  5. +

    if decryption succeeds and diff > (DATA_NUM_THRESHOLD * 2) then:

    + +
      +
    • saved_base_nonce = saved_base_nonce + DATA_NUM_THRESHOLD
      • +
    +
    6. +
+ +

First it takes the difference between the 2 byte number on the packet +and the last. Because the 3 values are unsigned 16 bit ints and rollover +is part of the math something like diff = (10 - 65536) means diff is +equal to 11.

+ +

Then it copies the saved base nonce to a temp nonce buffer.

+ +

Then it adds diff to the nonce (the nonce is in big endian format).

+ +

After if decryption was successful it checks if diff was bigger than 2/3 +of the value that can be contained in a 16 bit unsigned int and +increases the saved base nonce by 1/3 of the maximum value if it +succeeded.

+ +

This is only one of many ways that the nonce for each encrypted packet +can be calculated.

+ +

Encrypted packets that cannot be decrypted are simply dropped.

+ +

The reason for exchanging base nonces is because since the key for +encrypting packets is the same for received and sent packets there must +be a cryptographic way to make it impossible for someone to do an attack +where they would replay packets back to the sender and the sender would +think that those packets came from the other peer.

+ +

Data in the encrypted + packets:

+ +
[our recvbuffers buffer_start, (highest packet number handled + 1), (big endian)]
+[uint32_t packet number if lossless, sendbuffer buffer_end if lossy, (big endian)]
+[data]
+
+ +

Encrypted packets may be lossy or lossless. Lossy packets are simply +encrypted packets that are sent to the other. If they are lost, arrive +in the wrong order or even if an attacker duplicates them (be sure to +take this into account for anything that uses lossy packets) they will +simply be decrypted as they arrive and passed upwards to what should +handle them depending on the data id.

+ +

Lossless packets are packets containing data that will be delivered in +order by the implementation of the protocol. In this protocol, the +receiver tells the sender which packet numbers he has received and which +he has not and the sender must resend any packets that are dropped. Any +attempt at doubling packets will cause all (except the first received) +to be ignored.

+ +

Each lossless packet contains both a 4 byte number indicating the +highest packet number received and processed and a 4 byte packet number +which is the packet number of the data in the packet.

+ +

In lossy packets, the layout is the same except that instead of a packet +number, the second 4 byte number represents the packet number of a +lossless packet if one were sent right after. This number is used by the +receiver to know if any packets have been lost. (for example if it +receives 4 packets with numbers (0, 1, 2, 5) and then later a lossy +packet with this second number as: 8 it knows that packets: 3, 4, 6, 7 +have been lost and will request them)

+ +

How the reliability is achieved:

+ +

First it is important to say that packet numbers do roll over, the next +number after 0xFFFFFFFF (maximum value in 4 bytes) is 0. Hence, all the +mathematical operations dealing with packet numbers are assumed to be +done only on unsigned 32 bit integer unless said otherwise. For example +0 - 0xFFFFFFFF would equal to 1 because of the rollover.

+ +

When sending a lossless packet, the packet is created with its packet +number being the number of the last lossless packet created + 1 +(starting at 0). The packet numbers are used for both reliability and in +ordered delivery and so must be sequential.

+ +

The packet is then stored along with its packet number in order for the +peer to be able to send it again if the receiver does not receive it. +Packets are only removed from storage when the receiver confirms they +have received them.

+ +

The receiver receives packets and stores them along with their packet +number. When a receiver receives a packet he stores the packet along +with its packet number in an array. If there is already a packet with +that number in the buffer, the packet is dropped. If the packet number +is smaller than the last packet number that was processed, the packet is +dropped. A processed packet means it was removed from the buffer and +passed upwards to the relevant module.

+ +

Assuming a new connection, the sender sends 5 lossless packets to the +receiver: 0, 1, 2, 3, 4 are the packet numbers sent and the receiver +receives: 3, 2, 0, 2 in that order.

+ +

The receiver will save the packets and discards the second packet with +the number 2, he has: 0, 2, 3 in his buffer. He will pass the first +packet to the relevant module and remove it from the array but since +packet number 1 is missing he will stop there. Contents of the buffer +are now: 2, 3. The receiver knows packet number 1 is missing and will +request it from the sender by using a packet request packet:

+ +

data ids:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IDData
0padding (skipped until we hit a non zero (data id) byte)
1packet request packet (lossy packet)
2connection kill packet (lossy packet)
……
16+reserved for Messenger usage (lossless packets)
192+reserved for Messenger usage (lossy packets)
255reserved for Messenger usage (lossless packet)
+ +

Connection kill packets tell the other that the connection is over.

+ +

Packet numbers are the first byte of data in the packet.

+ +

packet request packet:

+ +
[uint8_t (1)][uint8_t num][uint8_t num][uint8_t num]...[uint8_t num]
+
+ +

Packet request packets are used by one side of the connection to request +packets from the other. To create a full packet request packet, the one +requesting the packet takes the last packet number that was processed +(sent to the relevant module and removed from the array (0 in the +example above)). Subtract the number of the first missing packet from +that number (1 - 0) = 1. Which means the full packet to request packet +number 1 will look like:

+ +
[uint32_t 1]
+[uint32_t 0]
+[uint8_t 1][uint8_t 1]
+
+ +

If packet number 4 was being requested as well, take the difference +between the packet number and the last packet number being requested (4

+
    +
  • +

    1) = 3. So the packet will look like:

    + +

    [uint32_t 1] + [uint32_t 0] + [uint8_t 1][uint8_t 1][uint8_t 3]

    +
    • +
+ +

But what if the number is greater than 255? Let’s say the peer needs to +request packets 3, 6, 1024, the packet will look like:

+ +
[uint32_t 1]
+[uint32_t 2]
+[uint8_t 1][uint8_t 3][uint8_t 3][uint8_t 0][uint8_t 0][uint8_t 0][uint8_t 253]
+
+ +

Each 0 in the packet represents adding 255 until a non 0 byte is reached +which is then added and the resulting requested number is what is left.

+ +

This request is designed to be small when requesting packets in real +network conditions where the requested packet numbers will be close to +each other. Putting each requested 4 byte packet number would be very +simple but would make the request packets unnecessarily large which is +why the packets look like this.

+ +

When a request packet is received, it will be decoded and all packets in +between the requested packets will be assumed to be successfully +received by the other.

+ +

Packet request packets are sent at least every 1 second in toxcore and +more when packets are being received.

+ +

The current formula used is (note that this formula is likely +sub-optimal):

+ +
REQUEST_PACKETS_COMPARE_CONSTANT = 50.0 double request_packet_interval =
+(REQUEST_PACKETS_COMPARE_CONSTANT /
+(((double)num_packets_array(&conn->recv_array) + 1.0) / (conn->packet_recv_rate
++ 1.0)));
+
+ +

num_packets_array(&conn->recv_array) returns the difference between +the highest packet number received and the last one handled. In the +toxcore code it refers to the total size of the current array (with the +holes which are the placeholders for not yet received packets that are +known to be missing).

+ +

conn->packet_recv_rate is the number of data packets successfully +received per second.

+ +

This formula was created with the logic that the higher the ’delay’ in +packets (num_packets_array(&conn->recv_array)) vs the speed of packets +received, the more request packets should be sent.

+ +

Requested packets are resent every time they can be resent as in they +will obey the congestion control and not bypass it. They are resent +once, subsequent request packets will be used to know if the packet was +received or if it should be resent.

+ +

The ping or rtt (round trip time) between two peers can be calculated by +saving the time each packet was sent and taking the difference between +the time the latest packet confirmed received by a request packet was +sent and the time the request packet was received. The rtt can be +calculated for every request packet. The lowest one (for all packets) +will be the closest to the real ping.

+ +

This ping or rtt can be used to know if a request packet that requests a +packet we just sent should be resent right away or we should wait or not +for the next one (to know if the other side actually had time to receive +the packet).

+ +

The congestion control algorithm has the goal of guessing how many +packets can be sent through the link every second before none can be +sent through anymore. How it works is basically to send packets faster +and faster until none can go through the link and then stop sending them +faster than that.

+ +

Currently the congestion control uses the following formula in toxcore +however that is probably not the best way to do it.

+ +

The current formula is to take the difference between the current size +of the send queue and the size of the send queue 1.2 seconds ago, take +the total number of packets sent in the last 1.2 seconds and subtract +the previous number from it.

+ +

Then divide this number by 1.2 to get a packet speed per second. If this +speed is lower than the minimum send rate of 8 packets per second, set +it to 8.

+ +

A congestion event can be defined as an event when the number of +requested packets exceeds the number of packets the congestion control +says can be sent during this frame. If a congestion event occurred +during the last 2 seconds, the packet send rate of the connection is set +to the send rate previously calculated, if not it is set to that send +rate times 1.25 in order to increase the speed.

+ +

Like I said this isn’t perfect and a better solution can likely be found +or the numbers tweaked.

+ +

To fix the possible issue where it would be impossible to send very low +bandwidth data like text messages when sending high bandwidth data like +files it is possible to make priority packets ignore the congestion +control completely by placing them into the send packet queue and +sending them even if the congestion control says not to. This is used in +toxcore for all non file transfer packets to prevent file transfers from +preventing normal message packets from being sent.

+ +

network.txt

+ +

The network module is the lowest file in toxcore that everything else +depends on. This module is basically a UDP socket wrapper, serves as the +sorting ground for packets received by the socket, initializes and +uninitializes the socket. It also contains many socket, networking +related and some other functions like a monotonic time function used by +other toxcore modules.

+ +

Things of note in this module are the maximum UDP packet size define +(MAX_UDP_PACKET_SIZE) which sets the maximum UDP packet size toxcore +can send and receive. The list of all UDP packet ids: NET_PACKET_. UDP +packet ids are the value of the first byte of each UDP packet and is how +each packet gets sorted to the right module that can handle it. +networking_registerhandler() is used by higher level modules in order +to tell the network object which packets to send to which module via a +callback.

+ +

It also contains datastructures used for ip addresses in toxcore. IP4 +and IP6 are the datastructures for ipv4 and ipv6 addresses, IP is the +datastructure for storing either (the family can be set to AF_INET +(ipv4) or AF_INET6 (ipv6). It can be set to another value like +TCP_ONION_FAMILY, TCP_INET, TCP_INET6 or TCP_FAMILY which are +invalid values in the network modules but valid values in some other +module and denote a special type of ip) and IP_Port stores an IP +datastructure with a port.

+ +

Since the network module interacts directly with the underlying +operating system with its socket functions it has code to make it work +on windows, linux, etc… unlike most modules that sit at a higher +level.

+ +

The network module currently uses the polling method to read from the +UDP socket. The networking_poll() function is called to read all the +packets from the socket and pass them to the callbacks set using the +networking_registerhandler() function. The reason it uses polling is +simply because it was easier to write it that way, another method would +be better here.

+ +

The goal of this module is to provide an easy interface to a UDP socket +and other networking related functions.

+ +

Onion

+ +

The goal of the onion module in Tox is to prevent peers that are not +friends from finding out the temporary DHT public key from a known long +term public key of the peer and to prevent peers from discovering the +long term public key of peers when only the temporary DHT key is known.

+ +

It makes sure only friends of a peer can find it and connect to it and +indirectly makes sure non friends cannot find the ip address of the peer +when knowing the Tox address of the friend.

+ +

The only way to prevent peers in the network from associating the +temporary DHT public key with the long term public key is to not +broadcast the long term key and only give others in the network that are +not friends the DHT public key.

+ +

The onion lets peers send their friends, whose real public key they know +as it is part of the Tox ID, their DHT public key so that the friends +can then find and connect to them without other peers being able to +identify the real public keys of peers.

+ +

So how does the onion work?

+ +

The onion works by enabling peers to announce their real public key to +peers by going through the onion path. It is like a DHT but through +onion paths. In fact it uses the DHT in order for peers to be able to +find the peers with ids closest to their public key by going through +onion paths.

+ +

In order to announce its real public key anonymously to the Tox network +while using the onion, a peer first picks 3 random nodes that it knows +(they can be from anywhere: the DHT, connected TCP relays or nodes found +while finding peers with the onion). The nodes should be picked in a way +that makes them unlikely to be operated by the same person perhaps by +looking at the ip addresses and looking if they are in the same subnet +or other ways. More research is needed to make sure nodes are picked in +the safest way possible.

+ +

The reason for 3 nodes is that 3 hops is what they use in Tor and other +anonymous onion based networks.

+ +

These nodes are referred to as nodes A, B and C. Note that if a peer +cannot communicate via UDP, its first peer will be one of the TCP relays +it is connected to, which will be used to send its onion packet to the +network.

+ +

TCP relays can only be node A or the first peer in the chain as the TCP +relay is essentially acting as a gateway to the network. The data sent +to the TCP Client module to be sent as a TCP onion packet by the module +is different from the one sent directly via UDP. This is because it +doesn’t need to be encrypted (the connection to the TCP relay server is +already encrypted).

+ +

First I will explain how communicating via onion packets work.

+ +

Note: nonce is a 24 byte nonce. The nested nonces are all the same as +the outer nonce.

+ +

Onion packet (request):

+ +

Initial (TCP) data sent as the data of an onion packet through the TCP +client module:

+ +
    +
  • +

    IP_Port of node B

    +
    • +
  • +

    A random public key PK1

    +
    • +
  • +

    Encrypted with the secret key SK1 and the public key of Node B and +the nonce:

    + +
      +
    • +

      IP_Port of node C

      +
      • +
    • +

      A random public key PK2

      +
      • +
    • +

      Encrypted with the secret key SK2 and the public key of Node C +and the nonce:

      + +
        +
      • +

        IP_Port of node D

        +
        • +
      • +

        Data to send to Node D

        +
        • +
      +
      • +
    +
    • +
+ +

Initial (UDP) (sent from us to node A):

+ +
    +
  • +

    uint8_t (0x80) packet id

    +
    • +
  • +

    Nonce

    +
    • +
  • +

    Our temporary DHT public key

    +
    • +
  • +

    Encrypted with our temporary DHT secret key and the public key of +Node A and the nonce:

    + +
      +
    • +

      IP_Port of node B

      +
      • +
    • +

      A random public key PK1

      +
      • +
    • +

      Encrypted with the secret key SK1 and the public key of Node B +and the nonce:

      + +
        +
      • +

        IP_Port of node C

        +
        • +
      • +

        A random public key PK2

        +
        • +
      • +

        Encrypted with the secret key SK2 and the public key of Node +C and the nonce:

        + +
          +
        • +

          IP_Port of node D

          +
          • +
        • +

          Data to send to Node D

          +
          • +
        +
        • +
      +
      • +
    +
    • +
+ +

(sent from node A to node B):

+ +
    +
  • +

    uint8_t (0x81) packet id

    +
    • +
  • +

    Nonce

    +
    • +
  • +

    A random public key PK1

    +
    • +
  • +

    Encrypted with the secret key SK1 and the public key of Node B and +the nonce:

    + +
      +
    • +

      IP_Port of node C

      +
      • +
    • +

      A random public key PK2

      +
      • +
    • +

      Encrypted with the secret key SK2 and the public key of Node C +and the nonce:

      + +
        +
      • +

        IP_Port of node D

        +
        • +
      • +

        Data to send to Node D

        +
        • +
      +
      • +
    +
    • +
  • +

    Nonce

    +
    • +
  • +

    Encrypted with temporary symmetric key of Node A and the nonce:

    + +
      +
    • IP_Port (of us)
      • +
    +
    • +
+ +

(sent from node B to node C):

+ +
    +
  • +

    uint8_t (0x82) packet id

    +
    • +
  • +

    Nonce

    +
    • +
  • +

    A random public key PK1

    +
    • +
  • +

    Encrypted with the secret key SK1 and the public key of Node C and +the nonce:

    + +
      +
    • +

      IP_Port of node D

      +
      • +
    • +

      Data to send to Node D

      +
      • +
    +
    • +
  • +

    Nonce

    +
    • +
  • +

    Encrypted with temporary symmetric key of Node B and the nonce:

    + +
      +
    • +

      IP_Port (of Node A)

      +
      • +
    • +

      Nonce

      +
      • +
    • +

      Encrypted with temporary symmetric key of Node A and the nonce:

      + +
        +
      • IP_Port (of us)
        • +
      +
      • +
    +
    • +
+ +

(sent from node C to node D):

+ +
    +
  • +

    Data to send to Node D

    +
    • +
  • +

    Nonce

    +
    • +
  • +

    Encrypted with temporary symmetric key of Node C and the nonce:

    + +
      +
    • +

      IP_Port (of Node B)

      +
      • +
    • +

      Nonce

      +
      • +
    • +

      Encrypted with temporary symmetric key of Node B and the nonce:

      + +
        +
      • +

        IP_Port (of Node A)

        +
        • +
      • +

        Nonce

        +
        • +
      • +

        Encrypted with temporary symmetric key of Node A and the +nonce:

        + +
          +
        • IP_Port (of us)
          • +
        +
        • +
      +
      • +
    +
    • +
+ +

Onion packet (response):

+ +

initial (sent from node D to node C):

+ +
    +
  • +

    uint8_t (0x8c) packet id

    +
    • +
  • +

    Nonce

    +
    • +
  • +

    Encrypted with the temporary symmetric key of Node C and the nonce:

    + +
      +
    • +

      IP_Port (of Node B)

      +
      • +
    • +

      Nonce

      +
      • +
    • +

      Encrypted with the temporary symmetric key of Node B and the +nonce:

      + +
        +
      • +

        IP_Port (of Node A)

        +
        • +
      • +

        Nonce

        +
        • +
      • +

        Encrypted with the temporary symmetric key of Node A and the +nonce:

        + +
          +
        • IP_Port (of us)
          • +
        +
        • +
      +
      • +
    +
    • +
  • +

    Data to send back

    +
    • +
+ +

(sent from node C to node B):

+ +
    +
  • +

    uint8_t (0x8d) packet id

    +
    • +
  • +

    Nonce

    +
    • +
  • +

    Encrypted with the temporary symmetric key of Node B and the nonce:

    + +
      +
    • +

      IP_Port (of Node A)

      +
      • +
    • +

      Nonce

      +
      • +
    • +

      Encrypted with the temporary symmetric key of Node A and the +nonce:

      + +
        +
      • IP_Port (of us)
        • +
      +
      • +
    +
    • +
  • +

    Data to send back

    +
    • +
+ +

(sent from node B to node A):

+ +
    +
  • +

    uint8_t (0x8e) packet id

    +
    • +
  • +

    Nonce

    +
    • +
  • +

    Encrypted with the temporary symmetric key of Node A and the nonce:

    + +
      +
    • IP_Port (of us)
      • +
    +
    • +
  • +

    Data to send back

    +
    • +
+ +

(sent from node A to us):

+ +
    +
  • Data to send back
    • +
+ +

Each packet is encrypted multiple times so that only node A will be able +to receive and decrypt the first packet and know where to send it to, +node B will only be able to receive that decrypted packet, decrypt it +again and know where to send it and so on. You will also notice a piece +of encrypted data (the sendback) at the end of the packet that grows +larger and larger at every layer with the IP of the previous node in it. +This is how the node receiving the end data (Node D) will be able to +send data back.

+ +

When a peer receives an onion packet, they will decrypt it, encrypt the +coordinates (IP/port) of the source along with the already existing +encrypted data (if it exists) with a symmetric key known only by the +peer and only refreshed every hour (in toxcore) as a security measure to +force expire paths.

+ +

Here’s a diagram how it works:

+ +
peer
+  -> [onion1[onion2[onion3[data]]]] -> Node A
+  -> [onion2[onion3[data]]][sendbackA] -> Node B
+  -> [onion3[data]][sendbackB[sendbackA]] -> Node C
+  -> [data][SendbackC[sendbackB[sendbackA]]]-> Node D (end)
+
+Node D
+  -> [SendbackC[sendbackB[sendbackA]]][response] -> Node C
+  -> [sendbackB[sendbackA]][response] -> Node B
+  -> [sendbackA][response] -> Node A
+  -> [response] -> peer
+
+ +

The random public keys in the onion packets are temporary public keys +generated for and used for that onion path only. This is done in order +to make it difficult for others to link different paths together. Each +encrypted layer must have a different public key. This is the reason why +there are multiple keys in the packet definintions above.

+ +

The nonce is used to encrypt all the layers of encryption. This 24 byte +nonce should be randomly generated. If it isn’t randomly generated and +has a relation to nonces used for other paths it could be possible to +tie different onion paths together.

+ +

The IP_Port is an ip and port in packed +format:

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1TOX_AF_INET (2) for IPv4 or TOX_AF_INET6 (10) for IPv6
4 \| 16IP address (4 bytes if IPv4, 16 if IPv6)
12 \| 0Zeroes
2uint16_t Port
+ +

If IPv4 the format is padded with 12 bytes of zeroes so that both IPv4 +and IPv6 have the same stored size.

+ +

The IP_Port will always end up being of size 19 bytes. This is to make +it hard to know if an ipv4 or ipv6 ip is in the packet just by looking +at the size. The 12 bytes of zeros when ipv4 must be set to 0 and not +left uninitialized as some info may be leaked this way if it stays +uninitialized. All numbers here are in big endian format.

+ +

The IP_Port in the sendback data can be in any format as long as the +length is 19 bytes because only the one who writes it can decrypt it and +read it, however, using the previous format is recommended because of +code reuse. The nonce in the sendback data must be a 24 byte nonce.

+ +

Each onion layers has a different packed id that identifies it so that +an implementation knows exactly how to handle them. Note that any data +being sent back must be encrypted, appear random and not leak +information in any way as all the nodes in the path will see it.

+ +

If anything is wrong with the received onion packets (decryption fails) +the implementation should drop them.

+ +

The implementation should have code for each different type of packet +that handles it, adds (or decrypts) a sendback and sends it to the next +peer in the path. There are a lot of packets but an implementation +should be very straightforward.

+ +

Note that if the first node in the path is a TCP relay, the TCP relay +must put an identifier (instead of an IP/Port) in the sendback so that +it knows that any response should be sent to the appropriate peer +connected to the TCP relay.

+ +

This explained how to create onion packets and how they are sent back. +Next is what is actually sent and received on top of these onion packets +or paths.

+ +

Note: nonce is a 24 byte nonce.

+ +

announce request packet:

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x83)
24Nonce
32A public key (real or temporary)
?Payload
+ +

The public key is our real long term public key if we want to announce +ourselves, a temporary one if we are searching for friends.

+ +

The payload is encrypted with the secret key part of the sent public +key, the public key of Node D and the nonce, and +contains:

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
32Ping ID
32Public key we are searching for
32Public key that we want those sending back data packets to use
8Data to send back in response
+ +

If the ping id is zero, respond with an announce response packet.

+ +

If the ping id matches the one the node sent in the announce response +and the public key matches the one being searched for, add the part used +to send data to our list. If the list is full make it replace the +furthest entry.

+ +

data to route request packet:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x85)
32Public key of destination node
24Nonce
32Temporary just generated public key
variablePayload
+ +

The payload is encrypted with that temporary secret key and the nonce +and the public key from the announce response packet of the destination +node. If Node D contains the ret data for the node, it sends the stuff +in this packet as a data to route response packet to the right node.

+ +

The data in the previous packet is in format:

+ + + + + + + + + + + + + + + + + + +
LengthContents
32Real public key of sender
variablePayload
+ +

The payload is encrypted with real secret key of the sender, the nonce +in the data packet and the real public key of the receiver:

+ + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t id
variableData (optional)
+ +

Data sent to us:

+ +

announce response packet:

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x84)
8Data to send back in response
24Nonce
variablePayload
+ +

The payload is encrypted with the DHT secret key of Node D, the public +key in the request and the nonce:

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t is_stored
32Ping ID or Public Key
variableMaximum of 4 nodes in packed node format (see DHT)
+ +

The packet contains a ping ID if is_stored is 0 or 2, or the public +key that must be used to send data packets if is_stored is 1.

+ +

If the is_stored is not 0, it means the information to reach the +public key we are searching for is stored on this node. is_stored is 2 +as a response to a peer trying to announce himself to tell the peer that +he is currently announced successfully.

+ +

data to route response packet:

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x86)
24Nonce
32Temporary just generated public key
variablePayload
+ +

The payload is encrypted with that temporary secret key, the nonce and +the public key from the announce response packet of the destination +node.

+ +

There are 2 types of request packets and 2 ’response’ packets to go with +them. The announce request is used to announce ourselves to a node and +announce response packet is used by the node to respond to this packet. +The data to route request packet is a packet used to send packets +through the node to another peer that has announced itself and that we +have found. The data to route response packet is what the node +transforms this packet into.

+ +

To announce ourselves to the network we must first find, using announce +packets, the peers with the DHT public key closest to our real public +key. We must then announce ourselves to these peers. Friends will then +be able to send messages to us using data to route packets by sending +them to these peers. To find the peers we have announced ourselves to, +our friends will find the peers closest to our real public key and ask +them if they know us. They will then be able to use the peers that know +us to send us some messages that will contain their DHT public key +(which we need to know to connect directly to them), TCP relays that +they are connected to (so we can connect to them with these relays if we +need to) and some DHT peers they are connected to (so we can find them +faster in the DHT).

+ +

Announce request packets are the same packets used slightly differently +if we are announcing ourselves or searching for peers that know one of +our friends.

+ +

If we are announcing ourselves we must put our real long term public key +in the packet and encrypt it with our long term private key. This is so +the peer we are announcing ourselves to can be sure that we actually own +that public key. If we are looking for peers we use a temporary public +key used only for packets looking for that peer in order to leak as +little information as possible. The ping_id is a 32 byte number which +is sent to us in the announce response and we must send back to the peer +in another announce request. This is done in order to prevent people +from easily announcing themselves many times as they have to prove they +can respond to packets from the peer before the peer will let them +announce themselves. This ping_id is set to 0 when none is known.

+ +

The public key we are searching for is set to our long term public key +when announcing ourselves and set to the long term public key of the +friend we are searching for if we are looking for peers.

+ +

When announcing ourselves, the public key we want others to use to send +us data back is set to a temporary public key and we use the private key +part of this key to decrypt packet routing data sent to us. This public +key is to prevent peers from saving old data to route packets from +previous sessions and be able to replay them in future Tox sessions. +This key is set to zero when searching for peers.

+ +

The sendback data is an 8 byte number that will be sent back in the +announce packet response. Its goal is to be used to learn which announce +request packet the response is responding to, and hence its location in +the unencrypted part of the response. This is needed in toxcore to find +and check info about the packet in order to decrypt it and handle it +correctly. Toxcore uses it as an index to its special ping_array.

+ +

Why don’t we use different packets instead of having one announce packet +request and one response that does everything? It makes it a lot more +difficult for possible attackers to know if we are merely announcing +ourselves or if we are looking for friends as the packets for both look +the same and are the same size.

+ +

The unencrypted part of an announce response packet contains the +sendback data, which was sent in the request this packet is responding +to and a 24 byte random nonce used to encrypt the encrypted part.

+ +

The is_stored number is set to either 0, 1 or 2. 0 means that the +public key that was being searched in the request isn’t stored or known +by this peer. 1 means that it is and 2 means that we are announced +successfully at that node. Both 1 and 2 are needed so that when clients +are restarted it is possible to reannounce without waiting for the +timeout of the previous announce. This would not otherwise be possible +as a client would receive response 1 without a ping_id which is needed +in order to reannounce successfully.

+ +

When the is_stored number is 0 or 2, the next 32 bytes is a ping_id. +When is_stored is 1 it corresponds to a public key (the send back data +public key set by the friend in their announce request) that must be +used to encrypt and send data to the friend.

+ +

Then there is an optional maximum 4 nodes, in DHT packed nodes format +(see DHT), attached to the response which denote the 4 DHT peers with +the DHT public keys closest to the searched public key in the announce +request known by the peer (see DHT). To find these peers, toxcore uses +the same function as is used to find peers for get node DHT responses. +Peers wanting to announce themselves or searching for peers that ’know’ +their friends will recursively query closer and closer peers until they +find the closest possible and then either announce themselves to them or +just ping them every once in a while to know if their friend can be +contacted. Note that the distance function used for this is the same as +the Tox DHT.

+ +

Data to route request packets are packets used to send data directly to +another peer via a node that knows that peer. The public key is the +public key of the final destination where we want the packet to be sent +(the real public key of our friend). The nonce is a 24 byte random nonce +and the public key is a random temporary public key used to encrypt the +data in the packet and, if possible, only to send packets to this friend +(we want to leak as little info to the network as possible so we use +temp public keys as we don’t want a peer to see the same public keys and +be able to link things together). The data is encrypted data that we +want to send to the peer with the public key.

+ +

The route response packets are just the last elements (nonce, public +key, encrypted data) of the data to route request packet copied into a +new packet and sent to the appropriate destination.

+ +

To handle onion announce packets, toxcore first receives an announce +packet and decrypts it.

+ +

Toxcore generates ping_ids by taking a 32 byte sha hash of the current +time, some secret bytes generated when the instance is created, the +current time divided by a 300 second timeout, the public key of the +requester and the source ip/port that the packet was received from. +Since the ip/port that the packet was received from is in the ping_id, +the announce packets being sent with a ping id must be sent using the +same path as the packet that we received the ping_id from or +announcing will fail.

+ +

The reason for this 300 second timeout in toxcore is that it gives a +reasonable time (300 to 600 seconds) for peers to announce themselves.

+ +

Toxcore generates 2 different ping ids, the first is generated with the +current time (divided by 300) and the second with the current time + 300 +(divided by 300). The two ping ids are then compared to the ping ids in +the received packets. The reason for doing this is that storing every +ping id received might be expensive and leave us vulnerable to a DoS +attack, this method makes sure that the other cannot generate ping_ids +and must ask for them. The reason for the 2 ping_ids is that we want +to make sure that the timeout is at least 300 seconds and cannot be 0.

+ +

If one of the two ping ids is equal to the ping id in the announce +request, the sendback data public key and the sendback data are stored +in the datastructure used to store announced peers. If the +implementation has a limit to how many announced entries it can store, +it should only store the entries closest (determined by the DHT distance +function) to its DHT public key. If the entry is already there, the +information will simply be updated with the new one and the timeout will +be reset for that entry.

+ +

Toxcore has a timeout of 300 seconds for announce entries after which +they are removed which is long enough to make sure the entries don’t +expire prematurely but not long enough for peers to stay announced for +extended amounts of time after they go offline.

+ +

Toxcore will then copy the 4 DHT nodes closest to the public key being +searched to a new packet (the response).

+ +

Toxcore will look if the public key being searched is in the +datastructure. If it isn’t it will copy the second generated ping_id +(the one generated with the current time plus 300 seconds) to the +response, set the is_stored number to 0 and send the packet back.

+ +

If the public key is in the datastructure, it will check whether the +public key that was used to encrypt the announce packet is equal to the +announced public key, if it isn’t then it means that the peer is +searching for a peer and that we know it. This means the is_stored is +set to 1 and the sending back data public key in the announce entry is +copied to the packet.

+ +

If it (key used to encrypt the announce packet) is equal (to the +announced public key which is also the ’public key we are searching for’ +in the announce packet) meaning the peer is announcing itself and an +entry for it exists, the sending back data public key is checked to see +if it equals the one in the packet. If it is not equal it means that it +is outdated, probably because the announcing peer’s toxcore instance was +restarted and so their is_stored is set to 0, if it is equal it means +the peer is announced correctly so the is_stored is set to 2. The +second generated ping_id is then copied to the packet.

+ +

Once the packet is contructed a random 24 byte nonce is generated, the +packet is encrypted (the shared key used to decrypt the request can be +saved and used to encrypt the response to save an expensive key +derivation operation), the data to send back is copied to the +unencrypted part and the packet is sent back as an onion response +packet.

+ +

In order to announce itself using onion announce packets toxcore first +takes DHT peers, picks random ones and builds onion paths with them by +saving 3 nodes, calling it a path, generating some keypairs for +encrypting the onion packets and using them to send onion packets. If +the peer is only connected with TCP, the initial nodes will be bootstrap +nodes and connected TCP relays (for the first peer in the path). Once +the peer is connected to the onion he can fill up his list of known +peers with peers sent in announce responses if needed.

+ +

Onion paths have different timeouts depending on whether the path is +confirmed or unconfirmed. Unconfirmed paths (paths that core has never +received any responses from) have a timeout of 4 seconds with 2 tries +before they are deemed non working. This is because, due to network +conditions, there may be a large number of newly created paths that do +not work and so trying them a lot would make finding a working path take +much longer. The timeout for a confirmed path (from which a response was +received) is 10 seconds with 4 tries without a response. A confirmed +path has a maximum lifetime of 1200 seconds to make possible +deanonimization attacks more difficult.

+ +

Toxcore saves a maximum of 12 paths: 6 paths are reserved for announcing +ourselves and 6 others are used to search for friends. This may not be +the safest way (some nodes may be able to associate friends together) +however it is much more performant than having different paths for each +friend. The main benefit is that the announcing and searching are done +with different paths, which makes it difficult to know that peer with +real public key X is friends with Y and Z. More research is needed to +find the best way to do this. At first toxcore did have different paths +for each friend, however, that meant that each friend path was almost +never used (and checked). When using a low amount of paths for searching +there is less resources needed to find good paths. 6 paths are used +because 4 was too low and caused some performance issues because it took +longer to find some good paths at the beginning because only 4 could be +tried at a time. A too high number meanwhile would mean each path is +used (and tested) less. The reason why the numbers are the same for both +types of paths is for code simplification purposes.

+ +

To search/announce itself to peers, toxcore keeps the 8 closest peers +(12 for announcing) to each key it is searching (or announcing itself +to). To populate these it starts by sending announce requests to random +peers for all the public keys it is searching for. It then recursively +searches closer and closer peers (DHT distance function) until it no +longer finds any. It is important to make sure it is not too aggressive +at searching the peers as some might no longer be online but peers might +still send announce responses with their information. Toxcore keeps +lists of last pinged nodes for each key searched so as not to ping dead +nodes too aggressively.

+ +

Toxcore decides if it will send an announce packet to one of the 4 peers +in the announce response by checking if the peer would be stored as one +of the stored closest peers if it responded; if it would not be it +doesn’t send an announce request, if it would be it sends one.

+ +

Peers are only put in the closest peers array if they respond to an +announce request. If the peers fail to respond to 3 announce requests +they are deemed timed out and removed. When sending an announce request +to a peer to which we have been announcing ourselves for at least 90 +seconds and which has failed to respond to the previous 2 requests, +toxcore uses a random path for the request. This reduces the chances +that a good node will be removed due to bad paths.

+ +

The reason for the numbers of peers being 8 and 12 is that lower numbers +might make searching for and announcing too unreliable and a higher +number too bandwidth/resource intensive.

+ +

Toxcore uses ping_array (see ping_array) for the 8 byte sendback +data in announce packets to store information that it will need to +handle the response (key to decrypt it, why was it sent? (to announce +ourselves or to search? For what key? and some other info)). For +security purposes it checks to make sure the packet was received from +the right ip/port and checks if the key in the unencrypted part of the +packet is the right public key.

+ +

For peers we are announcing ourselves to, if we are not announced to +them toxcore tries every 3 seconds to announce ourselves to them until +they return that we have announced ourselves to them, then initially +toxcore sends an announce request packet every 15 seconds to see if we +are still announced and reannounce ourselves at the same time. Toxcore +sends every announce packet with the ping_id previously received from +that peer with the same path (if possible). Toxcore use a timeout of 120 +seconds rather than 15 seconds if we have been announcing to the peer +for at least 90 seconds, and the onion path we are are using for the +peer has also been alive for at least 90 seconds, and we have not been +waiting for at least 15 seconds for a response to a request sent to the +peer, nor for at least 10 seconds for a response to a request sent via +the path. The timeout of at most 120 seconds means a ping_id received +in the last packet will not have had time to expire (300 second minimum +timeout) before it is resent 120 seconds later.

+ +

For friends this is slightly different. It is important to start +searching for friends after we are fully announced. Assuming a perfect +network, we would only need to do a search for friend public keys only +when first starting the instance (or going offline and back online) as +peers starting up after us would be able to find us immediately just by +searching for us. If we start searching for friends after we are +announced we prevent a scenario where 2 friends start their clients at +the same time but are unable to find each other right away because they +start searching for each other while they have not announced themselves.

+ +

For this reason, after the peer is announced successfully, for 17 +seconds announce packets are sent aggressively every 3 seconds to each +known close peer (in the list of 8 peers) to search aggressively for +peers that know the peer we are searching for.

+ +

After this, toxcore sends requests once per 15 seconds initially, then +uses linear backoff to increase the interval. In detail, the interval +used when searching for a given friend is at least 15 and at most 2400 +seconds, and within these bounds is calculated as one quarter of the +time since we began searching for the friend, or since the friend was +last seen. For this purpose, a friend is considered to be seen when some +peer reports that the friend is announced, or we receive a DHT Public +Key packet from the friend, or we obtain a new DHT key for them from a +group, or a friend connection for the friend goes offline.

+ +

There are other ways this could be done and which would still work but, +if making your own implementation, keep in mind that these are likely +not the most optimized way to do things.

+ +

If we find peers (more than 1) that know a friend we will send them an +onion data packet with our DHT public key, up to 2 TCP relays we are +connected to and 2 DHT peers close to us to help the friend connect back +to us.

+ +

Onion data packets are packets sent as the data of data to route +packets.

+ +

Onion data packets:

+ + + + + + + + + + + + + + + + + + +
LengthContents
32Long term public key of sender
variablePayload
+ +

The payload is encrypted with long term private key of the sender, the +long term public key of the receiver and the nonce used in the data to +route request packet used to send this onion data packet (shaves off 24 +bytes).

+ +

DHT public key packet:

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x9c)
8uint64_t no_replay
32Our DHT public key
[39, 204]Maximum of 4 nodes in packed format
+ +

The packet will only be accepted if the no_replay number is greater +than the no_replay number in the last packet received.

+ +

The nodes sent in the packet comprise 2 TCP relays to which we are +connected (or fewer if there are not 2 available) and a number of DHT +nodes from our Close List, with the total number of nodes sent being at +most 4. The nodes chosen from the Close List are those closest in DHT +distance to us. This allows the friend to find us more easily in the +DHT, or to connect to us via a TCP relay.

+ +

Why another round of encryption? We have to prove to the receiver that +we own the long term public key we say we own when sending them our DHT +public key. Friend requests are also sent using onion data packets but +their exact format is explained in Messenger.

+ +

The no_replay number is protection if someone tries to replay an older +packet and should be set to an always increasing number. It is 8 bytes +so you should set a high resolution monotonic time as the value.

+ +

We send this packet every 30 seconds if there is more than one peer (in +the 8) that says they our friend is announced on them. This packet can +also be sent through the DHT module as a DHT request packet (see DHT) if +we know the DHT public key of the friend and are looking for them in the +DHT but have not connected to them yet. 30 second is a reasonable +timeout to not flood the network with too many packets while making sure +the other will eventually receive the packet. Since packets are sent +through every peer that knows the friend, resending it right away +without waiting has a high likelihood of failure as the chances of +packet loss happening to all (up to to 8) packets sent is low.

+ +

When sent as a DHT request packet (this is the data sent in the DHT +request packet):

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t (0x9c)
32Long term public key of sender
24Nonce
variableEncrypted payload
+ +

The payload is encrypted with long term private key of sender, the long +term public key of receiver and the nonce, and contains the DHT public +key packet.

+ +

When sent as a DHT request packet the DHT public key packet is (before +being sent as the data of a DHT request packet) encrypted with the long +term keys of both the sender and receiver and put in that format. This +is done for the same reason as the double encryption of the onion data +packet.

+ +

Toxcore tries to resend this packet through the DHT every 20 seconds. 20 +seconds is a reasonable resend rate which isn’t too aggressive.

+ +

Toxcore has a DHT request packet handler that passes received DHT public +key packets from the DHT module to this module.

+ +

If we receive a DHT public key packet, we will first check if the DHT +packet is from a friend, if it is not from a friend, it will be +discarded. The no_replay will then be checked to see if it is good and +no packet with a lower one was received during the session. The DHT key, +the TCP nodes in the packed nodes and the DHT nodes in the packed nodes +will be passed to their relevant modules. The fact that we have the DHT +public key of a friend means this module has achieved its goal.

+ +

If a friend is online and connected to us, the onion will stop all of +its actions for that friend. If the peer goes offline it will restart +searching for the friend as if toxcore was just started.

+ +

If toxcore goes offline (no onion traffic for 75 seconds) toxcore will +aggressively reannounce itself and search for friends as if it was just +started.

+ +

Ping array

+ +

Ping array is an array used in toxcore to store data for pings. It +enables the storage of arbitrary data that can then be retrieved later +by passing the 8 byte ping id that was returned when the data was +stored. It also frees data from pings that are older than a ping +expiring delay set when initializing the array.

+ +

Ping arrays are initialized with a size and a timeout parameter. The +size parameter denotes the maximum number of entries in the array and +the timeout denotes the number of seconds to keep an entry in the array. +Timeout and size must be bigger than 0.

+ +

Adding an entry to the ping array will make it return an 8 byte number +that can be used as the ping number of a ping packet. This number is +generated by first generating a random 8 byte number (toxcore uses the +cryptographic secure random number generator), dividing then multiplying +it by the total size of the array and then adding the index of the +element that was added. This generates a random looking number that will +return the index of the element that was added to the array. This number +is also stored along with the added data and the current time (to check +for timeouts). Data is added to the array in a cyclical manner (0, 1, 2, +3… (array size - 1), 0, 1, …). If the array is full, the oldest +element is overwritten.

+ +

To get data from the ping array, the ping number is passed to the +function to get the data from the array. The modulo of the ping number +with the total size of the array will return the index at which the data +is. If there is no data stored at this index, the function returns an +error. The ping number is then checked against the ping number stored +for this element, if it is not equal the function returns an error. If +the array element has timed out, the function returns an error. If all +the checks succeed the function returns the exact data that was stored +and it is removed from the array.

+ +

Ping array is used in many places in toxcore to efficiently keep track +of sent packets.

+ +

State Format

+ +

The reference Tox implementation uses a custom binary format to save the +state of a Tox client between restarts. This format is far from perfect +and will be replaced eventually. For the sake of maintaining +compatibility down the road, it is documented here.

+ +

The binary encoding of all integer types in the state format is a +fixed-width byte sequence with the integer encoded in Little Endian +unless stated otherwise.

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
4Zeroes
4uint32_t (0x15ED1B1F)
?List of sections
+ +

Sections

+ +

The core of the state format consists of a list of sections. Every +section has its type and length specified at the beginning. In some +cases, a section only contains one item and thus takes up the entire +length of the section. This is denoted with ’?’.

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
4uint32_t Length of this section
2uint16_t Section type
2uint16_t (0x01CE)
?Section
+ +

Section types:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameValue
NospamKeys0x01
DHT0x02
Friends0x03
Name0x04
StatusMessage0x05
Status0x06
TcpRelays0x0A
PathNodes0x0B
Conferences0x14
EOF0xFF
+ +

Not every section listed above is required to be present in order to +restore from a state file. Only NospamKeys is required.

+ +

Nospam and Keys (0x01)

+ + + + + + + + + + + + + + + + + + + + + + +
LengthContents
4uint32_t Nospam
32Long term public key
32Long term secret key
+ +

DHT (0x02)

+ +

This section contains a list of DHT-related sections.

+ + + + + + + + + + + + + + + + + + +
LengthContents
4uint32_t (0x159000D)
?List of DHT sections
+ +
DHT Sections
+ +

Every DHT section has the following structure:

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
4uint32_t Length of this section
2uint16_t DHT section type
2uint16_t (0x11CE)
?DHT section
+ +

DHT section types:

+ + + + + + + + + + + + + + +
NameValue
Nodes0x04
+ +
Nodes (0x04)
+ +

This section contains a list of nodes. These nodes are used to quickly +reconnect to the DHT after a Tox client is restarted.

+ + + + + + + + + + + + + + +
LengthContents
?List of nodes
+ +

The structure of a node is the same as Node Info. Note: this means +that the integers stored in these nodes are stored in Big Endian as +well.

+ +

Friends (0x03)

+ +

This section contains a list of friends. A friend can either be a peer +we’ve sent a friend request to or a peer we’ve accepted a friend request +from.

+ + + + + + + + + + + + + + +
LengthContents
?List of friends
+ +

Friend:

+ +

The integers in this structure are stored in Big Endian format.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t Status
32Long term public key
1024Friend request message as a byte string
1PADDING
2uint16_t Size of the friend request message
128Name as a byte string
2uint16_t Size of the name
1007Status message as a byte string
1PADDING
2uint16_t Size of the status message
1uint8_t User status (see also: USERSTATUS)
3PADDING
4uint32_t Nospam (only used for sending a friend request)
8uint64_t Last seen time
+ +

Status can be one of:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
StatusMeaning
0Not a friend
1Friend added
2Friend request sent
3Confirmed friend
4Friend online
+ +

Name (0x04)

+ + + + + + + + + + + + + + +
LengthContents
?Name as a UTF-8 encoded string
+ +

Status Message (0x05)

+ + + + + + + + + + + + + + +
LengthContents
?Status message as a UTF-8 encoded string
+ +

Status (0x06)

+ + + + + + + + + + + + + + +
LengthContents
1uint8_t User status (see also: USERSTATUS)
+ +

Tcp Relays (0x0A)

+ +

This section contains a list of TCP relays.

+ + + + + + + + + + + + + + +
LengthContents
?List of TCP relays
+ +

The structure of a TCP relay is the same as Node Info. Note: this +means that the integers stored in these nodes are stored in Big Endian +as well.

+ +

Path Nodes (0x0B)

+ +

This section contains a list of path nodes used for onion routing.

+ + + + + + + + + + + + + + +
LengthContents
?List of path nodes
+ +

The structure of a path node is the same as Node Info. Note: this +means that the integers stored in these nodes are stored in Big Endian +as well.

+ +

Conferences (0x14)

+ +

This section contains a list of saved conferences.

+ + + + + + + + + + + + + + +
LengthContents
?List of conferences
+ +

Conference:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
1uint8_t Groupchat type
32Groupchat id
4uint32_t Message number
2uint16_t Lossy message number
2uint16_t Peer number
4uint32_t Number of peers
1uint8_t Title length
?Title
?List of peers
+ +

All peers other than the saver are saved, including frozen peers. On +reload, they all start as frozen.

+ +

Peer:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LengthContents
32Long term public key
32DHT public key
2uint16_t Peer number
8uint64_t Last active timestamp
1uint8_t Name length
?Name
+ +

EOF (0xFF)

+ +

This section indicates the end of the state file. This section doesn’t +have any content and thus its length is 0.

+ +
    +
  1. +

    We use a “real” peer count, which is the number of confirmed peers +in the peerlist (that is, peers who you have successfully handshaked +and exchanged peer info with).

    +
    2. +
  2. +

    The peer roles checksum is calculated as follows: Make an unsigned +16-bit sum of each confirmed peer’s role plus the first byte of +their respective public key, then add to this an unsigned 16-bit sum +of the sanctions credentials hash.

    +
    3. +
+ +
+
+ + + + + + + diff --git a/pyproject.toml b/pyproject.toml index d78a793..8f311c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,8 @@ [project] -name = "tox_wrapper" +name = "toxygen_wrapper" description = "A Python3 ctypes wrapping of c-toxcore into Python." authors = [{ name = "Ingvar", email = "Ingvar@gitgub.com" } ] -requires-python = ">=3.6" +requires-python = ">3.6" keywords = ["tox", "python3", "ctypes"] classifiers = [ "License :: OSI Approved", @@ -20,13 +20,11 @@ classifiers = [ dynamic = ["version", "readme", ] # cannot be dynamic ['license'] [project.scripts] -tox_wrapper_tests = "tox_wrapper.tests.tests_wrapper:main" -toxygen_echo = "tox_wrapper.toxygen_echo:main" +toxygen_wrapper_tests = "toxygen_wrapper.tests.tests_wrapper:main" +toxygen_echo = "toxygen_wrapper.toxygen_echo:main" - -# ... [tool.setuptools.dynamic] -version = {attr = "tox_wrapper.__version__"} +version = {attr = "toxygen_wrapper.__version__"} readme = {file = ["README.md"]} [project.license] @@ -40,7 +38,5 @@ requires = ["setuptools >= 61.0"] build-backend = "setuptools.build_meta" [tool.setuptools] -packages = ["tox_wrapper", "tox_wrapper.tests"] +packages = ["toxygen_wrapper", "toxygen_wrapper.tests"] -#[tool.setuptools.packages.find] -#where = "src" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a650ec3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +# the versions are the current ones tested - may work with earlier versions +toxygen_wrapper >= 1.0.0 +yaml +msgpack +coloredlogs +# nmap diff --git a/setup.cfg b/setup.cfg index 81c44d1..0f54ffa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,7 +12,6 @@ classifiers = Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: Implementation :: CPython - Framework :: AsyncIO description='Tox ctypes wrapping into Python' long_description='Tox ctypes wrapping of c-toxcore into Python3' url='https://git.plastiras.org/emdee/toxygen_wrapper/' @@ -20,19 +19,10 @@ keywords='ctypes Tox messenger' [options] zip_safe = false -#python_requires = >=3.6 -include_package_data = false +python_requires = >=3.6 package_dir= =src -packages = ["tox_wrapper", "tox_wrapper.tests"] - - -[options.packages.find] -where=src - -[options.entry_points] -console_scripts = - tox_wrapper_tests = tox_wrapper.tests.tests_wrapper:main +packages = ["toxygen_wrapper", "toxygen_wrapper.tests"] [easy_install] zip_ok = false diff --git a/src/toxygen_wrapper/__init__.py b/src/toxygen_wrapper/__init__.py new file mode 100644 index 0000000..c03bf1f --- /dev/null +++ b/src/toxygen_wrapper/__init__.py @@ -0,0 +1,7 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- +# You need a libs directory beside this directory +# and you need to link your libtoxcore.so and libtoxav.so +# and libtoxencryptsave.so into ../libs/ +# Link all 3 to libtoxcore.so if you have only libtoxcore.so + +__version__ = "1.0.0" diff --git a/src/toxygen_wrapper/libtox.py b/src/toxygen_wrapper/libtox.py new file mode 100644 index 0000000..bf39dd0 --- /dev/null +++ b/src/toxygen_wrapper/libtox.py @@ -0,0 +1,87 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- +import os +import sys +from ctypes import CDLL + +# You need a libs directory beside this directory +# and you need to link your libtoxcore.so and libtoxav.so +# and libtoxencryptsave.so into ../libs/ +# Link all 3 to libtoxcore.so if you have only libtoxcore.so +try: + import utils.util as util + sLIBS_DIR = util.get_libs_directory() +except ImportError: + sLIBS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), + 'libs') + +# environment variable TOXCORE_LIBS overrides +d = os.environ.get('TOXCORE_LIBS', '') +if d and os.path.exists(d): + sLIBS_DIR = d + if os.environ.get('DEBUG', ''): + print ('DBUG: Setting TOXCORE_LIBS to ' +d) +del d + +class LibToxCore: + + def __init__(self): + platform = sys.platform + if platform == 'win32': + libtoxcore = 'libtox.dll' + elif platform == 'darwin': + libtoxcore = 'libtoxcore.dylib' + else: + libtoxcore = 'libtoxcore.so' + + # libtoxcore and libsodium may be installed in your os + # give libs/ precedence + libFile = os.path.join(sLIBS_DIR, libtoxcore) + if os.path.isfile(libFile): + self._libtoxcore = CDLL(libFile) + else: + self._libtoxcore = CDLL(libtoxcore) + + def __getattr__(self, item): + return self._libtoxcore.__getattr__(item) + +class LibToxAV: + + def __init__(self): + platform = sys.platform + if platform == 'win32': + # on Windows av api is in libtox.dll + self._libtoxav = CDLL(os.path.join(sLIBS_DIR, 'libtox.dll')) + elif platform == 'darwin': + self._libtoxav = CDLL('libtoxcore.dylib') + else: + libFile = os.path.join(sLIBS_DIR, 'libtoxav.so') + if os.path.isfile(libFile): + self._libtoxav = CDLL(libFile) + else: + self._libtoxav = CDLL('libtoxav.so') + + def __getattr__(self, item): + return self._libtoxav.__getattr__(item) + +# figure out how to see if we have a combined library + +class LibToxEncryptSave: + + def __init__(self): + platform = sys.platform + if platform == 'win32': + # on Windows profile encryption api is in libtox.dll + self._lib_tox_encrypt_save = CDLL(os.path.join(sLIBS_DIR, 'libtox.dll')) + elif platform == 'darwin': + self._lib_tox_encrypt_save = CDLL('libtoxcore.dylib') + else: + libFile = os.path.join(sLIBS_DIR, 'libtoxencryptsave.so') + if os.path.isfile(libFile): + self._lib_tox_encrypt_save = CDLL(libFile) + else: + self._lib_tox_encrypt_save = CDLL('libtoxencryptsave.so') + + def __getattr__(self, item): + return self._lib_tox_encrypt_save.__getattr__(item) + +# figure out how to see if we have a combined library diff --git a/src/toxygen_wrapper/tests/__init__.py b/src/toxygen_wrapper/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/toxygen_wrapper/tests/socks.py b/src/toxygen_wrapper/tests/socks.py new file mode 100644 index 0000000..fa1b25e --- /dev/null +++ b/src/toxygen_wrapper/tests/socks.py @@ -0,0 +1,391 @@ +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +""" + +Minor modifications made by Christopher Gilbert (http://motomastyle.com/) +for use in PyLoris (http://pyloris.sourceforge.net/) + +Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/) +mainly to merge bug fixes found in Sourceforge + +Minor modifications made by Eugene Dementiev (http://www.dementiev.eu/) + +""" + +import socket +import struct +import sys + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 + +_defaultproxy = None +_orgsocket = socket.socket + +class ProxyError(Exception): pass +class GeneralProxyError(ProxyError): pass +class Socks5AuthError(ProxyError): pass +class Socks5Error(ProxyError): pass +class Socks4Error(ProxyError): pass +class HTTPError(ProxyError): pass + +_generalerrors = ("success", + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input") + +_socks5errors = ("succeeded", + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") + +_socks4errors = ("request granted", + "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + +def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None) -> None: + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype, addr, port, rdns, username, password) + +def wrapmodule(module) -> None: + """wrapmodule(module) + Attempts to replace a module's socket library with a SOCKS socket. Must set + a default proxy using setdefaultproxy(...) first. + This will only work on modules that import socket directly into the namespace; + most of the Python Standard Library falls into this category. + """ + if _defaultproxy != None: + module.socket.socket = socksocket + else: + raise GeneralProxyError((4, "no proxy specified")) + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): + _orgsocket.__init__(self, family, type, proto, _sock) + if _defaultproxy != None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + + def __recvall(self, count): + """__recvall(count) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = self.recv(count) + while len(data) < count: + d = self.recv(count-len(data)) + if not d: raise GeneralProxyError((0, "connection closed unexpectedly")) + data = data + d + return data + + def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype, addr, port, rdns, username, password) + + def __negotiatesocks5(self, destaddr, destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall(struct.pack('BBBB', 0x05, 0x02, 0x00, 0x02)) + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall(struct.pack('BBB', 0x05, 0x01, 0x00)) + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0:1] != chr(0x05).encode(): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1:2] == chr(0x00).encode(): + # No authentication is required + pass + elif chosenauth[1:2] == chr(0x02).encode(): + # Okay, we need to perform a basic username/password + # authentication. + self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0:1] != chr(0x01).encode(): + # Bad response + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if authstat[1:2] != chr(0x00).encode(): + # Authentication failed + self.close() + raise Socks5AuthError((3, _socks5autherrors[3])) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == chr(0xFF).encode(): + raise Socks5AuthError((2, _socks5autherrors[2])) + else: + raise GeneralProxyError((1, _generalerrors[1])) + # Now we can request the actual connection + req = struct.pack('BBB', 0x05, 0x01, 0x00) + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + chr(0x01).encode() + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3]: + # Resolve remotely + ipaddr = None + if type(destaddr) != type(b''): # python3 + destaddr_bytes = destaddr.encode(encoding='idna') + else: + destaddr_bytes = destaddr + req = req + chr(0x03).encode() + chr(len(destaddr_bytes)).encode() + destaddr_bytes + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + chr(0x01).encode() + ipaddr + req = req + struct.pack(">H", destport) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0:1] != chr(0x05).encode(): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + elif resp[1:2] != chr(0x00).encode(): + # Connection failed + self.close() + if ord(resp[1:2])<=8: + raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])])) + else: + raise Socks5Error((9, _socks5errors[9])) + # Get the bound address/port + elif resp[3:4] == chr(0x01).encode(): + boundaddr = self.__recvall(4) + elif resp[3:4] == chr(0x03).encode(): + resp = resp + self.recv(1) + boundaddr = self.__recvall(ord(resp[4:5])) + else: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + boundport = struct.unpack(">H", self.__recvall(2))[0] + self.__proxysockname = (boundaddr, boundport) + if ipaddr != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return _orgsocket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self,destaddr,destport) -> None: + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3]: + ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01) + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] != None: + req = req + self.__proxy[4] + req = req + chr(0x00).encode() + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv: + req = req + destaddr + chr(0x00).encode() + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0:1] != chr(0x00).encode(): + # Bad data + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if resp[1:2] != chr(0x5A).encode(): + # Server returned an error + self.close() + if ord(resp[1:2]) in (91, 92, 93): + self.close() + raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90])) + else: + raise Socks4Error((94, _socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0]) + if rmtrslv != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def __negotiatehttp(self, destaddr, destport) -> None: + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if not self.__proxy[3]: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + self.sendall(("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n").encode()) + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n".encode()) == -1: + recv = self.recv(1) + if not recv: + raise GeneralProxyError((1, _generalerrors[1])) + resp = resp + recv + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ".encode(), 2) + if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode, statusline[2])) + self.__proxysockname = ("0.0.0.0", 0) + self.__proxypeername = (addr, destport) + + def connect(self, destpair) -> None: + """connect(self, despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (type(destpair[0]) != type('')) or (type(destpair[1]) != int): + raise GeneralProxyError((5, _generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] != None: + portnum = int(self.__proxy[2]) + else: + portnum = 1080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks5(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatesocks4(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatehttp(destpair[0], destpair[1]) + elif self.__proxy[0] == None: + _orgsocket.connect(self, (destpair[0], destpair[1])) + else: + raise GeneralProxyError((4, _generalerrors[4])) diff --git a/src/toxygen_wrapper/tests/support_http.py b/src/toxygen_wrapper/tests/support_http.py new file mode 100644 index 0000000..f3bc975 --- /dev/null +++ b/src/toxygen_wrapper/tests/support_http.py @@ -0,0 +1,163 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- + +import os +import sys +import logging +from io import BytesIO +import urllib +import traceback + +global LOG +LOG = logging.getLogger('app.'+'ts') + +try: + import pycurl +except ImportError: + pycurl = None +try: + import requests +except ImportError: + requests = None + +lNO_PROXY = ['localhost', '127.0.0.1'] +CONNECT_TIMEOUT = 20.0 + +def bAreWeConnected() -> bool: + # FixMe: Linux only + sFile = f"/proc/{os.getpid()}/net/route" + if not os.path.isfile(sFile): return None + i = 0 + for elt in open(sFile, "r").readlines(): + if elt.startswith('Iface'): continue + if elt.startswith('lo'): continue + i += 1 + return i > 0 + +def pick_up_proxy_from_environ() -> dict: + retval = dict() + if os.environ.get('socks_proxy', ''): + # socks_proxy takes precedence over https/http + proxy = os.environ.get('socks_proxy', '') + i = proxy.find('//') + if i >= 0: proxy = proxy[i+2:] + retval['proxy_host'] = proxy.split(':')[0] + retval['proxy_port'] = proxy.split(':')[-1] + retval['proxy_type'] = 2 + retval['udp_enabled'] = False + elif os.environ.get('https_proxy', ''): + # https takes precedence over http + proxy = os.environ.get('https_proxy', '') + i = proxy.find('//') + if i >= 0: proxy = proxy[i+2:] + retval['proxy_host'] = proxy.split(':')[0] + retval['proxy_port'] = proxy.split(':')[-1] + retval['proxy_type'] = 1 + retval['udp_enabled'] = False + elif os.environ.get('http_proxy', ''): + proxy = os.environ.get('http_proxy', '') + i = proxy.find('//') + if i >= 0: proxy = proxy[i+2:] + retval['proxy_host'] = proxy.split(':')[0] + retval['proxy_port'] = proxy.split(':')[-1] + retval['proxy_type'] = 1 + retval['udp_enabled'] = False + else: + retval['proxy_host'] = '' + retval['proxy_port'] = '' + retval['proxy_type'] = 0 + retval['udp_enabled'] = True + return retval + +def download_url(url:str, settings:str = None) -> None: + if not bAreWeConnected(): return '' + + if settings is None: + settings = pick_up_proxy_from_environ() + + if pycurl: + LOG.debug('Downloading with pycurl: ' + str(url)) + buffer = BytesIO() + c = pycurl.Curl() + c.setopt(c.URL, url) + c.setopt(c.WRITEDATA, buffer) + # Follow redirect. + c.setopt(c.FOLLOWLOCATION, True) + + # cookie jar + cjar = os.path.join(os.environ['HOME'], '.local', 'jar.cookie') + if os.path.isfile(cjar): + c.setopt(c.COOKIEFILE, cjar) + # LARGS+=( --cookie-jar --junk-session-cookies ) + + #? c.setopt(c.ALTSVC_CTRL, 16) + + c.setopt(c.NOPROXY, ','.join(lNO_PROXY)) + #? c.setopt(c.CAINFO, certifi.where()) + if settings['proxy_type'] == 2 and settings['proxy_host']: + socks_proxy = 'socks5h://'+settings['proxy_host']+':'+str(settings['proxy_port']) + settings['udp_enabled'] = False + c.setopt(c.PROXY, socks_proxy) + c.setopt(c.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME) + elif settings['proxy_type'] == 1 and settings['proxy_host']: + https_proxy = 'https://'+settings['proxy_host']+':'+str(settings['proxy_port']) + c.setopt(c.PROXY, https_proxy) + elif settings['proxy_type'] == 1 and settings['proxy_host']: + http_proxy = 'http://'+settings['proxy_host']+':'+str(settings['proxy_port']) + c.setopt(c.PROXY, http_proxy) + c.setopt(c.PROTOCOLS, c.PROTO_HTTPS) + try: + c.perform() + c.close() + #? assert c.getinfo(c.RESPONSE_CODE) < 300 + result = buffer.getvalue() + # Body is a byte string. + LOG.info('nodes loaded with pycurl: ' + str(url)) + return result + except Exception as ex: + LOG.error('TOX Downloading error with pycurl: ' + str(ex)) + LOG.error('\n' + traceback.format_exc()) + # drop through + + if requests: + LOG.debug('Downloading with requests: ' + str(url)) + try: + headers = dict() + headers['Content-Type'] = 'application/json' + proxies = dict() + if settings['proxy_type'] == 2 and settings['proxy_host']: + socks_proxy = 'socks5://'+settings['proxy_host']+':'+str(settings['proxy_port']) + settings['udp_enabled'] = False + proxies['https'] = socks_proxy + elif settings['proxy_type'] == 1 and settings['proxy_host']: + https_proxy = 'https://'+settings['proxy_host']+':'+str(settings['proxy_port']) + proxies['https'] = https_proxy + elif settings['proxy_type'] == 1 and settings['proxy_host']: + http_proxy = 'http://'+settings['proxy_host']+':'+str(settings['proxy_port']) + proxies['http'] = http_proxy + req = requests.get(url, + headers=headers, + proxies=proxies, + timeout=CONNECT_TIMEOUT) + # max_retries=3 + assert req.status_code < 300 + result = req.content + LOG.info('nodes loaded with requests: ' + str(url)) + return result + except Exception as ex: + LOG.error('TOX Downloading error with requests: ' + str(ex)) + # drop through + + if not settings['proxy_type']: # no proxy + LOG.debug('Downloading with urllib no proxy: ' + str(url)) + try: + req = urllib.request.Request(url) + req.add_header('Content-Type', 'application/json') + response = urllib.request.urlopen(req) + result = response.read() + LOG.info('nodes loaded with no proxy: ' + str(url)) + return result + except Exception as ex: + LOG.error('TOX Downloading ' + str(ex)) + return '' + + return '' diff --git a/src/toxygen_wrapper/tests/support_onions.py b/src/toxygen_wrapper/tests/support_onions.py new file mode 100644 index 0000000..324c889 --- /dev/null +++ b/src/toxygen_wrapper/tests/support_onions.py @@ -0,0 +1,573 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- + +import getpass +import os +import re +import select +import shutil +import socket +import sys +import time +from typing import Union, Callable, Union + +if False: + import cepa as stem + from cepa.connection import MissingPassword + from cepa.control import Controller + from cepa.util.tor_tools import is_valid_fingerprint +else: + import stem + from stem.connection import MissingPassword + from stem.control import Controller + from stem.util.tor_tools import is_valid_fingerprint + +global LOG +import logging +import warnings + +warnings.filterwarnings('ignore') +LOG = logging.getLogger() + +bHAVE_TORR = shutil.which('tor-resolve') + +yKNOWN_ONIONS = """ + - facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd # facebook + - duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad # ddg + - zkaan2xfbuxia2wpf7ofnkbz6r5zdbbvxbunvp5g2iebopbfc4iqmbad # hks +""" +# grep -B 1 '
  • bool: + # FixMe: Linux only + sFile = f"/proc/{os.getpid()}/net/route" + if not os.path.isfile(sFile): return None + i = 0 + for elt in open(sFile, "r").readlines(): + if elt.startswith('Iface'): continue + if elt.startswith('lo'): continue + i += 1 + return i > 0 + +def sMapaddressResolv(target:str, iPort:int = 9051, log_level:int = 10) -> str: + if not stem: + LOG.warn('please install the stem Python package') + return '' + + try: + controller = oGetStemController(log_level=log_level) + + map_dict = {"0.0.0.0": target} + map_ret = controller.map_address(map_dict) + + return map_ret + except Exception as e: + LOG.exception(e) + return '' + +def vwait_for_controller(controller, wait_boot:int = 10) -> None: + if bAreWeConnected() is False: + raise SystemExit("we are not connected") + percent = i = 0 + # You can call this while boostrapping + while percent < 100 and i < wait_boot: + bootstrap_status = controller.get_info("status/bootstrap-phase") + progress_percent = re.match('.* PROGRESS=([0-9]+).*', bootstrap_status) + percent = int(progress_percent.group(1)) + LOG.info(f"Bootstrapping {percent}%") + time.sleep(5) + i += 5 + +def bin_to_hex(raw_id:int, length: Union[int, None] = None) -> str: + if length is None: length = len(raw_id) + res = ''.join('{:02x}'.format(raw_id[i]) for i in range(length)) + return res.upper() + +def lIntroductionPoints(controller=None, lOnions:list = [], itimeout:int = 120, log_level:int = 10): + """now working !!! stem 1.8.x timeout must be huge >120 + 'Provides the descriptor for a hidden service. The **address** is the + '.onion' address of the hidden service ' + What about Services? + """ + try: + from cryptography.utils import int_from_bytes + except ImportError: + import cryptography.utils + + # guessing - not in the current cryptography but stem expects it + def int_from_bytes(**args): return int.to_bytes(*args) + cryptography.utils.int_from_bytes = int_from_bytes + # this will fai if the trick above didnt work + from stem.prereq import is_crypto_available + is_crypto_available(ed25519=True) + + from queue import Empty + + from stem import Timeout + from stem.client.datatype import LinkByFingerprint + from stem.descriptor.hidden_service import HiddenServiceDescriptorV3 + + if type(lOnions) not in [set, tuple, list]: + lOnions = list(lOnions) + if controller is None: + controller = oGetStemController(log_level=log_level) + l = [] + for elt in lOnions: + LOG.info(f"controller.get_hidden_service_descriptor {elt}") + try: + desc = controller.get_hidden_service_descriptor(elt, + await_result=True, + timeout=itimeout) + # LOG.log(40, f"{dir(desc)} get_hidden_service_descriptor") + # timeouts 20 sec + # mistakenly a HSv2 descriptor + hs_address = HiddenServiceDescriptorV3.from_str(str(desc)) # reparse as HSv3 + oInnerLayer = hs_address.decrypt(elt) + # LOG.log(40, f"{dir(oInnerLayer)}") + + # IntroductionPointV3 + n = oInnerLayer.introduction_points + if not n: + LOG.warn(f"NO introduction points for {elt}") + continue + LOG.info(f"{elt} {len(n)} introduction points") + lp = [] + for introduction_point in n: + for linkspecifier in introduction_point.link_specifiers: + if isinstance(linkspecifier, LinkByFingerprint): + # LOG.log(40, f"Getting fingerprint for {linkspecifier}") + if hasattr(linkspecifier, 'fingerprint'): + assert len(linkspecifier.value) == 20 + lp += [bin_to_hex(linkspecifier.value)] + LOG.info(f"{len(lp)} introduction points for {elt}") + l += lp + except (Empty, Timeout,) as e: # noqa + LOG.warn(f"Timed out getting introduction points for {elt}") + except stem.DescriptorUnavailable as e: + LOG.error(e) + except Exception as e: + LOG.exception(e) + return l + +def zResolveDomain(domain:str) -> int: + try: + ip = sTorResolve(domain) + except Exception as e: # noqa + ip = '' + if ip == '': + try: + lpair = getaddrinfo(domain, 443) + except Exception as e: + LOG.warn(f"{e}") + lpair = None + if lpair is None: + LOG.warn(f"TorResolv and getaddrinfo failed for {domain}") + return '' + ip = lpair[0] + return ip + +def sTorResolve(target:str, + verbose:bool = False, + sHost:str = '127.0.0.1', + iPort:int = 9050, + SOCK_TIMEOUT_SECONDS:float = 10.0, + SOCK_TIMEOUT_TRIES:int = 3, + ) -> str: + MAX_INFO_RESPONSE_PACKET_LENGTH = 8 + if '@' in target: + LOG.warn(f"sTorResolve failed invalid hostname {target}") + return '' + target = target.strip('/') + seb = b"\x04\xf0\x00\x00\x00\x00\x00\x01\x00" + seb += bytes(target, 'US-ASCII') + b"\x00" + assert len(seb) == 10 + len(target), str(len(seb)) + repr(seb) + +# LOG.debug(f"0 Sending {len(seb)} to The TOR proxy {seb}") + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((sHost, iPort)) + + sock.settimeout(SOCK_TIMEOUT_SECONDS) + oRet = sock.sendall(seb) # noqa + + i = 0 + data = '' + while i < SOCK_TIMEOUT_TRIES: + i += 1 + time.sleep(3) + lReady = select.select([sock.fileno()], [], [], + SOCK_TIMEOUT_SECONDS) + if not lReady[0]: continue + try: + flags=socket.MSG_WAITALL + data = sock.recv(MAX_INFO_RESPONSE_PACKET_LENGTH, flags) + except socket.timeout: + LOG.warn(f"4 The TOR proxy {(sHost, iPort)}" \ + +" didnt reply in " + str(SOCK_TIMEOUT_SECONDS) + " sec." + +" #" +str(i)) + except Exception as e: + LOG.error("4 The TOR proxy " \ + +repr((sHost, iPort)) \ + +" errored with " + str(e) + +" #" +str(i)) + sock.close() + return '' + else: + if len(data) > 0: break + + if len(data) == 0: + if i > SOCK_TIMEOUT_TRIES: + sLabel = "5 No reply #" + else: + sLabel = "5 No data #" + LOG.warn(f"sTorResolve: {sLabel} {i} on {sHost}:{iPort}") + sock.close() + return '' + + assert len(data) >= 8 + packet_sf = data[1] + if packet_sf == 90: + # , "%d" % packet_sf + assert f"{packet_sf}" == "90", f"packet_sf = {packet_sf}" + return f"{data[4]}.{data[5]}.{data[6]}.{data[7]}" + else: + # 91 + LOG.warn(f"tor-resolve failed for {target} on {sHost}:{iPort}") + + os.system(f"tor-resolve -4 {target} > /tmp/e 2>/dev/null") +# os.system("strace tor-resolve -4 "+target+" 2>&1|grep '^sen\|^rec'") + + return '' + +def getaddrinfo(sHost:str, sPort:str) -> list: + # do this the explicit way = Ive seen the compact connect fail + # >>> sHost, sPort = 'l27.0.0.1', 33446 + # >>> sock.connect((sHost, sPort)) + # socket.gaierror: [Errno -2] Name or service not known + try: + lElts = socket.getaddrinfo(sHost, int(sPort), socket.AF_INET) + lElts = list(filter(lambda elt: elt[1] == socket.SOCK_DGRAM, lElts)) + assert len(lElts) == 1, repr(lElts) + lPair = lElts[0][-1] + assert len(lPair) == 2, repr(lPair) + assert type(lPair[1]) == int, repr(lPair) + except (socket.gaierror, OSError, BaseException) as e: + LOG.error(e) + return None + return lPair + +def icheck_torrc(sFile:str, oArgs) -> int: + l = open(sFile, 'rt').readlines() + a = {} + for elt in l: + elt = elt.strip() + if not elt or ' ' not in elt: continue + (k, v,) = elt.split(' ', 1) + a[k] = v + keys = a + + if 'HashedControlPassword' not in keys: + LOG.info('Add HashedControlPassword for security') + print('run: tor --hashcontrolpassword ') + if 'ExcludeExitNodes' in keys: + elt = 'BadNodes.ExcludeExitNodes.BadExit' + LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}") + print(f"move to the {elt} section as a list") + if 'GuardNodes' in keys: + elt = 'GoodNodes.GuardNodes' + LOG.warn(f"Remove GuardNodes and move then to {oArgs.good_nodes}") + print(f"move to the {elt} section as a list") + if 'ExcludeNodes' in keys: + elt = 'BadNodes.ExcludeNodes.BadExit' + LOG.warn(f"Remove ExcludeNodes and move then to {oArgs.bad_nodes}") + print(f"move to the {elt} section as a list") + if 'ControlSocket' not in keys and os.path.exists('/run/tor/control'): + LOG.info('Add ControlSocket /run/tor/control for us') + print('ControlSocket /run/tor/control GroupWritable RelaxDirModeCheck') + if 'UseMicrodescriptors' not in keys or keys['UseMicrodescriptors'] != '1': + LOG.info('Add UseMicrodescriptors 0 for us') + print('UseMicrodescriptors 0') + if 'AutomapHostsSuffixes' not in keys: + LOG.info('Add AutomapHostsSuffixes for onions') + print('AutomapHostsSuffixes .exit,.onion') + if 'AutoMapHostsOnResolve' not in keys: + LOG.info('Add AutoMapHostsOnResolve for onions') + print('AutoMapHostsOnResolve 1') + if 'VirtualAddrNetworkIPv4' not in keys: + LOG.info('Add VirtualAddrNetworkIPv4 for onions') + print('VirtualAddrNetworkIPv4 172.16.0.0/12') + return 0 + +def lExitExcluder(oArgs, iPort:int = 9051, log_level:int = 10) -> list: + """ + https://raw.githubusercontent.com/nusenu/noContactInfo_Exit_Excluder/main/exclude_noContactInfo_Exits.py + """ + if not stem: + LOG.warn('please install the stem Python package') + return '' + LOG.debug('lExcludeExitNodes') + + try: + controller = oGetStemController(log_level=log_level) + # generator + relays = controller.get_server_descriptors() + except Exception as e: + LOG.error(f'Failed to get relay descriptors {e}') + return None + + if controller.is_set('ExcludeExitNodes'): + LOG.info('ExcludeExitNodes is in use already.') + return None + + exit_excludelist=[] + LOG.debug("Excluded exit relays:") + for relay in relays: + if relay.exit_policy.is_exiting_allowed() and not relay.contact: + if is_valid_fingerprint(relay.fingerprint): + exit_excludelist.append(relay.fingerprint) + LOG.debug("https://metrics.torproject.org/rs.html#details/%s" % relay.fingerprint) + else: + LOG.warn('Invalid Fingerprint: %s' % relay.fingerprint) + + try: + controller.set_conf('ExcludeExitNodes', exit_excludelist) + LOG.info('Excluded a total of %s exit relays without ContactInfo from the exit position.' % len(exit_excludelist)) + except Exception as e: + LOG.exception('ExcludeExitNodes ' +str(e)) + return exit_excludelist + +if __name__ == '__main__': + target = 'duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad' + controller = oGetStemController(log_level=10) + lIntroductionPoints(controller, [target], itimeout=120) diff --git a/src/toxygen_wrapper/tests/support_testing.py b/src/toxygen_wrapper/tests/support_testing.py new file mode 100644 index 0000000..e7645cf --- /dev/null +++ b/src/toxygen_wrapper/tests/support_testing.py @@ -0,0 +1,1020 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- + +import argparse +import contextlib +import inspect +import json +import logging +import os +import re +import select +import shutil +import socket +import sys +import time +import traceback +import unittest +import traceback +from ctypes import * +from random import Random +import functools +from typing import Union, Callable, Union + +random = Random() + +try: + import coloredlogs + if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ: + os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red' + # https://pypi.org/project/coloredlogs/ +except ImportError as e: + coloredlogs = False +try: + import stem +except ImportError as e: + stem = False +try: + import nmap +except ImportError as e: + nmap = False + +import tox_wrapper +import tox_wrapper.toxcore_enums_and_consts as enums + +from tox_wrapper.tests.support_http import bAreWeConnected +from tox_wrapper.tests.support_onions import (is_valid_fingerprint, + lIntroductionPoints, + oGetStemController, + sMapaddressResolv, sTorResolve) + +# LOG=util.log +global LOG +LOG = logging.getLogger() + +# callbacks can be called in any thread so were being careful +def LOG_ERROR(l): print('EROR< '+l) +def LOG_WARN(l): print('WARN< '+l) +def LOG_INFO(l): + bIsVerbose = not hasattr(__builtins__, 'app') or app.oArgs.loglevel <= 20-1 + if bIsVerbose: print('INFO< '+l) +def LOG_DEBUG(l): + bIsVerbose = not hasattr(__builtins__, 'app') or app.oArgs.loglevel <= 10-1 + if bIsVerbose: print('DBUG< '+l) +def LOG_TRACE(l): + bIsVerbose = not hasattr(__builtins__, 'app') or app.oArgs.loglevel < 10-1 + pass # print('TRACE+ '+l) + +def LOG_ERROR(l:str) -> None: print('ERROR: '+l) +def LOG_WARN(l:str) -> None: print('WARN: ' +l) +def LOG_INFO(l:str) -> None: print('INFO: ' +l) +def LOG_DEBUG(l:str) -> None: print('DEBUG: '+l) +def LOG_TRACE(l:str) -> None: pass # print('TRACE+ '+l) +def LOG_LOG(l:str) -> None: print('CORE: ' +l) + +try: + from trepan.api import debug + from trepan.interfaces import server as Mserver +except: +# print('trepan3 TCP server NOT available.') + pass +else: +# print('trepan3 TCP server available.') + def trepan_handler(num=None, f=None): + connection_opts={'IO': 'TCP', 'PORT': 6666} + intf = Mserver.ServerInterface(connection_opts=connection_opts) + dbg_opts = {'interface': intf } + print(f'Starting TCP server listening on port 6666.') + debug(dbg_opts=dbg_opts) + return + +# self._audio_thread.isAlive +iTHREAD_TIMEOUT = 1 +iTHREAD_SLEEP = 1 +iTHREAD_JOINS = 8 +iNODES = 6 +fSOCKET_TIMEOUT = 15.0 + +lToxSamplerates = [8000, 12000, 16000, 24000, 48000] +lToxSampleratesK = [8, 12, 16, 24, 48] +lBOOLEANS = [ + 'local_discovery_enabled', + 'udp_enabled', + 'ipv6_enabled', + 'trace_enabled', + 'compact_mode', + 'allow_inline', + 'notifications', + 'sound_notifications', + 'calls_sound', + 'hole_punching_enabled', + 'dht_announcements_enabled', + 'save_history', + 'download_nodes_list' + ] + +sDIR = os.environ.get('TMPDIR', '/tmp') +sTOX_VERSION = "1000002018" +bHAVE_NMAP = shutil.which('nmap') +bHAVE_JQ = shutil.which('jq') +bHAVE_BASH = shutil.which('bash') +bHAVE_TORR = shutil.which('tor-resolve') + +lDEAD_BS = [ + # Failed to resolve "tox3.plastiras.org" + "tox3.plastiras.org", + 'tox.kolka.tech', + # here and gone + '122-116-39-151.hinet-ip.hinet.net', + # IPs that do not reverse resolve + '49.12.229.145', + "46.101.197.175", + '114.35.245.150', + '172.93.52.70', + '195.123.208.139', + '205.185.115.131', + # IPs that do not rreverse resolve + 'yggnode.cf', '188.225.9.167', + '85-143-221-42.simplecloud.ru', '85.143.221.42', + # IPs that do not ping + '104.244.74.69', 'tox.plastiras.org', + '195.123.208.139', + 'gt.sot-te.ch', '32.226.5.82', + # suspicious IPs + 'tox.abilinski.com', '172.103.164.250', '172.103.164.250.tpia.cipherkey.com', + ] + +def assert_main_thread() -> None: + from PyQt5 import QtCore, QtWidgets + from qtpy.QtWidgets import QApplication + + # this "instance" method is very useful! + app_thread = QtWidgets.QApplication.instance().thread() + curr_thread = QtCore.QThread.currentThread() + if app_thread != curr_thread: + raise RuntimeError('attempt to call MainWindow.append_message from non-app thread') + +@contextlib.contextmanager +def ignoreStdout() -> None: + devnull = os.open(os.devnull, os.O_WRONLY) + old_stdout = os.dup(1) + sys.stdout.flush() + os.dup2(devnull, 1) + os.close(devnull) + try: + yield + finally: + os.dup2(old_stdout, 1) + os.close(old_stdout) + +@contextlib.contextmanager +def ignoreStderr() -> None: + devnull = os.open(os.devnull, os.O_WRONLY) + old_stderr = os.dup(2) + sys.stderr.flush() + os.dup2(devnull, 2) + os.close(devnull) + try: + yield + finally: + os.dup2(old_stderr, 2) + os.close(old_stderr) + +def clean_booleans(oArgs) -> None: + for key in lBOOLEANS: + if not hasattr(oArgs, key): continue + val = getattr(oArgs, key) + if type(val) == bool: continue + if val in ['False', 'false', '0']: + setattr(oArgs, key, False) + else: + setattr(oArgs, key, True) + +def toxygen_log_cb(_, level: int, source, line: int, func, message, userdata=None): + """ + * @param level The severity of the log message. + * @param source The source file from which the message originated. + * @param line The source line from which the message originated. + * @param func The function from which the message originated. + * @param message The log message. + * @param user_data The user data pointer passed to tox_new in options. + """ + try: + if type(source) == bytes: + source = str(source, 'UTF-8') + if type(func) == bytes: + func = str(func, 'UTF-8') + if type(message) == bytes: + message = str(message, 'UTF-8') + if source == 'network.c': + if line in [944, 660, 781, 789]: return + squelch='network family 10 (probably IPv6) on IPv4 socket' + if message.find(squelch) > 0: return + if message.find('07 = GET_NODES') > 0: return + elif source == 'TCP_common.c': + squelch='read_tcp_packet recv buffer has' + if message.find(squelch) > 0: return + return + elif source == 'Messenger.c': + if line in [2691, 2764]: return + LOG_LOG(f"{source}#{line}:{func} {message}") + except Exception as e: + LOG_WARN(f"toxygen_log_cb EXCEPTION {e}\n{traceback.format_exc()}") + +def on_log(iTox, level, filename, line, func, message, *data) -> None: + # LOG.debug(repr((level, filename, line, func, message,))) + tox_log_cb(level, filename, line, func, message) + +def tox_log_cb(_, level:int, source, line:int , func, message, userdata=None) -> None: + """ + * @param level The severity of the log message. + * @param source The source file from which the message originated. + * @param line The source line from which the message originated. + * @param func The function from which the message originated. + * @param message The log message. + * @param user_data The user data pointer passed to tox_new in options. + """ + if type(func) == bytes: + func = str(func, 'utf-8') + message = str(message, 'UTF-8') + source = str(source, 'UTF-8') + + if source == 'network.c': + if line in [944, 660, 781, 789]: return + # CORE: network.c#789:loglogdata [05 = ] T=> 10= 81.169.136.229:33445 (0: OK) | 01000151a988e582...a5x + # CORE: network.c#781:loglogdata [dd = ] T=> 128E 51.15.227.109:33445 (11: Resource temporarily unavailable) | d4f98b02ddf79693...76 + # root WARNING 3network.c#944:b'send_packet'attempted to send message with network family 10 (probably IPv6) on IPv4 socket + if message.find('07 = GET_NODES') > 0: return + if source == 'TCP_common.c': return + + i = message.find(' | ') + if i > 0: + message = message[:i] + # message = source +'#' +str(line) +':'+func +' '+message + + name = 'core' + # old level is meaningless + level = 10 # LOG.level + + i = message.find('(0: OK)') + if i > 0: + level = 10 # LOG.debug + else: + i = message.find('(1: ') + if i > 0: + level = 30 # LOG.warn + else: + level = 20 # LOG.info + +# o = LOG.makeRecord(source, level, func, line, message, list(), None) + # LOG.handle(o) + LOG_TRACE(f"{level}: {func}{line} {message}") + return + + elif level == 1: + LOG.critical(f"{level}: {message}") + elif level == 2: + LOG.error(f"{level}: {message}") + elif level == 3: + LOG.warn(f"{level}: {message}") + elif level == 4: + LOG.info(f"{level}: {message}") + elif level == 5: + LOG.debug(f"{level}: {message}") + else: + LOG_TRACE(f"{level}: {message}") + +def vAddLoggerCallback(tox_options, callback=toxygen_log_cb) -> None: + if callback is None: + tox_wrapper.tox.Tox.libtoxcore.tox_options_set_log_callback( + tox_options._options_pointer, + POINTER(None)()) + tox_options.self_logger_cb = None + LOG.debug("toxcore logging disabled") + return + + c_callback = CFUNCTYPE(None, c_void_p, c_int, c_char_p, c_int, c_char_p, c_char_p, c_void_p) + tox_options.self_logger_cb = c_callback(callback) + tox_wrapper.tox.Tox.libtoxcore.tox_options_set_log_callback( + tox_options._options_pointer, + tox_options.self_logger_cb) + LOG.debug("toxcore logging enabled") + +def get_user_config_path(): + system = sys.platform + if system == 'windows': + return os.path.join(os.getenv('APPDATA'), 'Tox/') + elif system == 'darwin': + return os.path.join(os.getenv('HOME'), 'Library/Application Support/Tox/') + else: + return os.path.join(os.getenv('HOME'), '.config/tox/') + +def oMainArgparser(_=None, iMode=0): + # 'Mode: 0=chat 1=chat+audio 2=chat+audio+video default: 0' + if not os.path.exists('/proc/sys/net/ipv6'): + bIpV6 = 'False' + else: + bIpV6 = 'True' + lIpV6Choices=[bIpV6, 'False'] + + sNodesJson = _get_nodes_path(None) + if not os.path.exists(sNodesJson): sNodesJson = '' + + logfile = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'toxygen.log') + + parser = argparse.ArgumentParser(add_help=True) + parser.add_argument('--proxy_host', '--proxy-host', type=str, + # oddball - we want to use '' as a setting + default='0.0.0.0', + help='proxy host') + parser.add_argument('--proxy_port', '--proxy-port', default=0, type=int, + help='proxy port') + parser.add_argument('--proxy_type', '--proxy-type', default=0, type=int, + choices=[0,1,2], + help='proxy type 0=noproxy, 1=http, 2=socks') + parser.add_argument('--tcp_port', '--tcp-port', default=0, type=int, + help='tcp relay server port') + parser.add_argument('--udp_enabled', type=str, default='True', + choices=['True', 'False'], + help='En/Disable udp') + parser.add_argument('--ipv6_enabled', type=str, default=bIpV6, + choices=lIpV6Choices, + help=f"En/Disable ipv6 - default {bIpV6}") + parser.add_argument('--trace_enabled', type=str, + default='False', + choices=['True','False'], + help='Debugging from toxcore logger_trace') + parser.add_argument('--download_nodes_list', type=str, default='False', + choices=['True', 'False'], + help='Download nodes list') + parser.add_argument('--nodes_json', type=str, + default=sNodesJson) + parser.add_argument('--network', type=str, + choices=['main', 'local'], + default='main') + parser.add_argument('--download_nodes_url', type=str, + default='https://nodes.tox.chat/json') + parser.add_argument('--logfile', default=logfile, + help='Filename for logging - start with + for stdout too') + parser.add_argument('--loglevel', default=logging.INFO, type=int, + # choices=[logging.info,logging.trace,logging.debug,logging.error] + help='Threshold for logging (lower is more) default: 20') + parser.add_argument('--mode', type=int, default=iMode, + choices=[0,1,2], + help='Mode: 0=chat 1=chat+audio 2=chat+audio+video default: 0') + parser.add_argument('--hole_punching_enabled',type=str, + default='False', choices=['True','False'], + help='En/Enable hole punching') + parser.add_argument('--dht_announcements_enabled',type=str, + default='True', choices=['True','False'], + help='En/Disable DHT announcements') +# argparse.ArgumentError: argument --save_history: conflicting option string: --save_history +# parser.add_argument('--save_history', type=str, default='True', +# choices=['True', 'False'], +# help='En/Disable saving history') + parser.add_argument('--socket_timeout', type=float, default=fSOCKET_TIMEOUT, + help='Socket timeout set during bootstrap in sec.') + return parser + +def get_video_indexes() -> list: + # Linux + return [str(l[5:]) for l in os.listdir('/dev/') if l.startswith('video')] + +def get_audio(): + with ignoreStderr(): + import pyaudio + oPyA = pyaudio.PyAudio() + + input_devices = output_devices = 0 + for i in range(oPyA.get_device_count()): + device = oPyA.get_device_info_by_index(i) + if device["maxInputChannels"]: + input_devices += 1 + if device["maxOutputChannels"]: + output_devices += 1 + # {'index': 21, 'structVersion': 2, 'name': 'default', 'hostApi': 0, 'maxInputChannels': 64, 'maxOutputChannels': 64, 'defaultLowInputLatency': 0.008707482993197279, 'defaultLowOutputLatency': 0.008707482993197279, 'defaultHighInputLatency': 0.034829931972789115, 'defaultHighOutputLatency': 0.034829931972789115, 'defaultSampleRate': 44100.0} + audio = {'input': oPyA.get_default_input_device_info()['index'] if input_devices else -1, + 'output': oPyA.get_default_output_device_info()['index'] if output_devices else -1, + 'enabled': input_devices and output_devices} + return audio + +def oToxygenToxOptions(oArgs, logger_cb=None): + data = None + tox_options = tox_wrapper.tox.Tox.options_new() + if oArgs.proxy_type: + tox_options.contents.proxy_type = int(oArgs.proxy_type) + tox_options.contents.proxy_host = bytes(oArgs.proxy_host, 'UTF-8') + tox_options.contents.proxy_port = int(oArgs.proxy_port) + tox_options.contents.udp_enabled = False + else: + tox_options.contents.udp_enabled = oArgs.udp_enabled + if not os.path.exists('/proc/sys/net/ipv6'): + oArgs.ipv6_enabled = False + else: + tox_options.contents.ipv6_enabled = oArgs.ipv6_enabled + + tox_options.contents.tcp_port = int(oArgs.tcp_port) + tox_options.contents.dht_announcements_enabled = oArgs.dht_announcements_enabled + tox_options.contents.hole_punching_enabled = oArgs.hole_punching_enabled + + # overrides + tox_options.contents.local_discovery_enabled = False + tox_options.contents.experimental_thread_safety = False + # REQUIRED!! + if oArgs.ipv6_enabled and not os.path.exists('/proc/sys/net/ipv6'): + LOG.warning('Disabling IPV6 because /proc/sys/net/ipv6 does not exist' + repr(oArgs.ipv6_enabled)) + tox_options.contents.ipv6_enabled = False + else: + tox_options.contents.ipv6_enabled = bool(oArgs.ipv6_enabled) + + if data: # load existing profile + tox_options.contents.savedata_type = enums.TOX_SAVEDATA_TYPE['TOX_SAVE'] + tox_options.contents.savedata_data = c_char_p(data) + tox_options.contents.savedata_length = len(data) + else: # create new profile + tox_options.contents.savedata_type = enums.TOX_SAVEDATA_TYPE['NONE'] + tox_options.contents.savedata_data = None + tox_options.contents.savedata_length = 0 + + #? tox_options.contents.log_callback = LOG + if tox_options._options_pointer and logger_cb: + LOG.debug("Adding logging to tox_options._options_pointer ") + vAddLoggerCallback(tox_options, logger_cb) + else: + LOG.warning("No tox_options._options_pointer " +repr(tox_options._options_pointer)) + + return tox_options + +def vSetupLogging(oArgs) -> None: + global LOG + logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S') + logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S' + logging._defaultFormatter.default_msec_format = '' + + add = None + kwargs = dict(level=oArgs.loglevel, + format='%(levelname)-8s %(message)s') + if oArgs.logfile: + add = oArgs.logfile.startswith('+') + sub = oArgs.logfile.startswith('-') + if add or sub: + oArgs.logfile = oArgs.logfile[1:] + kwargs['filename'] = oArgs.logfile + + if coloredlogs: + # https://pypi.org/project/coloredlogs/ + aKw = dict(level=oArgs.loglevel, + logger=LOG, + stream=sys.stdout, + fmt='%(name)s %(levelname)s %(message)s' + ) + coloredlogs.install(**aKw) + if oArgs.logfile: + oHandler = logging.FileHandler(oArgs.logfile) + LOG.addHandler(oHandler) + else: + logging.basicConfig(**kwargs) + if add: + oHandler = logging.StreamHandler(sys.stdout) + LOG.addHandler(oHandler) + + LOG.info(f"Setting loglevel to {oArgs.loglevel!s}") + + +def setup_logging(oArgs) -> None: + global LOG + if coloredlogs: + aKw = dict(level=oArgs.loglevel, + logger=LOG, + fmt='%(name)s %(levelname)s %(message)s') + if oArgs.logfile: + oFd = open(oArgs.logfile, 'wt') + setattr(oArgs, 'log_oFd', oFd) + aKw['stream'] = oFd + coloredlogs.install(**aKw) + if oArgs.logfile: + oHandler = logging.StreamHandler(stream=sys.stdout) + LOG.addHandler(oHandler) + else: + aKw = dict(level=oArgs.loglevel, + format='%(name)s %(levelname)-4s %(message)s') + if oArgs.logfile: + aKw['filename'] = oArgs.logfile + logging.basicConfig(**aKw) + + logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S') + logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S' + logging._defaultFormatter.default_msec_format = '' + + LOG.setLevel(oArgs.loglevel) +# LOG.trace = lambda l: LOG.log(0, repr(l)) + LOG.info(f"Setting loglevel to {oArgs.loglevel!s}") + +def signal_handler(num, f) -> None: + from trepan.api import debug + from trepan.interfaces import server as Mserver + connection_opts={'IO': 'TCP', 'PORT': 6666} + intf = Mserver.ServerInterface(connection_opts=connection_opts) + dbg_opts = {'interface': intf} + LOG.info('Starting TCP server listening on port 6666.') + debug(dbg_opts=dbg_opts) + return + +def merge_args_into_settings(args:list, settings:dict) -> None: + if args: + if not hasattr(args, 'audio'): + LOG.warn('No audio ' +repr(args)) + settings['audio'] = getattr(args, 'audio') + if not hasattr(args, 'video'): + LOG.warn('No video ' +repr(args)) + settings['video'] = getattr(args, 'video') + for key in settings.keys(): + # proxy_type proxy_port proxy_host + not_key = 'not_' +key + if hasattr(args, key): + val = getattr(args, key) + if type(val) == bytes: + # proxy_host - ascii? + # filenames - ascii? + val = str(val, 'UTF-8') + settings[key] = val + elif hasattr(args, not_key): + val = not getattr(args, not_key) + settings[key] = val + clean_settings(settings) + return + +def clean_settings(self:dict) -> None: + # failsafe to ensure C tox is bytes and Py settings is str + + # overrides + self['mirror_mode'] = False + self['save_history'] = True + # REQUIRED!! + if not os.path.exists('/proc/sys/net/ipv6'): + LOG.warn('Disabling IPV6 because /proc/sys/net/ipv6 does not exist') + self['ipv6_enabled'] = False + + if 'proxy_type' in self and self['proxy_type'] == 0: + self['proxy_host'] = '' + self['proxy_port'] = 0 + + if 'proxy_type' in self and self['proxy_type'] != 0 and \ + 'proxy_host' in self and self['proxy_host'] != '' and \ + 'proxy_port' in self and self['proxy_port'] != 0: + if 'udp_enabled' in self and self['udp_enabled']: + # We don't currently support UDP over proxy. + LOG.info("UDP enabled and proxy set: disabling UDP") + self['udp_enabled'] = False + if 'local_discovery_enabled' in self and self['local_discovery_enabled']: + LOG.info("local_discovery_enabled enabled and proxy set: disabling local_discovery_enabled") + self['local_discovery_enabled'] = False + if 'dht_announcements_enabled' in self and self['dht_announcements_enabled']: + LOG.info("dht_announcements_enabled enabled and proxy set: disabling dht_announcements_enabled") + self['dht_announcements_enabled'] = False + + if 'auto_accept_path' in self and \ + type(self['auto_accept_path']) == bytes: + self['auto_accept_path'] = str(self['auto_accept_path'], 'UTF-8') + + LOG.debug("Cleaned settings") + +def lSdSamplerates(iDev:int) -> list: + try: + import sounddevice as sd + except ImportError: + return [] + samplerates = (32000, 44100, 48000, 96000, ) + device = iDev + supported_samplerates = [] + for fs in samplerates: + try: + sd.check_output_settings(device=device, samplerate=fs) + except Exception as e: + # LOG.debug(f"Sample rate not supported {fs}" +' '+str(e)) + pass + else: + supported_samplerates.append(fs) + return supported_samplerates + +def _get_nodes_path(oArgs): + if oArgs and hasattr(oArgs, 'nodes_json') and \ + oArgs.nodes_json and os.path.isfile(oArgs.nodes_json): + default = oArgs.nodes_json + else: + default = os.path.join(get_user_config_path(), 'toxygen_nodes.json') +# default = os.path.join(os.getenv('HOME'), '.config', 'tox', 'toxygen_nodes.json') + LOG.debug(f"_get_nodes_path: {default}") + return default + +DEFAULT_NODES_COUNT = 8 + +global aNODES +aNODES = {} + +# @functools.lru_cache(maxsize=12) TypeError: unhashable type: 'Namespace' +def generate_nodes(oArgs, + nodes_count:int = DEFAULT_NODES_COUNT, + ipv:str = 'ipv4', + udp_not_tcp=True) -> dict: + global aNODES + sKey = ipv + sKey += ',0' if udp_not_tcp else ',1' + if sKey in aNODES and aNODES[sKey]: + return aNODES[sKey] + sFile = _get_nodes_path(oArgs) +# assert os.path.isfile(sFile), sFile + lNodes = generate_nodes_from_file(sFile, + nodes_count=nodes_count, + ipv=ipv, + udp_not_tcp=udp_not_tcp) + assert lNodes + aNODES[sKey] = lNodes + return aNODES[sKey] + +aNODES_CACHE = {} +def generate_nodes_from_file(sFile:str, + nodes_count:int = DEFAULT_NODES_COUNT, + ipv:str = 'ipv4', + udp_not_tcp:bool = True, + ) -> dict: + """https://github.com/TokTok/c-toxcore/issues/469 +I had a conversation with @irungentoo on IRC about whether we really need to call tox_bootstrap() when having UDP disabled and why. The answer is yes, because in addition to TCP relays (tox_add_tcp_relay()), toxcore also needs to know addresses of UDP onion nodes in order to work correctly. The DHT, however, is not used when UDP is disabled. tox_bootstrap() function resolves the address passed to it as argument and calls onion_add_bs_node_path() and DHT_bootstrap() functions. Although calling DHT_bootstrap() is not really necessary as DHT is not used, we still need to resolve the address of the DHT node in order to populate the onion routes with onion_add_bs_node_path() call. +""" + global aNODES_CACHE + + key = ipv + key += ',0' if udp_not_tcp else ',1' + if key in aNODES_CACHE: + sorted_nodes = aNODES_CACHE[key] + else: + try: + with open(sFile, 'rt') as fl: + json_nodes = json.loads(fl.read())['nodes'] + except Exception as e: + LOG.error(f"generate_nodes_from_file error {sFile}\n{e}") + return [] + else: + LOG.debug("generate_nodes_from_file " +sFile) + + if udp_not_tcp: + nodes = [(node[ipv], node['port'], node['public_key'],) for + node in json_nodes if node[ipv] != 'NONE' \ + and node["status_udp"] in [True, "true"] + ] + else: + nodes = [] + elts = [(node[ipv], node['tcp_ports'], node['public_key'],) \ + for node in json_nodes if node[ipv] != 'NONE' \ + and node["status_tcp"] in [True, "true"] + ] + for (ipv, ports, public_key,) in elts: + for port in ports: + nodes += [(ipv, port, public_key)] + if not nodes: + LOG.warn(f'empty generate_nodes from {sFile} {json_nodes!r}') + return [] + sorted_nodes = nodes + aNODES_CACHE[key] = sorted_nodes + + random.shuffle(sorted_nodes) + if nodes_count is not None and len(sorted_nodes) > nodes_count: + sorted_nodes = sorted_nodes[-nodes_count:] + LOG.debug(f"generate_nodes_from_file {sFile} len={len(sorted_nodes)}") + return sorted_nodes + +def tox_bootstrapd_port() -> int: + port = 33446 + sFile = '/etc/tox-bootstrapd.conf' + if os.path.exists(sFile): + with open(sFile, 'rt') as oFd: + for line in oFd.readlines(): + if line.startswith('port = '): + port = int(line[7:]) + return port + +def bootstrap_local(elts:list, lToxes:list, oArgs=None): + if os.path.exists('/run/tox-bootstrapd/tox-bootstrapd.pid'): + LOG.debug('/run/tox-bootstrapd/tox-bootstrapd.pid') + iRet = True + else: + iRet = os.system("netstat -nle4|grep -q :33") + if iRet > 0: + LOG.warn(f'bootstraping local No local DHT running') + LOG.info(f'bootstraping local') + return bootstrap_udp(elts, lToxes, oArgs) + +def lDNSClean(l:list) -> list: + global lDEAD_BS + # list(set(l).difference(set(lDEAD_BS))) + return [elt for elt in l if elt not in lDEAD_BS] + +def lExitExcluder(oArgs, iPort:int =9051) -> list: + """ + https://raw.githubusercontent.com/nusenu/noContactInfo_Exit_Excluder/main/exclude_noContactInfo_Exits.py + """ + if not stem: + LOG.warn('please install the stem Python package') + return '' + LOG.debug('lExcludeExitNodes') + + try: + controller = oGetStemController(log_level=10) + # generator + relays = controller.get_server_descriptors() + except Exception as e: + LOG.error(f'Failed to get relay descriptors {e}') + return None + + if controller.is_set('ExcludeExitNodes'): + LOG.info('ExcludeExitNodes is in use already.') + return None + + exit_excludelist=[] + LOG.debug("Excluded exit relays:") + for relay in relays: + if relay.exit_policy.is_exiting_allowed() and not relay.contact: + if is_valid_fingerprint(relay.fingerprint): + exit_excludelist.append(relay.fingerprint) + LOG.debug("https://metrics.torproject.org/rs.html#details/%s" % relay.fingerprint) + else: + LOG.warn('Invalid Fingerprint: %s' % relay.fingerprint) + + try: + controller.set_conf('ExcludeExitNodes', exit_excludelist) + LOG.info('Excluded a total of %s exit relays without ContactInfo from the exit position.' % len(exit_excludelist)) + except Exception as e: + LOG.exception('ExcludeExitNodes ' +str(e)) + return exit_excludelist + +aHOSTS = {} +@functools.lru_cache(maxsize=20) +def sDNSLookup(host:str) -> str: + global aHOSTS + ipv = 0 + if host in lDEAD_BS: +# LOG.warn(f"address skipped because in lDEAD_BS {host}") + return '' + if host in aHOSTS: + return aHOSTS[host] + + try: + s = host.replace('.','') + int(s) + ipv = 4 + except: + try: + s = host.replace(':','') + int(s) + ipv = 6 + except: pass + + if ipv > 0: +# LOG.debug(f"v={ipv} IP address {host}") + return host + + LOG.debug(f"sDNSLookup {host}") + ip = '' + if host.endswith('.tox') or host.endswith('.onion'): + if False and stem: + ip = sMapaddressResolv(host) + if ip: return ip + + ip = sTorResolve(host) + if ip: return ip + + if not bHAVE_TORR: + LOG.warn(f"onion address skipped because no tor-resolve {host}") + return '' + try: + sout = f"/tmp/TR{os.getpid()}.log" + i = os.system(f"tor-resolve -4 {host} > {sout}") + if not i: + LOG.warn(f"onion address skipped because tor-resolve on {host}") + return '' + ip = open(sout, 'rt').read() + if ip.endswith('failed.'): + LOG.warn(f"onion address skipped because tor-resolve failed on {host}") + return '' + LOG.debug(f"onion address tor-resolve {ip} on {host}") + return ip + except: + pass + else: + try: + ip = socket.gethostbyname(host) + LOG.debug(f"host={host} gethostbyname IP address {ip}") + if ip: + aHOSTS[host] = ip + return ip + # drop through + except: + # drop through + pass + + if ip == '': + try: + sout = f"/tmp/TR{os.getpid()}.log" + i = os.system(f"dig {host} +timeout=15|grep ^{host}|sed -e 's/.* //'> {sout}") + if not i: + LOG.warn(f"address skipped because dig failed on {host}") + return '' + ip = open(sout, 'rt').read().strip() + LOG.debug(f"address dig {ip} on {host}") + aHOSTS[host] = ip + return ip + except: + ip = host + LOG.debug(f'sDNSLookup {host} -> {ip}') + if ip and ip != host: + aHOSTS[host] = ip + return ip + +def bootstrap_udp(lelts:list, lToxes:list[int], oArgs=None, fsocket_timeout:float = fSOCKET_TIMEOUT) -> None: + global lDEAD_BS + lelts = lDNSClean(lelts) + socket.setdefaulttimeout(fsocket_timeout) + for oTox in lToxes: + random.shuffle(lelts) + if hasattr(oTox, 'oArgs'): + oArgs = oTox.oArgs + if hasattr(oArgs, 'contents') and oArgs.contents.proxy_type != 0: + lelts = lelts[:1] + +# LOG.debug(f'bootstrap_udp DHT bootstraping {oTox.name} {len(lelts)}') + for largs in lelts: + assert len(largs) == 3 + host, port, key = largs + assert host; assert port; assert key + if host in lDEAD_BS: continue + ip = sDNSLookup(host) + if not ip: + LOG.warn(f'bootstrap_udp to host={host} port={port} did not resolve ip={ip}') + lDEAD_BS.append(host) + continue + + if type(port) == str: + port = int(port) + try: + assert len(key) == 64, key + # NOT ip + oRet = oTox.bootstrap(host, + port, + key) + except Exception as e: + if oArgs is None or ( + hasattr(oArgs, 'contents') and oArgs.contents.proxy_type == 0): + pass + # LOG.error(f'bootstrap_udp failed to host={host} port={port} {e}') + continue + if not oRet: + LOG.warn(f'bootstrap_udp failed to {host} : {oRet}') + elif oTox.self_get_connection_status() != enums.TOX_CONNECTION['NONE']: + LOG.info(f'bootstrap_udp to {host} connected') + break + else: +# LOG.debug(f'bootstrap_udp to {host} not connected') + pass + +def bootstrap_tcp(lelts:list, lToxes:list, oArgs=None, fsocket_timeout:float = fSOCKET_TIMEOUT) -> None: + global lDEAD_BS + socket.setdefaulttimeout(fsocket_timeout) + lelts = lDNSClean(lelts) + for oTox in lToxes: + if hasattr(oTox, 'oArgs'): oArgs = oTox.oArgs + random.shuffle(lelts) +# LOG.debug(f'bootstrap_tcp bootstapping {oTox.name} {len(lelts)}') + for (host, port, key,) in lelts: + assert host; assert port;assert key + if host in lDEAD_BS: continue + ip = sDNSLookup(host) + if not ip: + LOG.warn(f'bootstrap_tcp to {host} did not resolve ip={ip}') + lDEAD_BS.append(host) + continue + #? ip = host + if host.endswith('.onion') and stem: + l = lIntroductionPoints(host) + if not l: + LOG.warn(f'bootstrap_tcp to {host} has no introduction points') + continue + if type(port) == str: + port = int(port) + try: + assert len(key) == 64, key + oRet = oTox.add_tcp_relay(ip, + port, + key) + except Exception as e: + # The address could not be resolved to an IP address, or the IP address passed was invalid. + LOG.warn(f'bootstrap_tcp to {host} : ' +str(e)) + continue + if not oRet: + LOG.warn(f'bootstrap_tcp failed to {host} : {oRet}') + elif hasattr(oTox, 'mycon_time') and oTox.mycon_time == 1: + LOG.debug(f'bootstrap_tcp to {host} not yet connected') + elif hasattr(oTox, 'mycon_status') and oTox.mycon_status is False: + LOG.debug(f'bootstrap_tcp to {host} not True') + elif oTox.self_get_connection_status() != enums.TOX_CONNECTION['NONE']: + LOG.info(f'bootstrap_tcp to {host} connected') + break + else: +# LOG.debug(f'bootstrap_tcp to {host} but not connected' +# +f" last={int(oTox.mycon_time)}" ) + pass + +def iNmapInfoNmap(sProt:str, sHost:str, sPort:str, key=None, environ=None, cmd:str = '') -> int: + if sHost in ['-', 'NONE']: return 0 + if not nmap: return 0 + nmps = nmap.PortScanner + if sProt in ['socks', 'socks5', 'tcp4']: + prot = 'tcp' + cmd = f" -Pn -n -sT -p T:{sPort}" + else: + prot = 'udp' + cmd = f" -Pn -n -sU -p U:{sPort}" + LOG.debug(f"iNmapInfoNmap cmd={cmd}") + sys.stdout.flush() + o = nmps().scan(hosts=sHost, arguments=cmd) + aScan = o['scan'] + ip = list(aScan.keys())[0] + state = aScan[ip][prot][sPort]['state'] + LOG.info(f"iNmapInfoNmap: to {sHost} {state}") + return 0 + +def iNmapInfo(sProt:str, sHost:str, sPort:str, key=None, environ=None, cmd:str = 'nmap'): + if sHost in ['-', 'NONE']: return 0 + sFile = os.path.join("/tmp", f"{sHost}.{os.getpid()}.nmap") + if sProt in ['socks', 'socks5', 'tcp4']: + cmd += f" -Pn -n -sT -p T:{sPort} {sHost} | grep /tcp " + else: + cmd += f" -Pn -n -sU -p U:{sPort} {sHost} | grep /udp " + LOG.debug(f"iNmapInfo cmd={cmd}") + sys.stdout.flush() + iRet = os.system(cmd +f" >{sFile} 2>&1 ") + LOG.debug(f"iNmapInfo cmd={cmd} iRet={iRet}") + if iRet != 0: + return iRet + assert os.path.exists(sFile), sFile + with open(sFile, 'rt') as oFd: + l = oFd.readlines() + assert len(l) + l = [line for line in l if line and not line.startswith('WARNING:')] + s = '\n'.join([s.strip() for s in l]) + LOG.info(f"iNmapInfo: to {sHost}\n{s}") + return 0 + + +# bootstrap_iNmapInfo(lElts, self._args, sProt) +def bootstrap_iNmapInfo(lElts:list, oArgs, protocol:str = "tcp4", bIS_LOCAL:bool = False, iNODES:int = iNODES, cmd:str = 'nmap') -> bool: + if not bIS_LOCAL and not bAreWeConnected(): + LOG.warn(f"bootstrap_iNmapInfo not local and NOT CONNECTED") + return True + if os.environ['USER'] != 'root': + LOG.warn(f"bootstrap_iNmapInfo not ROOT USER={os.environ['USER']}") + cmd = 'sudo ' +cmd + + lRetval = [] + LOG.info(f"bootstrap_iNmapInfo testing nmap={nmap} len={len(lElts[:iNODES])}") + for elts in lElts[:iNODES]: + host, port, key = elts + ip = sDNSLookup(host) + if not ip: + LOG.info(f"bootstrap_iNmapInfo to {host} did not resolve ip={ip}") + continue + if type(port) == str: + port = int(port) + iRet = -1 + try: + if not nmap: + iRet = iNmapInfo(protocol, ip, port, key, cmd=cmd) + else: + iRet = iNmapInfoNmap(protocol, ip, port, key) + if iRet != 0: + LOG.warn('iNmapInfo to ' +repr(host) +' retval=' +str(iRet)) + lRetval += [False] + else: + LOG.info('iNmapInfo to ' +repr(host) +' retval=' +str(iRet)) + lRetval += [True] + except Exception as e: + LOG.exception('iNmapInfo to {host} : ' +str(e) + ) + lRetval += [False] + return any(lRetval) + +def caseFactory(cases:list) -> list: + """We want the tests run in order.""" + if len(cases) > 1: + ordered_cases = sorted(cases, key=lambda f: inspect.findsource(f)[1]) + else: + ordered_cases = cases + return ordered_cases + +def suiteFactory(*testcases): + """We want the tests run in order.""" + linen = lambda f: getattr(tc, f).__code__.co_firstlineno + lncmp = lambda a, b: linen(a) - linen(b) + + test_suite = unittest.TestSuite() + for tc in testcases: + test_suite.addTest(unittest.makeSuite(tc, sortUsing=lncmp)) + return test_suite diff --git a/src/toxygen_wrapper/tests/tests_wrapper.py b/src/toxygen_wrapper/tests/tests_wrapper.py new file mode 100644 index 0000000..d6bc16f --- /dev/null +++ b/src/toxygen_wrapper/tests/tests_wrapper.py @@ -0,0 +1,2284 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- +# +# @file tests.py +# @author Wei-Ning Huang (AZ) +# +# Copyright (C) 2013 - 2014 Wei-Ning Huang (AZ) +# All Rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +"""Originaly from https://github.com/oxij/PyTox c-toxcore-02 branch +which itself was forked from https://github.com/aitjcize/PyTox/ + +Modified to work with +""" + +import ctypes +import faulthandler +import hashlib +import logging +import os +import random +import re +import sys +import threading +import traceback +import unittest +from ctypes import * +from typing import Union, Callable, Union + +faulthandler.enable() + +import warnings +warnings.filterwarnings('ignore') + +try: + from io import BytesIO + import certifi + import pycurl +except ImportError: + pycurl = None + +# from pyannotate_runtime import collect_types + +try: + import coloredlogs + os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red' +except ImportError as e: + logging.log(logging.DEBUG, f"coloredlogs not available: {e}") + coloredlogs = None + +try: + import color_runner +except ImportError as e: + logging.log(logging.DEBUG, f"color_runner not available: {e}") + color_runner = None + +import tox_wrapper +import tox_wrapper.toxcore_enums_and_consts as enums +from tox_wrapper.tox import Tox, UINT32_MAX, ToxError + +from tox_wrapper.toxcore_enums_and_consts import (TOX_ADDRESS_SIZE, TOX_CONNECTION, + TOX_FILE_CONTROL, + TOX_MESSAGE_TYPE, + TOX_SECRET_KEY_SIZE, + TOX_USER_STATUS) + +try: + import support_testing as ts +except ImportError: + import tox_wrapper.tests.support_testing as ts + +try: + from tests.toxygen_tests import test_sound_notification + bIS_NOT_TOXYGEN = False +except ImportError: + bIS_NOT_TOXYGEN = True + +# from PyQt5 import QtCore +import time + +sleep = time.sleep + +global LOG +LOG = logging.getLogger('TestS') +if False: + def LOG_ERROR(l: str) -> None: LOG.error('+ '+l) + def LOG_WARN(l: str) -> None: LOG.warn('+ '+l) + def LOG_INFO(l: str) -> None: LOG.info('+ '+l) + def LOG_DEBUG(l: str) -> None: LOG.debug('+ '+l) + def LOG_TRACE(l: str) -> None: pass # print('+ '+l) +else: + # just print to stdout so there is NO complications from logging. + def LOG_ERROR(l: str) -> None: print('EROR+ '+l) + def LOG_WARN(l: str) -> None: print('WARN+ '+l) + def LOG_INFO(l: str) -> None: print('INFO+ '+l) + def LOG_DEBUG(l: str) -> None: print('DEBUG+ '+l) + def LOG_TRACE(l: str) -> None: pass # print('TRAC+ '+l) + +ADDR_SIZE = 38 * 2 +CLIENT_ID_SIZE = 32 * 2 +THRESHOLD = 120 # >25 +fSOCKET_TIMEOUT = 15.0 + +iN = 6 + +global oTOX_OPTIONS +oTOX_OPTIONS = {} + +bIS_LOCAL = 'new' in sys.argv or 'local' in sys.argv or 'newlocal' in sys.argv +bUSE_NOREQUEST = None + +def expectedFailure(test_item): + test_item.__unittest_expecting_failure__ = True + return test_item + +def expectedFail(reason: str): + """ + expectedFailure with a reason + """ + def decorator(test_item): + test_item.__unittest_expecting_failure__ = True + return test_item + return decorator + +class ToxOptions(): + def __init__(self): + self.ipv6_enabled = True + self.udp_enabled = True + self.proxy_type = 0 + self.proxy_host = '' + self.proxy_port = 0 + self.start_port = 0 + self.end_port = 0 + self.tcp_port = 0 + self.savedata_type = 0 # 1=toxsave, 2=secretkey + self.savedata_data = b'' + self.savedata_length = 0 + self.local_discovery_enabled = False + self.dht_announcements_enabled = True + self.hole_punching_enabled = False + self.experimental_thread_safety = False + +class App(): + def __init__(self): + self.mode = 0 +oAPP = App() + +class AliceTox(Tox): + + def __init__(self, opts, args, app=None): + + super(AliceTox, self).__init__(opts, app=app) + self._address = self.self_get_address() + self.name = 'alice' + self._opts = opts + self._app = app + self._args = args + +class BobTox(Tox): + + def __init__(self, opts, args, app=None): + super(BobTox, self).__init__(opts, app=app) + self._address = self.self_get_address() + self.name = 'bob' + self._opts = opts + self._app = app + self._args = args + +class BaseThread(threading.Thread): + + def __init__(self, name=None, target=None): + if name: + super().__init__(name=name, target=target) + else: + super().__init__(target=target) + self._stop_thread = False + self.name = name + + def stop_thread(self, timeout: int = -1) -> None: + self._stop_thread = True + if timeout < 0: + timeout = ts.iTHREAD_TIMEOUT + i = 0 + while i < ts.iTHREAD_JOINS: + self.join(timeout) + if not self.is_alive(): break + i = i + 1 + else: + LOG.warning(f"{self.name} BLOCKED") + +class ToxIterateThread(BaseThread): + + def __init__(self, tox): + super().__init__(name='ToxIterateThread') + self._tox = tox + + def run(self) -> None: + while not self._stop_thread: + self._tox.iterate() + sleep(self._tox.iteration_interval() / 1000) + +global bob, alice +bob = alice = None + +def prepare(self): + global bob, alice + def bobs_on_self_connection_status(iTox, connection_state, *args) -> None: + status = connection_state + self.bob.dht_connected = status + self.bob.mycon_time = time.time() + try: + if status != TOX_CONNECTION['NONE']: + LOG_INFO(f"bobs_on_self_connection_status TRUE {status}" \ + +f" last={int(self.bob.mycon_time)}" ) + self.bob.mycon_status = True + else: + LOG_DEBUG(f"bobs_on_self_connection_status FALSE {status}" \ + +f" last={int(self.bob.mycon_time)}" ) + self.bob.mycon_status = False + except Exception as e: + LOG_ERROR(f"bobs_on_self_connection_status {e}") + else: + if self.bob.self_get_connection_status() != status: + LOG_WARN(f"bobs_on_self_connection_status DISAGREE {status}") + + def alices_on_self_connection_status(iTox, connection_state: int, *args) -> None: + global oTOX_OARGS + #FixMe connection_num + status = connection_state + self.alice.dht_connected = status + self.alice.mycon_time = time.time() + try: + if status != TOX_CONNECTION['NONE']: + LOG_INFO(f"alices_on_self_connection_status TRUE {status}" \ + +f" last={int(self.alice.mycon_time)}" ) + self.alice.mycon_status = True + else: + LOG_DEBUG(f"alices_on_self_connection_status FALSE {status}" \ + +f" last={int(self.alice.mycon_time)}" ) + self.alice.mycon_status = False + except Exception as e: + LOG_ERROR(f"alices_on_self_connection_status error={e}") + + opts = oTestsToxOptions(oTOX_OARGS) + global bUSE_NOREQUEST + bUSE_NOREQUEST = oTOX_OARGS.norequest == 'True' + + alice = AliceTox(opts, oTOX_OARGS, app=oAPP) + alice.dht_connected = -1 + alice.mycon_status = False + alice.mycon_time = 1 + alice.callback_self_connection_status(alices_on_self_connection_status) + + # only bob logs trace_enabled + if oTOX_OARGS.trace_enabled: + LOG.info(f"toxcore trace_enabled") + ts.vAddLoggerCallback(opts) + else: + LOG.debug(f"toxcore trace_enabled=False") + + bob = BobTox(opts, oTOX_OARGS, app=oAPP) + bob.dht_connected = -1 + bob.mycon_status = False + bob.mycon_time = 1 + bob.callback_self_connection_status(bobs_on_self_connection_status) + if not bIS_LOCAL and not ts.bAreWeConnected(): + LOG.warning(f"doOnce not local and NOT CONNECTED") + return [bob, alice] + +class ToxSuite(unittest.TestCase): + failureException = AssertionError + + @classmethod + def setUpClass(cls) -> None: + global oTOX_OARGS + assert oTOX_OPTIONS + + cls.lUdp = ts.generate_nodes( + oArgs=oTOX_OARGS, + nodes_count=2*ts.iNODES, + ipv='ipv4', + udp_not_tcp=True) + + cls.lTcp = ts.generate_nodes( + oArgs=oTOX_OARGS, + nodes_count=2*ts.iNODES, + ipv='ipv4', + udp_not_tcp=False) + + def tearDown(self) -> None: + """ + """ + if hasattr(self, 'bob') and self.bob.self_get_friend_list_size() >= 1: + LOG.warn(f"tearDown BOBS STILL HAS A FRIEND LIST {self.bob.self_get_friend_list()}") + for elt in self.bob.self_get_friend_list(): + self.bob.friend_delete(elt) + if hasattr(self, 'alice') and self.alice.self_get_friend_list_size() >= 1: + LOG.warn(f"tearDown ALICE STILL HAS A FRIEND LIST {self.alice.self_get_friend_list()}") + for elt in self.alice.self_get_friend_list(): + self.alice.friend_delete(elt) + +# LOG.debug(f"tearDown threads={threading.active_count()}") + if hasattr(self, 'bob'): + bob.callback_self_connection_status(None) + if hasattr(self.bob, 'main_loop'): + self.bob._main_loop.stop_thread() + del self.bob._main_loop +# self.bob.kill() + del self.bob + if hasattr(self, 'alice'): + alice.callback_self_connection_status(None) + if hasattr(self.alice, 'main_loop'): + self.alice._main_loop.stop_thread() + del self.alice._main_loop +# self.alice.kill() + del self.alice + + @classmethod + def tearDownClass(cls) -> None: + if hasattr(cls, 'bob'): + cls.bob._main_loop.stop_thread() + cls.bob.kill() + del cls.bob + if hasattr(cls, 'alice'): + cls.alice._main_loop.stop_thread() + cls.alice.kill() + del cls.alice + + def bBobNeedAlice(self) -> bool: + """ + """ + if hasattr(self, 'baid') and self.baid >= 0 and \ + self.baid in self.bob.self_get_friend_list(): + LOG.warn(f"setUp ALICE IS ALREADY IN BOBS FRIEND LIST") + return False + elif self.bob.self_get_friend_list_size() >= 1: + LOG.warn(f"setUp BOB STILL HAS A FRIEND LIST") + return False + return True + + def bAliceNeedAddBob (self) -> bool: + if hasattr(self, 'abid') and self.abid >= 0 and \ + self.abid in self.alice.self_get_friend_list(): + LOG.warn(f"setUp BOB IS ALREADY IN ALICES FRIEND LIST") + return False + elif self.alice.self_get_friend_list_size() >= 1: + LOG.warn(f"setUp ALICE STILL HAS A FRIEND LIST") + return False + return True + + def setUp(self): + cls = self + if not hasattr(cls, 'alice') and not hasattr(cls, 'bob'): + l = prepare(cls) + assert l + cls.bob, cls.alice = l + if not hasattr(cls.bob, '_main_loop'): +#? cls.bob._main_loop = ToxIterateThread(cls.bob) +#? cls.bob._main_loop.start() + LOG.debug(f"cls.bob._main_loop: ") # {threading.enumerate()} + if not hasattr(cls.alice, '_main_loop'): +#? cls.alice._main_loop = ToxIterateThread(cls.alice) +#? cls.alice._main_loop.start() + LOG.debug(f"cls.alice._main_loop: ") # {threading.enumerate()} + + self.bBobNeedAlice() + self.bAliceNeedAddBob() + + def run(self, result=None) -> None: + """ Stop after first error """ + if result and not result.errors: + super(ToxSuite, self).run(result) + + def get_connection_status(self) -> bool: + if self.bob.mycon_time <= 1 or self.alice.mycon_time <= 1: + pass + # drop through + elif self.bob.dht_connected == TOX_CONNECTION['NONE']: + return False + elif self.alice.dht_connected == TOX_CONNECTION['NONE']: + return False + + # if not self.connected + if self.bob.self_get_connection_status() == TOX_CONNECTION['NONE']: + return False + if self.alice.self_get_connection_status() == TOX_CONNECTION['NONE']: + return False + return True + + def loop(self, n) -> None: + """ + t:iterate + t:iteration_interval + """ + interval = self.bob.iteration_interval() + for i in range(n): + self.alice.iterate() + self.bob.iterate() + sleep(interval / 1000.0) + + def call_bootstrap(self, num: Union[int, None] = None, lToxes:Union[list[int], None] =None, i:int =0, fsocket_timeout:float = fSOCKET_TIMEOUT) -> None: + global oTOX_OARGS + if num == None: num=ts.iNODES + if lToxes is None: + lToxes = [self.alice, self.bob] +# LOG.debug(f"call_bootstrap network={oTOX_OARGS.network}") + if oTOX_OARGS.network in ['new', 'newlocal', 'localnew']: + ts.bootstrap_local(self.lUdp, lToxes) + elif not ts.bAreWeConnected(): + LOG.warning('we are NOT CONNECTED') + else: + random.shuffle(self.lUdp) + if oTOX_OARGS.proxy_port > 0: + lElts = self.lUdp[:1] + else: + lElts = self.lUdp[:num+i] + LOG.debug(f"call_bootstrap ts.bootstrap_udp {len(lElts)}") + ts.bootstrap_udp(lElts, lToxes, fsocket_timeout=fsocket_timeout) + random.shuffle(self.lTcp) + lElts = self.lTcp[:num+i] + LOG.debug(f"call_bootstrap ts.bootstrap_tcp {len(lElts)}") + ts.bootstrap_tcp(lElts, lToxes, fsocket_timeout=fsocket_timeout) + + def group_until_connected(self, otox, group_number:int, num: Union[int, None] = None, iMax:int = THRESHOLD, fsocket_timeout:float = fSOCKET_TIMEOUT) -> bool: + """ + """ + i = 0 + bRet = None + while i <= iMax : + iRet = otox.group_is_connected(group_number) + if iRet == True or iRet == 0: + bRet = True + break + if i % 5 == 0: + j = i//5 + self.call_bootstrap(num, lToxes=None, i=j, fsocket_timeout=fsocket_timeout) + s = '' + if i == 0: s = '\n' + LOG.info(s+"group_until_connected " \ + +" #" + str(i) \ + +" iRet=" +repr(iRet) \ + +f" BOBS={otox.mycon_status}" \ + +f" last={int(otox.mycon_time)}" ) + i += 1 + self.loop(100) + else: + bRet = False + + if bRet: + LOG.info(f"group_until_connected True i={i}" \ + +f" iMax={iMax}" \ + +f" BOB={otox.self_get_connection_status()}" \ + +f" last={int(otox.mycon_time)}" ) + return True + else: + LOG.warning(f"group_until_connected False i={i}" \ + +f" iMax={iMax}" \ + +f" BOB={otox.self_get_connection_status()}" \ + +f" last={int(otox.mycon_time)}" ) + return False + + def loop_until_connected(self, otox=None, num: Union[int, None] = None, fsocket_timeout:float = fSOCKET_TIMEOUT) -> bool: + """ + t:on_self_connection_status + t:self_get_connection_status + """ + i = 0 + bRet = None + if otox is None: otox = self.bob + while i <= otox._args.test_timeout : + if (self.alice.mycon_status and self.bob.mycon_status): + bRet = True + break + if i % 5 == 0: + j = i//5 + self.call_bootstrap(num, lToxes=None, i=j, fsocket_timeout=fsocket_timeout) + s = '' + if i == 0: s = '\n' + LOG.info(s+"loop_until_connected " \ + +" #" + str(i) \ + +" BOB=" +repr(self.bob.self_get_connection_status()) \ + +" ALICE=" +repr(self.alice.self_get_connection_status()) + +f" BOBS={self.bob.mycon_status}" \ + +f" ALICES={self.alice.mycon_status}" \ + +f" last={int(self.bob.mycon_time)}" ) + if (self.alice.mycon_status and self.bob.mycon_status): + bRet = True + break + if (self.alice.self_get_connection_status() and + self.bob.self_get_connection_status()): + LOG_WARN(f"loop_until_connected disagree status() DISAGREE" \ + +f' self.bob.mycon_status={self.bob.mycon_status}' \ + +f' alice.mycon_status={self.alice.mycon_status}' \ + +f" last={int(self.bob.mycon_time)}" ) + bRet = True + break + i += 1 + self.loop(100) + else: + bRet = False + + if bRet or \ + ( self.bob.self_get_connection_status() != TOX_CONNECTION['NONE'] and \ + self.alice.self_get_connection_status() != TOX_CONNECTION['NONE'] ): + LOG.info(f"loop_until_connected returning True {i}" \ + +f" BOB={self.bob.self_get_connection_status()}" \ + +f" ALICE={self.alice.self_get_connection_status()}" \ + +f" last={int(self.bob.mycon_time)}" ) + return True + else: + otox._args.test_timeout += 5 + LOG.warning(f"loop_until_connected returning False {i}" \ + +f" BOB={self.bob.self_get_connection_status()}" \ + +f" ALICE={self.alice.self_get_connection_status()}" \ + +f" last={int(self.bob.mycon_time)}" ) + return False + + def wait_objs_attr(self, objs: list, attr: str, fsocket_timeout:float = fSOCKET_TIMEOUT) -> bool: + i = 0 + otox = objs[0] + while i <= otox._args.test_timeout: + if i % 5 == 0: + num = None + j = 0 + j = i//5 + self.call_bootstrap(num, lToxes=objs, i=j, fsocket_timeout=fsocket_timeout) + LOG.debug(f"wait_objs_attr {objs} for {attr} {i}") + if all([getattr(obj, attr) for obj in objs]): + return True + self.loop(100) + i += 1 + else: + otox._args.test_timeout += 1 + LOG.warn(f"wait_objs_attr for {attr} i >= {otox._args.test_timeout}") + + return all([getattr(obj, attr) is not None for obj in objs]) + + def wait_otox_attrs(self, obj, attrs: list[str], fsocket_timeout:float = fSOCKET_TIMEOUT) -> bool: + assert all(attrs), f"wait_otox_attrs {attrs}" + i = 0 + otox = obj + while i <= otox._args.test_timeout: + if i % 5 == 0: + num = None + j = 0 + if obj.mycon_time == 1: + num = 4 + j = i//5 + if obj.self_get_connection_status() == TOX_CONNECTION['NONE']: + self.call_bootstrap(num, lToxes=[obj], i=j, fsocket_timeout=fsocket_timeout) + LOG.debug(f"wait_otox_attrs {obj.name} for {attrs} {i}" \ + +f" last={int(obj.mycon_time)}") + if all([getattr(obj, attr) is not None for attr in attrs]): + return True + self.loop(100) + i += 1 + else: + LOG.warning(f"wait_otox_attrs i >= {otox._args.test_timeout} attrs={attrs} results={[getattr(obj, attr) for attr in attrs]}") + + return all([getattr(obj, attr) for attr in attrs]) + + def wait_ensure_exec(self, method, args:list, fsocket_timeout:float = fSOCKET_TIMEOUT) -> bool: + i = 0 + oRet = None + while i <= self.bob._args.test_timeout: + if i % 5 == 0: + j = i//5 + self.call_bootstrap(num=None, lToxes=None, i=j, fsocket_timeout=fsocket_timeout) + LOG.debug("wait_ensure_exec " \ + +" " +str(method) + +" " +str(i)) + try: + oRet = method(*args) + if oRet: + LOG.info(f"wait_ensure_exec oRet {oRet!r}") + return True + except ArgumentError as e: + # ArgumentError('This client is currently NOT CONNECTED to the friend.') + # dunno + LOG.warning(f"wait_ensure_exec ArgumentError {e}") + return False + except Exception as e: + LOG.warning(f"wait_ensure_exec EXCEPTION {e}") + return False + sleep(3) + i += 1 + else: + LOG.error(f"wait_ensure_exec i >= {1*self.bob._args.test_timeout}") + return False + + return oRet + + def bob_add_alice_as_friend_norequest(self) -> bool: + if not self.bBobNeedAlice(): return True + + iRet = self.bob.friend_add_norequest(self.alice._address) + if iRet < 0: + return False + self.baid = self.bob.friend_by_public_key(self.alice._address) + assert self.baid >= 0, self.baid + assert self.bob.friend_exists(self.baid), "bob.friend_exists" + assert not self.bob.friend_exists(self.baid + 1) + assert self.baid in self.bob.self_get_friend_list() + assert self.bob.self_get_friend_list_size() >= 1 + return True + + def alice_add_bob_as_friend_norequest(self) -> bool: + if not self.bAliceNeedAddBob(): return True + + iRet = self.alice.friend_add_norequest(self.bob._address) + if iRet < 0: + return False + self.abid = self.alice.friend_by_public_key(self.bob._address) + assert self.abid >= 0, self.abid + assert self.abid in self.alice.self_get_friend_list() + assert self.alice.friend_exists(self.abid), "alice.friend_exists" + assert not self.alice.friend_exists(self.abid + 1) + assert self.alice.self_get_friend_list_size() >= 1 + return True + + def both_add_as_friend(self) -> bool: + if bUSE_NOREQUEST: + assert self.bob_add_alice_as_friend() + assert self.alice_add_bob_as_friend_norequest() + else: + assert self.bob_add_alice_as_friend_norequest() + assert self.alice_add_bob_as_friend_norequest() + if not hasattr(self, 'baid') or self.baid < 0: + LOG.warn("both_add_as_friend no bob, baid") + if not hasattr(self, 'abid') or self.abid < 0: + LOG.warn("both_add_as_friend no alice, abid") + return True + + def both_add_as_friend_norequest(self) -> bool: + if self.bBobNeedAlice(): + assert self.bob_add_alice_as_friend_norequest() + if self.bAliceNeedAddBob(): + assert self.alice_add_bob_as_friend_norequest() + if not hasattr(self, 'baid') or self.baid < 0: + LOG.warn("both_add_as_friend_norequest no bob, baid") + if not hasattr(self, 'abid') or self.abid < 0: + LOG.warn("both_add_as_friend_norequest no alice, abid") + + #: Test last online +#? assert self.alice.friend_get_last_online(self.abid) is not None +#? assert self.bob.friend_get_last_online(self.baid) is not None + return True + + def bob_add_alice_as_friend(self) -> bool: + """ + t:friend_add + t:on_friend_request + t:friend_by_public_key + """ + MSG = 'Alice, this is Bob.' + sSlot = 'friend_request' + if not self.bBobNeedAlice(): return True + + def alices_on_friend_request(iTox, + public_key, + message_data, + message_data_size, + *largs) -> None: + try: + assert str(message_data, 'UTF-8') == MSG + LOG_INFO(f"alices_on_friend_request: {sSlot} = True ") + except Exception as e: + LOG_WARN(f"alices_on_friend_request: EXCEPTION {e}") + # return + setattr(self.bob, sSlot, True) + + setattr(self.bob, sSlot, None) + inum = -1 + try: + inum = self.bob.friend_add(self.alice._address, bytes(MSG, 'UTF-8')) + assert inum >= 0, f"bob_add_alice_as_friend !>= 0 {inum}" + self.alice.callback_friend_request(alices_on_friend_request) + if not self.wait_otox_attrs(self.bob, [sSlot]): + LOG_WARN(f"bob_add_alice_as_friend NO setting {sSlot}") + # return False + self.baid = self.bob.friend_by_public_key(self.alice._address) + assert self.baid >= 0, self.baid + assert self.bob.friend_exists(self.baid) + assert not self.bob.friend_exists(self.baid + 1) + assert self.bob.self_get_friend_list_size() >= 1 + assert self.baid in self.bob.self_get_friend_list() + except Exception as e: + LOG.error(f"bob_add_alice_as_friend EXCEPTION {e}") + return False + finally: + self.bob.callback_friend_message(None) + + return True + + def alice_add_bob_as_friend(self) -> bool: + """ + t:friend_add + t:on_friend_request + t:friend_by_public_key + """ + MSG = 'Bob, this is Alice.' + sSlot = 'friend_request' + if not self.bAliceNeedAddBob(): return True + + def bobs_on_friend_request(iTox, + public_key, + message_data, + message_data_size, + *largs) -> None: + LOG_DEBUG(f"bobs_on_friend_request: " +repr(message_data)) + try: + assert str(message_data, 'UTF-8') == MSG + except Exception as e: + LOG_WARN(f"bobs_on_friend_request: Exception {e}") + # return + setattr(self.alice, sSlot, True) + + LOG_INFO(f"bobs_on_friend_request: {sSlot} = True ") + setattr(self.alice, sSlot, None) + inum = -1 + try: + inum = self.alice.friend_add(self.bob._address, bytes(MSG, 'UTF-8')) + assert inum >= 0, f"alice.friend_add !>= 0 {inum}" + self.bob.callback_friend_request(bobs_on_friend_request) + if not self.wait_otox_attrs(self.alice, [sSlot]): + LOG_WARN(f"alice.friend_add NO wait {sSlot}") + #? return False + self.abid = self.alice.friend_by_public_key(self.bob._address) + assert self.abid >= 0, self.abid + assert self.alice.friend_exists(self.abid), "not exists" + assert not self.alice.friend_exists(self.abid + 1), "exists +1" + assert self.abid in self.alice.self_get_friend_list(), "not in list" + assert self.alice.self_get_friend_list_size() >= 1, "list size" + except Exception as e: + LOG.error(f"alice.friend_add EXCEPTION {e}") + return False + finally: + self.bob.callback_friend_message(None) + return True + + def bob_add_alice_as_friend_and_status(self) -> bool: + if bUSE_NOREQUEST: + assert self.bob_add_alice_as_friend_norequest() + else: + assert self.bob_add_alice_as_friend() + + #: Wait until both are online + sSlot = 'friend_conn_status' + setattr(self.bob, sSlot, False) + def bobs_on_friend_connection_status(iTox, friend_id, iStatus, *largs) -> None: + LOG_INFO(f"bobs_on_friend_connection_status {friend_id} ?>=0" +repr(iStatus)) + setattr(self.bob, sSlot, False) + + sSlot = 'friend_status' + setattr(self.bob, sSlot, None) + def bobs_on_friend_status(iTox, friend_id, iStatus, *largs) -> None: + LOG_INFO(f"bobs_on_friend_status {friend_id} ?>=0" +repr(iStatus)) + setattr(self.bob, sSlot, False) + + sSlot = 'friend_conn_status' + setattr(self.alice, sSlot, None) + def alices_on_friend_connection_status(iTox, friend_id, iStatus, *largs) -> None: + LOG_INFO(f"alices_on_friend_connection_status {friend_id} ?>=0 " +repr(iStatus)) + setattr(self.alice, sSlot, False) + + sSlot = 'friend_status' + setattr(self.alice, sSlot, None) + def alices_on_friend_status(iTox, friend_id, iStatus, *largs) -> None: + LOG_INFO(f"alices_on_friend_status {friend_id} ?>=0 " +repr(iStatus)) + setattr(self.alice, sSlot, False) + + try: + LOG.info("bob_add_alice_as_friend_and_status waiting for alice connections") + if not self.wait_otox_attrs(self.alice, + ['friend_conn_status', + 'friend_status']): + return False + + self.bob.callback_friend_connection_status(bobs_on_friend_connection_status) + self.bob.callback_friend_status(bobs_on_friend_status) + self.alice.callback_friend_connection_status(alices_on_friend_connection_status) + self.alice.callback_friend_status(alices_on_friend_status) + + LOG.info("bob_add_alice_as_friend_and_status waiting for bob connections") + if not self.wait_otox_attrs(self.bob, + ['friend_conn_status', + 'friend_status']): + LOG_WARN('bob_add_alice_as_friend_and_status NO') + # return False + except Exception as e: + LOG.error(f"bob_add_alice_as_friend_and_status ERROR {e}") + return False + finally: + self.alice.callback_friend_connection_status(None) + self.bob.callback_friend_connection_status(None) + self.alice.callback_friend_status(None) + self.bob.callback_friend_status(None) + return True + + def bob_to_alice_connected(self) -> bool: + assert hasattr(self, 'baid') + iRet = self.bob.friend_get_connection_status(self.baid) + if iRet == TOX_CONNECTION['NONE']: + LOG.warn("bob.friend_get_connection_status") + return False + return True + + def alice_to_bob_connected(self) -> bool: + assert hasattr(self, 'abid') + iRet = self.alice.friend_get_connection_status(self.abid) + if iRet == TOX_CONNECTION['NONE']: + LOG.error("alice.friend_get_connection_status") + return False + return True + + def otox_test_groups_create(self, + otox, + group_name='test_group', + nick='test_nick', + topic='Test Topic', # str + ) -> int: + privacy_state = enums.TOX_GROUP_PRIVACY_STATE['PUBLIC'] + + iGrp = otox.group_new(privacy_state, group_name, nick) + assert iGrp >= 0 + LOG.info(f"group iGrp={iGrp}") + + otox.group_set_topic(iGrp, topic) + assert otox.group_get_topic(iGrp) == topic + assert otox.group_get_topic_size(iGrp) == len(topic) + + name = otox.group_get_name(iGrp) + if type(name) == bytes: + name = str(name, 'utf-8') + assert name == group_name, name + assert otox.group_get_name_size(iGrp) == len(group_name) + + sPk = otox.group_self_get_public_key(iGrp) + assert otox.group_get_password_size(iGrp) >= 0 + sP = otox.group_get_password(iGrp) + assert otox.group_get_privacy_state(iGrp) == privacy_state + + assert otox.group_get_number_groups() > 0, "numg={otox.group_get_number_groups()}" + LOG.info(f"group pK={sPk} iGrp={iGrp} numg={otox.group_get_number_groups()}") + return iGrp + + def otox_verify_group(self, otox, iGrp) -> None: + """ + group_self_get_name + group_self_get_peer_id + group_self_get_public_key + group_self_get_role + group_self_get_status + group_self_set_name + """ + + group_number = iGrp + try: + assert type(iGrp) == int, "otox_test_groups_join iGrp not an int" + assert iGrp < UINT32_MAX, "otox_test_groups_join iGrp failure UINT32_MAX" + assert iGrp >= 0, f"otox_test_groups_join iGrp={iGrp} < 0" + sGrp = otox.group_get_chat_id(iGrp) + assert len(sGrp) == enums.TOX_GROUP_CHAT_ID_SIZE * 2, \ + f"group sGrp={sGrp} {len(sGrp)} != {enums.TOX_GROUP_CHAT_ID_SIZE * 2}" + sPk = otox.group_self_get_public_key(iGrp) + LOG.info(f"otox_verify_group sPk={sPk} iGrp={iGrp} n={otox.group_get_number_groups()}") + + sName = otox.group_self_get_name(iGrp) + iStat = otox.group_self_get_status(iGrp) + iId = otox.group_self_get_peer_id(iGrp) + iRole = otox.group_self_get_role(iGrp) + iStat = otox.group_self_get_status(iGrp) + LOG.info(f"otox_verify_group sName={sName} iStat={iStat} iId={iId} iRole={iRole} iStat={iStat}") + + assert otox.group_self_set_name(iGrp, "NewName") + + bRet = otox.group_is_connected(iGrp) + except Exception as e: + LOG.warn(f"group_is_connected EXCEPTION {e}") + return -1 + # chat->connection_state == CS_CONNECTED || chat->connection_state == CS_CONNECTING; + if not bRet: + LOG.warn(f"group_is_connected WARN not connected iGrp={iGrp} n={otox.group_get_number_groups()}") + else: + LOG.info(f"group_is_connected SUCCESS connected iGrp={iGrp} n={otox.group_get_number_groups()}") + try: + bRet = self.group_until_connected(otox, iGrp, iMax=2*otox._args.test_timeout) + except Exception as e: + LOG.error(f"group_until_connected EXCEPTION {e}") + return -1 + # chat->connection_state == CS_CONNECTED || chat->connection_state == CS_CONNECTING; + if bRet: + LOG.warn(f"group_until_connected WARN not connected iGrp={iGrp} n={otox.group_get_number_groups()}") + else: + LOG.info(f"group_until_connected SUCCESS connected iGrp={iGrp} n={otox.group_get_number_groups()}") + + message = bytes('hello', 'utf-8') + bRet = otox.group_send_message(iGrp, TOX_MESSAGE_TYPE['NORMAL'], message) + if not bRet: + LOG.warn(f"group_send_message {bRet}") + else: + LOG.debug(f"group_send_message {bRet}") + + # 360497DA684BCE2A500C1AF9B3A5CE949BBB9F6FB1F91589806FB04CA039E313 + # 75D2163C19FEFFE51508046398202DDC321E6F9B6654E99BAE45FFEC134F05DE + def otox_test_groups_join(self, otox, + chat_id="75d2163c19feffe51508046398202ddc321e6f9b6654e99bae45ffec134f05de", + nick='nick', + topic='Test Topic', # str + ): + status = '' + password = '' + LOG.debug(f"group_join nick={nick} chat_id={chat_id}") + try: + iGrp = otox.group_join(chat_id, password, nick, status) + LOG.info(f"otox_test_groups_join SUCCESS iGrp={iGrp} chat_id={chat_id}") + self.otox_verify_group(otox, iGrp) + + except Exception as e: + # gui + LOG.error(f"otox_test_groups_join EXCEPTION {e}") + raise + + return iGrp + + def otox_test_groups(self, + otox, + group_name='test_group', + nick='test_nick', + topic='Test Topic', # str + ) -> int: + + try: + iGrp = self.otox_test_groups_create(otox, group_name, nick, topic) + self.otox_verify_group(otox, iGrp) + except Exception as e: + LOG.error(f"otox_test_groups ERROR {e}") + raise + + # unfinished + # tox_group_peer_exit_cb + # tox_callback_group_peer_join + # tox.callback_group_peer_status + # tox.callback_group_peer_name + # tox.callback_group_peer_exit + # tox.callback_group_peer_join + return iGrp + + def wait_friend_get_connection_status(self, otox, fid:int, n:int = iN) -> int: + i = 0 + while i < n: + iRet = otox.friend_get_connection_status(fid) + if iRet == TOX_CONNECTION['NONE']: +# LOG.debug(f"wait_friend_get_connection_status NOT CONNECTED i={i} {iRet}") + self.loop_until_connected(otox) + else: + LOG.info(f"wait_friend_get_connection_status {iRet}") + return True + i += 1 + else: + LOG.error(f"wait_friend_get_connection_status n={n}") + return False + + def warn_if_no_cb(self, alice, sSlot:str) -> None: + if not hasattr(alice, sSlot+'_cb') or \ + not getattr(alice, sSlot+'_cb'): + LOG.warning(f"self.bob.{sSlot}_cb NOT EXIST") + + def warn_if_cb(self, alice, sSlot:str) -> None: + if hasattr(self.bob, sSlot+'_cb') and \ + getattr(self.bob, sSlot+'_cb'): + LOG.warning(f"self.bob.{sSlot}_cb EXIST") + + # tests are executed in order + def test_notice_log(self) -> None: # works + notice = '/var/lib/tor/.SelekTOR/3xx/cache/9050/notice.log' + if os.path.exists(notice): + iRet = os.system(f"sudo sed -e '1,/.notice. Bootstrapped 100%/d' {notice}" + \ + "| grep 'Tried for 120 seconds to get a connection to :0.'") + if iRet == 0: + raise SystemExit("seconds to get a connection to :0") + else: + LOG.debug(f"checked {notice}") + + def test_tests_logging(self): # works + with self.assertLogs('foo', level='INFO') as cm: + logging.getLogger('foo').info('first message') + logging.getLogger('foo.bar').error('second message') + logging.getLogger('foo.bar.baz').debug('third message') + self.assertEqual(cm.output, ['INFO:foo:first message', + 'ERROR:foo.bar:second message']) + + def test_hash(self): # works + otox = self.bob + string = 'abcdef' + name = otox.hash(bytes(string, 'utf-8')) + assert name + string = b'abcdef' + name = otox.hash(string) + assert name + LOG.info(f"test_hash: {string} -> {name} ") + + def test_tests_start(self) -> None: # works + """ + t:hash + t:kill + t:libtoxcore + t:options_default + t:options_free + t:options_new + t:self_get_toxid + """ + LOG.info("test_tests_start " ) + port = ts.tox_bootstrapd_port() + + assert len(self.bob._address) == 2*TOX_ADDRESS_SIZE, len(self.bob._address) + assert len(self.alice._address) == 2*TOX_ADDRESS_SIZE, \ + len(self.alice._address) + + assert self.bob.self_get_address() == self.bob._address + assert self.alice.self_get_address() == self.alice._address + + def test_bootstrap_local_netstat(self) -> None: # works + """ + t:callback_file_chunk_request + t:callback_file_recv + t:callback_file_recv_chunk + t:callback_file_recv_control + t:callback_friend_connection_status + t:callback_friend_lossless_packet + t:callback_friend_lossy_packet + t:callback_friend_message + t:callback_friend_name + t:callback_friend_read_receipt + t:callback_friend_request + t:callback_friend_status + t:callback_friend_status_message + t:callback_friend_typing + t:callback_group_custom_packet + t:callback_group_invite + """ + if oTOX_OARGS.network not in ['new', 'newlocal', 'local']: + return + + port = ts.tox_bootstrapd_port() + if not port: + return + iStatus = os.system(f"""netstat -nle4 | grep :{port}""") + if iStatus == 0: + LOG.info(f"bootstrap_local_netstat port {port} iStatus={iStatus}") + else: + LOG.warning(f"bootstrap_local_netstat NOT {port} iStatus={iStatus}") + + def test_bootstrap_local(self, fsocket_timeout:float = fSOCKET_TIMEOUT) -> bool: # works + """ + t:call_bootstrap + t:add_tcp_relay + t:self_get_dht_id + """ + # get port from /etc/tox-bootstrapd.conf 33445 + self.call_bootstrap(fsocket_timeout=fsocket_timeout) + # ts.bootstrap_local(self, self.lUdp) + i = 0 + iStatus = -1 + while i < 10: + i = i + 1 + iStatus = self.bob.self_get_connection_status() + if iStatus != TOX_CONNECTION['NONE']: + break + sleep(3) + else: + pass + + o1 = self.alice.self_get_dht_id() + assert len(o1) == 64 + o2 = self.bob.self_get_dht_id() + assert len(o2) == 64 + +# if o1 != o2: LOG.warning(f"bootstrap_local DHT NOT same {o1} {o2} iStatus={iStatus}") + + iStatus = self.bob.self_get_connection_status() + if iStatus != TOX_CONNECTION['NONE']: + LOG.info(f"bootstrap_local connected iStatus={iStatus}") + return True + iStatus = self.alice.self_get_connection_status() + if iStatus != TOX_CONNECTION['NONE']: + LOG.info(f"bootstrap_local connected iStatus={iStatus}") + return True + LOG.warning(f"bootstrap_local NOT CONNECTED iStatus={iStatus}") + return False + + @unittest.skipIf(os.geteuid() != 0, 'must be root') + def test_bootstrap_iNmapInfo(self) -> None: # works + global oTOX_OARGS +# if os.environ['USER'] != 'root': +# return + iStatus = self.bob.self_get_connection_status() + LOG.info(f"test_bootstrap_iNmapInfo connected bob iStatus={iStatus}") + if oTOX_OARGS.network in ['new', 'newlocal', 'localnew']: + lElts = self.lUdp + elif oTOX_OARGS.proxy_port > 0: + lElts = self.lTcp + else: + lElts = self.lUdp + lRetval = [] + random.shuffle(lElts) + # assert + ts.bootstrap_iNmapInfo(lElts, oTOX_OARGS, "tcp4", bIS_LOCAL=bIS_LOCAL, iNODES=8) + + def test_self_get_secret_key(self) -> None: # works + """ + t:self_get_secret_key + """ + # test_self_get_secret_key + CRYPTO_SECRET_KEY_SIZE = 32 + secret_key = create_string_buffer(CRYPTO_SECRET_KEY_SIZE) + oRet0 = self.alice.self_get_secret_key(secret_key) + assert oRet0, repr(oRet0) + LOG.info('test_self_get_secret_key ' +repr(oRet0)) + assert len(str(oRet0)) + del secret_key + + def test_self_get_public_keys(self) -> None: # works + """ + t:self_get_secret_key + t:self_get_public_key + """ + + LOG.info('test_self_get_public_keys self.alice.self_get_secret_key') + oRet0 = self.alice.self_get_secret_key() + assert len(oRet0) + LOG.info('test_self_get_public_keys ' +repr(oRet0)) + oRet1 = self.alice.self_get_public_key() + assert len(oRet1) + LOG.info('test_self_get_public_keys ' +repr(oRet1)) + assert oRet0 != oRet1, repr(oRet0) +' != ' +repr(oRet1) + + def test_self_name(self) -> None: # works + """ + t:self_set_name + t:self_get_name + t:self_get_name_size + """ + self.alice.self_set_name('Alice') + assert self.alice.self_get_name() == 'Alice' + assert self.alice.self_get_name_size() == len('Alice') + self.bob.self_set_name('Bob') + assert self.bob.self_get_name() == 'Bob' + assert self.bob.self_get_name_size() == len('Bob') + + @unittest.skip('loud') + @unittest.skipIf(bIS_NOT_TOXYGEN or oTOX_OARGS.mode == 0, 'not testing in toxygen') + def test_sound_notification(self) -> None: # works + """ + Plays sound notification + :param type of notification + """ + from tests.toxygen_tests import test_sound_notification + test_sound_notification(self) + + def test_address(self) -> None: # works + """ + t:self_get_address + t:self_get_nospam + t:self_set_nospam + t:self_get_keys + """ + assert len(self.alice.self_get_address()) == ADDR_SIZE + assert len(self.bob.self_get_address()) == ADDR_SIZE + + self.alice.self_set_nospam(0x12345678) + assert self.alice.self_get_nospam() == 0x12345678 + self.loop(50) + + if hasattr(self.alice, 'self_get_keys'): + pk, sk = self.alice.self_get_keys() + assert pk == self.alice.self_get_address()[:CLIENT_ID_SIZE] + + def test_status_message(self) -> None: # works + """ + t:self_get_status_message + t:self_get_status_message_size + """ + MSG = 'Happy' + self.alice.self_set_status_message(MSG) + self.loop(100) + assert self.alice.self_get_status_message() == MSG, \ + self.alice.self_get_status_message() +' is not ' +MSG + assert self.alice.self_get_status_message_size() == len(MSG) + + def test_self_get_udp_port(self) -> None: # works + """ + t:self_get_udp_port + """ + if hasattr(oTOX_OPTIONS, 'udp_port') and oTOX_OPTIONS.udp_port: + o = self.alice.self_get_udp_port() + LOG.info('self_get_udp_port alice ' +repr(o)) + assert o > 0 + o = self.bob.self_get_udp_port() + LOG.info('self_get_udp_port bob ' +repr(o)) + assert o > 0 + + def test_self_get_tcp_port(self) -> None: # works + """ + t:self_get_tcp_port + """ + if hasattr(oTOX_OPTIONS, 'tcp_port') and oTOX_OPTIONS.tcp_port: + # errors if tcp_port <= 0 + o = self.alice.self_get_tcp_port() + LOG.info('self_get_tcp_port ' +repr(o)) + o = self.bob.self_get_tcp_port() + LOG.info('self_get_tcp_port ' +repr(o)) + + def test_get_dht_id(self) -> None: # works + """ + t:self_get_dht_id + """ + o1 = self.alice.self_get_dht_id() + assert len(o1) == 64 + o2 = self.bob.self_get_dht_id() + assert len(o2) == 64 + + def test_bob_add_alice_as_friend_norequest(self) -> None: # works + """ + t:friend_delete + t:friend_exists + t:friend_add_norequest + t:friend_get_public_key + t:self_get_friend_list + t:self_get_friend_list_size + """ + i = len(self.bob.self_get_friend_list()) + assert self.bob_add_alice_as_friend_norequest() + assert len(self.bob.self_get_friend_list()) == i + 1 + #: Test last online + assert self.bob.friend_get_last_online(self.baid) is not None + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + + + def test_alice_add_bob_as_friend_norequest(self) -> None: # works - intermittent failures + """ + t:friend_delete + t:friend_exists + t:friend_get_public_key + t:self_get_friend_list + t:self_get_friend_list_size + """ + i = len(self.alice.self_get_friend_list()) + assert self.alice_add_bob_as_friend_norequest() + assert len(self.alice.self_get_friend_list()) == i + 1 + #: Test last online + assert self.alice.friend_get_last_online(self.abid) is not None + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + + def test_both_add_as_friend_norequest(self) -> None: # works + """ + t:friend_delete + t:friend_exists + t:friend_get_public_key + t:self_get_friend_list + t:self_get_friend_list_size + """ + try: + self.both_add_as_friend_norequest() + assert len(self.bob.self_get_friend_list()) > 0 + assert len(self.alice.self_get_friend_list()) > 0 + except AssertionError as e: + LOG.error(f"Failed test {e}") + raise + except Exception as e: + LOG.error(f"Failed test {e}") + raise + finally: + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + assert len(self.bob.self_get_friend_list()) == 0 + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + assert len(self.alice.self_get_friend_list()) == 0 + + def test_loop_until_connected(self) -> None: # works + assert self.loop_until_connected() + + def test_bob_add_alice_as_friend_and_status(self) -> None: + """ + t:friend_delete + t:friend_exists + t:friend_get_public_key + t:self_get_friend_list + t:self_get_friend_list_size + """ + self.bob_add_alice_as_friend_and_status() + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + + @unittest.skip('unfinished') + def test_alice_add_bob_as_friend_and_status(self) -> None: + assert self.alice_add_bob_as_friend_and_status() + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + + def test_bob_assert_connection_status(self) -> None: # works + if self.bob.self_get_connection_status() == TOX_CONNECTION['NONE']: + AssertionError("ERROR: NOT CONNECTED " \ + +repr(self.bob.self_get_connection_status())) + + def test_alice_assert_connection_status(self) -> None: # works + if self.alice.self_get_connection_status() == TOX_CONNECTION['NONE']: + AssertionError("ERROR: NOT CONNECTED " \ + +repr(self.alice.self_get_connection_status())) + + def test_bob_assert_mycon_status(self) -> None: # works + if self.bob.mycon_status == False: + AssertionError("ERROR: NOT CONNECTED " \ + +repr(self.bob.mycon_status)) + + def test_alice_assert_mycon_status(self) -> None: # works + if self.alice.mycon_status == False: + AssertionError("ERROR: NOT CONNECTED " \ + +repr(self.alice.mycon_status)) + + def test_bob_add_alice_as_friend(self) -> None: # works? + try: + if bUSE_NOREQUEST: + assert self.bob_add_alice_as_friend_norequest() + else: + assert self.bob_add_alice_as_friend() + #: Test last online + assert self.bob.friend_get_last_online(self.baid) is not None + except AssertionError as e: + LOG.error(f"Failed test {e}") + raise + except Exception as e: + LOG.error(f"Failed test {e}") + raise + finally: + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + if len(self.bob.self_get_friend_list()) > 0: + LOG.warn(f"WTF bob.self_get_friend_list() {bob.self_get_friend_list()}") + + def test_alice_add_bob_as_friend(self) -> None: # works! + try: + if bUSE_NOREQUEST: + assert self.alice_add_bob_as_friend_norequest() + else: + assert self.alice_add_bob_as_friend() + #: Test last online + assert self.alice.friend_get_last_online(self.abid) is not None + except AssertionError as e: + #WTF? + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + LOG.error(f"Failed test {e}") + raise + except Exception as e: + #WTF? + LOG.error(f"test_alice_add_bob_as_friend EXCEPTION {e}") + raise + finally: + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + if len(self.alice.self_get_friend_list()) > 0: + LOG.warn(f"WTF alice.self_get_friend_list() {alice.self_get_friend_list()}") + + def test_both_add_as_friend(self) -> None: # works + try: + if bUSE_NOREQUEST: + assert self.both_add_as_friend_norequest() + else: + assert self.both_add_as_friend() + except AssertionError as e: + LOG.warn(f"Failed test {e}") + raise + except Exception as e: + LOG.error(f"test_both_add_as_friend EXCEPTION {e}") + raise + finally: + if hasattr(self,'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + if hasattr(self,'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + + def test_groups_join(self) -> None: + """ + t:group_join + t:group_disconnect + t:group_leave + t:group_self_set_name + """ + if not self.get_connection_status(): + LOG.warning(f"test_groups_join NOT CONNECTED") + self.loop_until_connected(self.bob) + + iGrp = self.otox_test_groups_join(self.bob) + LOG.info(f"test_groups_join iGrp={iGrp}") + assert iGrp >= 0, f"test_groups_join iGrp={iGrp}" + try: + self.bob.group_disconnect(iGrp) + except Exception as e: + LOG.error(f"bob.group_disconnect EXCEPTION {e}") + raise + try: + self.bob.group_leave(iGrp, None) + except Exception as e: + LOG.error(f"bob.group_leave EXCEPTION {e}") + raise + + def test_groups(self) -> None: + """ + t:group_new + t:group_disconnect + t:group_get_name + t:group_get_name_size + t:group_get_topic + t:group_get_topic_size + t:group_get_privacy_state + t:group_self_set_name + t:group_get_number_groups + + t:group_founder_set_password + t:group_founder_set_peer_limit + t:group_founder_set_privacy_state + t:group_get_chat_id + t:group_get_password + t:group_get_password_size + t:group_get_peer_limit + t:group_invite_accept + t:group_invite_friend + t:group_is_connected + t:group_leave + t:group_mod_set_role + """ + iGrp = self.otox_test_groups(self.bob) + LOG.info(f"test_groups iGrp={iGrp}") + if iGrp >= 0: + try: + self.bob.group_disconnect(iGrp) + except Exception as e: + LOG.error(f"bob.group_disconnect EXCEPTION {e}") + raise + try: + self.bob.group_leave(iGrp, None) + except Exception as e: + LOG.error(f"bob.group_leave EXCEPTION {e}") + raise + +#? @unittest.skip("double free or corruption (fasttop)") + @expectedFail('fails') # assertion fails on == MSG + def test_on_friend_status_message(self) -> None: # fails + """ + t:self_set_status_message + t:self_get_status_message + t:self_get_status_message_size + t:friend_set_status_message + t:friend_get_status_message + t:friend_get_status_message_size + t:on_friend_status_message + """ + MSG = 'Happy' + sSlot = 'friend_status_message' + + def bob_on_friend_status_message(iTox, friend_id, new_status_message, new_status_size, *largs) -> None: + LOG_INFO(f"BOB_ON_friend_status_message friend_id={friend_id} " \ + +f"new_status_message={new_status_message}") + try: + assert str(new_status_message, 'UTF-8') == MSG + assert friend_id == self.baid + except Exception as e: + LOG_ERROR(f"BOB_ON_friend_status_message EXCEPTION {e}") + setattr(self.bob, sSlot, True) + + setattr(self.bob, sSlot, None) + try: + if bUSE_NOREQUEST: + assert self.bob_add_alice_as_friend_norequest() + assert self.alice_add_bob_as_friend_norequest() + else: + # no not connected error + assert self.bob_add_alice_as_friend() + assert self.alice_add_bob_as_friend_norequest() + + self.bob.callback_friend_status_message(bob_on_friend_status_message) + self.warn_if_no_cb(self.bob, sSlot) + status_message = bytes(MSG, 'utf-8') + self.alice.self_set_status_message(status_message) + if not self.wait_otox_attrs(self.bob, [sSlot]): + LOG_WARN(f"on_friend_status_message NO {sSlot}") + + assert self.bob.friend_get_status_message(self.baid) == MSG, \ + f"message={self.bob.friend_get_status_message(self.baid)}" + assert self.bob.friend_get_status_message_size(self.baid) == len(MSG), \ + f"message_len={self.bob.friend_get_status_message_size(self.baid)}" + + except AssertionError as e: + LOG.error(f"test_on_friend_status_message FAILED {e}") + raise + except Exception as e: + LOG.error(f"test_on_friend_status_message EXCEPTION {e}") + raise + finally: + self.bob.callback_friend_status(None) + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + + def test_friend(self) -> None: # works! sometimes + """ + t:friend_get_name + t:friend_get_name_size + t:on_friend_name + """ + + try: + #: Test friend request + if bUSE_NOREQUEST: + assert self.bob_add_alice_as_friend_norequest() + assert self.alice_add_bob_as_friend_norequest() + else: + # no not connected error + assert self.bob_add_alice_as_friend() + assert self.alice_add_bob_as_friend_norequest() + + a = self.alice.self_get_address()[:CLIENT_ID_SIZE] + assert self.bob.friend_get_public_key(self.baid) == a, \ + LOG.error(f"test_friend BAID {a}") + del a + + #: Test friend_get_public_key + b = self.bob.self_get_address()[:CLIENT_ID_SIZE] + assert self.alice.friend_get_public_key(self.abid) == b, \ + LOG.error(f"test_friend ABID {b}") + del b + except AssertionError as e: + LOG.error(f"Failed test {e}") + raise + except Exception as e: + LOG.error(f"test_friend EXCEPTION {e}") + raise + finally: + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + + @expectedFail('fails') # assert self.bob.friend_get_status(self.baid) == TOX_USER_STATUS['BUSY'] + def test_user_status(self) -> None: # fails + """ + t:self_get_status + t:self_set_status + t:friend_get_status + t:friend_get_status + t:on_friend_status + """ + sSlot = 'friend_status' + + setattr(self.bob, sSlot, None) + def bobs_on_friend_set_status(iTox, friend_id, new_status, *largs) -> None: + LOG_INFO(f"bobs_on_friend_set_status {friend_id} {new_status}") + try: + assert friend_id == self.baid + assert new_status in [TOX_USER_STATUS['BUSY'], TOX_USER_STATUS['AWAY']] + except Exception as e: + LOG_WARN(f"bobs_on_friend_set_status EXCEPTION {e}") + setattr(self.bob, sSlot, True) + + try: + if bUSE_NOREQUEST: + assert self.bob_add_alice_as_friend_norequest() + else: + assert self.bob_add_alice_as_friend() + if not self.get_connection_status(): + LOG.warning(f"test_user_status NOT CONNECTED self.get_connection_status") + self.loop_until_connected(self.bob) + + self.bob.callback_friend_status(bobs_on_friend_set_status) + self.warn_if_no_cb(self.bob, sSlot) + sSTATUS = TOX_USER_STATUS['BUSY'] + self.alice.self_set_status(sSTATUS) + sSlot = 'friend_status' + if not self.wait_otox_attrs(self.bob, [sSlot]): + # malloc(): unaligned tcache chunk detected + LOG_WARN(f'test_user_status NO {sSlot}') + + assert self.bob.friend_get_status(self.baid) == TOX_USER_STATUS['BUSY'], \ + f"friend_get_status {self.bob.friend_get_status(self.baid)} != {TOX_USER_STATUS['BUSY']}" + + except AssertionError as e: + LOG.error(f"test_user_status FAILED {e}") + raise + except Exception as e: + LOG.error(f"test_user_status EXCEPTION {e}") + raise + finally: + self.bob.callback_friend_status(None) + self.warn_if_cb(self.bob, sSlot) + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + + @unittest.skip('crashes') + def test_kill_remake(self) -> None: + """ + t:friend_get_kill_remake + t:on_friend_connection_status + """ + global oTOX_OARGS + sSlot = 'friend_connection_status' + setattr(self.bob, sSlot, None) + def bobs_on_friend_connection_status(iTox, friend_id, iStatus, *largs): + LOG_INFO(f"bobs_on_friend_connection_status " +repr(iStatus)) + try: + assert friend_id == self.baid + except Exception as e: + LOG_ERROR(f"bobs_on_friend_connection_status ERROR {e}") + setattr(self.bob, sSlot, True) + + opts = oTestsToxOptions(oTOX_OARGS) + setattr(self.bob, sSlot, True) + try: + if bUSE_NOREQUEST: + assert self.bob_add_alice_as_friend_norequest() + else: + assert self.bob_add_alice_as_friend() + + self.bob.callback_friend_connection_status(bobs_on_friend_connection_status) + + LOG.info("test_kill_remake killing alice") + self.alice.kill() #! bang + LOG.info("test_kill_remake making alice") + self.alice = Tox(opts, app=oAPP) + LOG.info("test_kill_remake maked alice") + + if not self.wait_otox_attrs(self.bob, [sSlot]): + LOG_WARN(f'test_kill_remake NO {sSlot}') + except AssertionError as e: + LOG.error(f"test_kill_remake Failed test {e}") + raise + except Exception as e: + LOG.error(f"bobs_on_friend_connection_status {e}") + raise + finally: + self.bob.callback_friend_connection_status(None) + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + + def test_alice_typing_status(self) -> None: # works + """ + t:on_friend_read_receipt + t:on_friend_typing + t:self_set_typing + t:friend_get_typing + t:friend_get_last_online + """ + + sSlot = 'friend_typing' + LOG.info("test_typing_status bob adding alice") + #: Test typing status + def bob_on_friend_typing(iTox, fid:int, is_typing, *largs) -> None: + LOG_INFO(f"BOB_ON_friend_typing is_typing={is_typing} fid={fid}") + try: + assert fid == self.baid + if is_typing is True: + assert self.bob.friend_get_typing(fid) is True + except Exception as e: + LOG_ERROR(f"BOB_ON_friend_typing {e}") + setattr(self.bob, sSlot, True) + + setattr(self.bob, sSlot, None) + try: + if bUSE_NOREQUEST: + assert self.both_add_as_friend_norequest() + else: + assert self.both_add_as_friend() + + if not self.get_connection_status(): + LOG.warning(f"test_friend_typing NOT CONNECTED") + self.loop_until_connected(self.bob) + + self.bob.callback_friend_typing(bob_on_friend_typing) + self.warn_if_no_cb(self.bob, sSlot) + self.alice.self_set_typing(self.abid, False) + if not self.wait_otox_attrs(self.bob, [sSlot]): + LOG_WARN(f"bobs_on_friend_typing NO {sSlot}") + except AssertionError as e: + LOG.error(f"Failed test {e}") + raise + except Exception as e: + LOG.error(f"test_alice_typing_status error={e}") + raise + finally: + self.bob.callback_friend_typing(None) + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + + @expectedFail('fails') # new name is empty + def test_friend_name(self) -> None: # works! + """ + t:self_set_name + t:friend_get_name + t:friend_get_name_size + t:on_friend_name + """ + + sSlot= 'friend_name' + #: Test friend request + + #: Test friend name + NEWNAME = 'Jenny' + + def bobs_on_friend_name(iTox, fid:int, newname, iNameSize, *largs) -> None: + LOG_INFO(f"bobs_on_friend_name {sSlot} {fid}") + try: + assert fid == self.baid + assert str(newname, 'UTF-8') == NEWNAME + except Exception as e: + LOG_ERROR(f"bobs_on_friend_name EXCEPTION {e}") + setattr(self.bob, sSlot, True) + + setattr(self.bob, sSlot, None) + try: + LOG.info("test_friend_name") + if bUSE_NOREQUEST: + assert self.bob_add_alice_as_friend_norequest() + else: + assert self.bob_add_alice_as_friend() + + self.bob.callback_friend_name(bobs_on_friend_name) + self.warn_if_no_cb(self.bob, sSlot) + self.alice.self_set_name(NEWNAME) + if not self.wait_otox_attrs(self.bob, [sSlot]): + LOG_WARN(f"bobs_on_friend_name NO {sSlot}") + + # name=None + assert self.bob.friend_get_name(self.baid) == NEWNAME, \ + f"{self.bob.friend_get_name(self.baid)} != {NEWNAME}" + assert self.bob.friend_get_name_size(self.baid) == len(NEWNAME), \ + f"{self.bob.friend_get_name_size(self.baid)} != {len(NEWNAME)}" + + except AssertionError as e: + LOG.error(f"test_friend_name Failed test {e}") + raise + except Exception as e: + LOG.error(f"test_friend EXCEPTION {e}") + raise + finally: + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + self.bob.callback_friend_name(None) + self.warn_if_cb(self.bob, sSlot) + + @expectedFail('fails') # This client is currently not connected to the friend. + def test_friend_message(self) -> None: # fails intermittently + """ + t:on_friend_action + t:on_friend_message + t:friend_send_message + """ + + #: Test message + MSG = 'Hi, Bob!' + sSlot = 'friend_message' + + def alices_on_friend_message(iTox, fid:int, msg_type, message, iSize, *largs) -> None: + LOG_DEBUG(f"alices_on_friend_message {fid} {message}") + try: + assert fid == self.alice.abid + assert msg_type == TOX_MESSAGE_TYPE['NORMAL'] + assert str(message, 'UTF-8') == MSG + except Exception as e: + LOG_ERROR(f"alices_on_friend_message EXCEPTION {e}") + else: + LOG_INFO(f"alices_on_friend_message {message}") + setattr(self.alice, sSlot, True) + + setattr(self.alice, sSlot, None) + self.alice.callback_friend_message(None) + try: + if bUSE_NOREQUEST: + assert self.both_add_as_friend_norequest() + else: + assert self.both_add_as_friend() + assert hasattr(self, 'baid'), \ + "both_add_as_friend_norequest no bob, baid" + assert hasattr(self, 'abid'), \ + "both_add_as_friend_norequest no alice, abid" + if not self.wait_friend_get_connection_status(self.bob, self.baid, n=2*iN): + LOG.warn('baid not connected') + if not self.wait_friend_get_connection_status(self.alice, self.abid, n=2*iN): + LOG.warn('abid not connected') + self.alice.callback_friend_message(alices_on_friend_message) + self.warn_if_no_cb(self.alice, sSlot) + + # dunno - both This client is currently NOT CONNECTED to the friend. + iMesId = self.bob.friend_send_message(self.baid, + TOX_MESSAGE_TYPE['NORMAL'], + bytes(MSG, 'UTF-8')) + assert iMesId >= 0, "iMesId >= 0" + if not self.wait_otox_attrs(self.alice, [sSlot]): + LOG_WARN(f"alices_on_friend_message NO {sSlot}") + except ArgumentError as e: + # ArgumentError('This client is currently NOT CONNECTED to the friend.') + # dunno + LOG.error(f"test_friend_message ArgumentError {e}") + raise + except AssertionError as e: + LOG.error(f"test_friend_message AssertionError {e}") + raise + except Exception as e: + LOG.error(f"test_friend_message EXCEPTION {e}") + raise + finally: + self.alice.callback_friend_message(None) + self.warn_if_cb(self.alice, sSlot) + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + + # This client is currently not connected to the friend. + def test_friend_action(self) -> None: # works! sometimes? + """ + t:on_friend_action + t:on_friend_message + t:friend_send_message + """ + + #: Test action + ACTION = 'Kick' + sSlot = 'friend_read_action' + setattr(self.bob, sSlot, None) + def UNUSEDtheir_on_friend_action(iTox, fid:int, msg_type, action, *largs): + LOG_DEBUG(f"their_on_friend_action {fid} {msg_type} {sSlot} {action}") + try: + assert msg_type == TOX_MESSAGE_TYPE['ACTION'] + assert action == ACTION + except Exception as e: + LOG_ERROR(f"their_on_friend_action EXCEPTION {sSlot} {e}") + else: + LOG_INFO(f"their_on_friend_action {sSlot} {action}") + setattr(self.bob, sSlot, True) + + sSlot = 'friend_read_receipt' + setattr(self.alice, sSlot, None) + def their_on_read_reciept(iTox, fid:int, msg_id, *largs) -> None: + LOG_DEBUG(f"their_on_read_reciept {fid} {msg_id}") + sSlot = 'friend_read_receipt' + try: + # should be the receivers id + if hasattr(bob, 'baid'): + assert fid == bob.baid + setattr(self.bob, sSlot, True) + elif hasattr(alice, 'abid'): + assert fid == alice.abid + setattr(self.alice, sSlot, True) + assert msg_id >= 0 + except Exception as e: + LOG_ERROR(f"their_on_read_reciept {sSlot} {e}") + else: + LOG_INFO(f"their_on_read_reciept {sSlot} fid={fid}") + + try: + if bUSE_NOREQUEST: + assert self.both_add_as_friend_norequest() + else: + assert self.both_add_as_friend() + + if not self.wait_friend_get_connection_status(self.bob, self.baid, n=iN): + LOG.warn('baid not connected') + if not self.wait_friend_get_connection_status(self.alice, self.abid, n=iN): + LOG.warn('abid not connected') + + self.bob.callback_friend_read_receipt(their_on_read_reciept) #was their_on_friend_action + self.alice.callback_friend_read_receipt(their_on_read_reciept) #was their_on_friend_action + self.warn_if_no_cb(self.bob, 'friend_read_receipt') + self.warn_if_no_cb(self.alice, 'friend_read_receipt') + if True: + iMsg = self.bob.friend_send_message(self.baid, + TOX_MESSAGE_TYPE['ACTION'], + bytes(ACTION, 'UTF-8')) + assert iMsg >= 0 + else: + assert self.wait_ensure_exec(self.bob.friend_send_message, + [self.baid, + TOX_MESSAGE_TYPE['ACTION'], + bytes(ACTION, 'UTF-8')]) + if not self.wait_otox_attrs(self.alice, [sSlot]): + LOG_WARN(f"alice test_friend_action NO {sSlot}") + except AssertionError as e: + LOG.error(f"Failed test {e}") + raise + except ArgumentError as e: + # ArgumentError('This client is currently NOT CONNECTED to the friend.') + # dunno + LOG.warning(f"test_friend_action {e}") + except Exception as e: + LOG.error(f"test_friend_action {e}") + raise + finally: + self.alice.callback_friend_read_receipt(None) + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + + @expectedFail('fails') # @unittest.skip('unfinished') + def test_file_transfer(self) -> None: # unfinished + """ + t:file_send + t:file_send_chunk + t:file_control + t:file_seek + t:file_get_file_id + t:on_file_recv + t:on_file_recv_control + t:on_file_recv_chunk + t:on_file_chunk_request + """ + + if bUSE_NOREQUEST: + assert self.both_add_as_friend_norequest() + else: + assert self.both_add_as_friend() + + FILE_NUMBER = 1 + FILE = os.urandom(1024 * 1024) + FILE_NAME = b"/tmp/test.bin" + if not os.path.exists(FILE_NAME): + with open(FILE_NAME, 'wb') as oFd: + oFd.write(FILE) + FILE_SIZE = len(FILE) + OFFSET = 567 + # was FILE_ID = FILE_NAME + FILE_ID = 32*'1' # + + m = hashlib.md5() + m.update(FILE[OFFSET:]) + FILE_DIGEST = m.hexdigest() + + CONTEXT = { 'FILE': bytes(), 'RECEIVED': 0, 'START': False, 'SENT': 0 } + + def alice_on_file_recv(iTox, fid:int, file_number:int, kind, size, filename) -> None: + LOG_DEBUG(f"ALICE_ON_file_recv fid={fid} {file_number}") + try: + assert size == FILE_SIZE + assert filename == FILE_NAME + retv = self.alice.file_seek(fid, file_number, OFFSET) + assert retv is True + self.alice.file_control(fid, file_number, TOX_FILE_CONTROL['RESUME']) + except Exception as e: + LOG_ERROR(f"ALICE_ON_file_recv {e}") + else: + LOG_INFO(f"ALICE_ON_file_recv " + str(fid)) + + def alice_on_file_recv_control(iTox, fid:int, file_number, control, *largs) -> None: + # TOX_FILE_CONTROL = { 'RESUME': 0, 'PAUSE': 1, 'CANCEL': 2,} + LOG_DEBUG(f"ALICE_ON_file_recv_control fid={fid} {file_number} {control}") + try: + assert FILE_NUMBER == file_number + # FixMe _FINISHED? + if False and control == TOX_FILE_CONTROL['RESUME']: + # assert CONTEXT['RECEIVED'] == FILE_SIZE + # m = hashlib.md5() + # m.update(CONTEXT['FILE']) + # assert m.hexdigest() == FILE_DIGEST + self.alice.completed = True + except Exception as e: + LOG_ERROR(f"ALICE_ON_file_recv {e}") + else: + LOG_INFO(f"ALICE_ON_file_recv " + str(fid)) + + self.alice.completed = False + def alice_on_file_recv_chunk(iTox, fid:int, file_number:int, position:int, iNumBytes, *largs) -> bool: + LOG_DEBUG(f"ALICE_ON_file_recv_chunk {fid} {file_number}") + # FixMe - use file_number and iNumBytes to get data? + data = '' + LOG_INFO(f"ALICE_ON_file_recv_chunk {fid}") + try: + if data is None: + assert CONTEXT['RECEIVED'] == (FILE_SIZE - OFFSET) + m = hashlib.md5() + m.update(CONTEXT['FILE']) + assert m.hexdigest() == FILE_DIGEST + self.alice.completed = True + self.alice.file_control(fid, file_number, TOX_FILE_CONTROL['CANCEL']) + return True + + CONTEXT['FILE'] += data + CONTEXT['RECEIVED'] += len(data) + # if CONTEXT['RECEIVED'] < FILE_SIZE: + # assert self.file_data_remaining( + # fid, file_number, 1) == FILE_SIZE - CONTEXT['RECEIVED'] + except Exception as e: + LOG_ERROR(f"ALICE_ON_file_recv_chunk {e}") + return False + return True + + # AliceTox.on_file_send_request = on_file_send_request + # AliceTox.on_file_control = on_file_control + # AliceTox.on_file_data = on_file_data + + try: + # required? + if not self.wait_friend_get_connection_status(self.bob, self.baid, n=2*iN): + LOG_WARN(f"bobs wait_friend_get_connection_status {2*iN}") + + if not self.wait_friend_get_connection_status(self.alice, self.abid, n=2*iN): + LOG_WARN(f"alices' wait_friend_get_connection_status {2*iN}") + + self.alice.callback_file_recv(alice_on_file_recv) + self.alice.callback_file_recv_control(alice_on_file_recv_control) + self.alice.callback_file_recv_chunk(alice_on_file_recv_chunk) + + self.bob.completed = False + def bob_on_file_recv_control2(iTox, fid:int, file_number:int, control) -> None: + LOG_DEBUG(f"BOB_ON_file_recv_control2 {fid} {file_number} control={control}") + if control == TOX_FILE_CONTROL['RESUME']: + CONTEXT['START'] = True + elif control == TOX_FILE_CONTROL['CANCEL']: + self.bob.completed = True + pass + + def bob_on_file_chunk_request(iTox, fid:int, file_number:int, position:int, length, *largs) -> None: + LOG_DEBUG(f"BOB_ON_file_chunk_request {fid} {file_number}") + if length == 0: + return + data = FILE[position:(position + length)] + self.bob.file_send_chunk(fid, file_number, position, data) + + sSlot = 'file_recv_control' + self.bob.callback_file_recv_control(bob_on_file_recv_control2) + self.bob.callback_file_chunk_request(bob_on_file_chunk_request) + + i = 0 + iKind = 0 + while i < 2: + i += 1 + try: + FN = self.bob.file_send(self.baid, iKind, FILE_SIZE, FILE_ID, FILE_NAME) + LOG.info(f"test_file_transfer bob.file_send {FN}") + except ArgumentError as e: + LOG.debug(f"test_file_transfer bob.file_send {e} {i}") + # ctypes.ArgumentError: This client is currently not connected to the friend + raise + else: + break + self.loop(100) + sleep(1) + else: + LOG.error(f"test_file_transfer bob.file_send 2") + raise AssertionError(f"test_file_transfer bob.file_send {self.bob._args.test_timeout // 2}") + + # UINT32_MAX + try: + FID = self.bob.file_get_file_id(self.baid, FN) + hexFID = "".join([hex(ord(c))[2:].zfill(2) for c in FILE_NAME]) + assert FID.startswith(hexFID.upper()) + except Exception as e: + LOG.warn(f"test_file_transfer:: {FILE_NAME} {hexFID} {e}") + LOG.debug('\n' + traceback.format_exc()) + + if not self.wait_otox_attrs(self.bob, ['completed']): + LOG_WARN(f"test_file_transfer Bob NO completed") + return False + if not self.wait_otox_attrs(self.alice, ['completed']): + LOG_WARN(f"test_file_transfer Alice NO completed") + return False + return True + + except (ArgumentError, ValueError,) as e: + # ValueError: non-hexadecimal number found in fromhex() arg at position 0 + LOG.error(f"test_file_transfer: {e}") + raise + + except Exception as e: + LOG.error(f"test_file_transfer:: {e}") + LOG.debug('\n' + traceback.format_exc()) + raise + + finally: + self.alice.callback_file_recv(None) + self.alice.callback_file_recv_control(None) + self.alice.callback_file_recv_chunk(None) + self.bob.callback_file_recv_control(None) + self.bob.callback_file_chunk_request(None) + if hasattr(self, 'baid') and self.baid >= 0: + self.bob.friend_delete(self.baid) + if hasattr(self, 'abid') and self.abid >= 0: + self.alice.friend_delete(self.abid) + + LOG_INFO(f"test_file_transfer:: self.wait_objs_attr completed") + + @unittest.skip('crashes') + def test_tox_savedata(self) -> None: # + """ + t:get_savedata_size + t:get_savedata + """ + # Fatal Python error: Aborted + # "/var/local/src/toxygen_wrapper/wrapper/tox.py", line 180 in kill + global oTOX_OARGS + + assert self.alice.get_savedata_size() > 0 + data = self.alice.get_savedata() + assert data is not None + addr = self.alice.self_get_address() + # self._address + + try: + LOG.info("test_tox_savedata alice.kill") + # crashes + self.alice.kill() + del self.alice + except: + pass + + oArgs = oTOX_OARGS + opts = oTestsToxOptions(oArgs) + opts.savedata_data = data + opts.savedata_length = len(data) + + self.alice = Tox(tox_options=opts) + if addr != self.alice.self_get_address(): + LOG.warning("test_tox_savedata " + + f"{addr} != {self.alice.self_get_address()}") + else: + LOG.info("passed test_tox_savedata") + + def test_kill(self) -> None: # + import threading + LOG.info(f"THE END {threading.active_count()}") + self.tearDown() + LOG.info(f"THE END {threading.enumerate()}") + + +def vOargsToxPreamble(oArgs, Tox, ToxTest) -> None: + + ts.vSetupLogging(oArgs) + + methods = set([x for x in dir(Tox) if not x[0].isupper() + and not x[0] == '_']) + docs = "".join([getattr(ToxTest, x).__doc__ for x in dir(ToxTest) + if getattr(ToxTest, x).__doc__ is not None]) + + tested = set(re.findall(r't:(.*?)\n', docs)) + not_tested = methods.difference(tested) + + logging.info('Test Coverage: %.2f%%' % (len(tested) * 100.0 / len(methods))) + if len(not_tested): + logging.info('Not tested:\n %s' % "\n ".join(sorted(list(not_tested)))) + + +### + +def iMain(oArgs, failfast=True) -> int: + +# collect_types.init_types_collection() + + vOargsToxPreamble(oArgs, Tox, ToxSuite) + # https://stackoverflow.com/questions/35930811/how-to-sort-unittest-testcases-properly/35930812#35930812 + cases = ts.suiteFactory(*ts.caseFactory([ToxSuite])) + if color_runner: + runner = color_runner.runner.TextTestRunner(verbosity=2, failfast=failfast) + else: + runner = unittest.TextTestRunner(verbosity=2, failfast=failfast, warnings='ignore') + +# with collect_types.collect(): + runner.run(cases) + # collect_types.dump_stats('tests_wrapper.out') + +def oTestsToxOptions(oArgs): + data = None + tox_options = tox_wrapper.tox.Tox.options_new() + if oArgs.proxy_type: + tox_options.contents.proxy_type = int(oArgs.proxy_type) + tox_options.contents.proxy_host = bytes(oArgs.proxy_host, 'UTF-8') + tox_options.contents.proxy_port = int(oArgs.proxy_port) + tox_options.contents.udp_enabled = oArgs.udp_enabled = False + else: + tox_options.contents.udp_enabled = oArgs.udp_enabled + if not os.path.exists('/proc/sys/net/ipv6'): + oArgs.ipv6_enabled = False + else: + tox_options.contents.ipv6_enabled = oArgs.ipv6_enabled + + tox_options.contents.tcp_port = int(oArgs.tcp_port) + tox_options.contents.dht_announcements_enabled = oArgs.dht_announcements_enabled + tox_options.contents.hole_punching_enabled = oArgs.hole_punching_enabled + + # overrides + tox_options.contents.local_discovery_enabled = False + tox_options.contents.experimental_thread_safety = False + # REQUIRED!! + if oArgs.ipv6_enabled and not os.path.exists('/proc/sys/net/ipv6'): + LOG.warning('Disabling IPV6 because /proc/sys/net/ipv6 does not exist' + repr(oArgs.ipv6_enabled)) + tox_options.contents.ipv6_enabled = False + else: + tox_options.contents.ipv6_enabled = bool(oArgs.ipv6_enabled) + + if data: # load existing profile + tox_options.contents.savedata_type = enums.TOX_SAVEDATA_TYPE['TOX_SAVE'] + tox_options.contents.savedata_data = c_char_p(data) + tox_options.contents.savedata_length = len(data) + else: # create new profile + tox_options.contents.savedata_type = enums.TOX_SAVEDATA_TYPE['NONE'] + tox_options.contents.savedata_data = None + tox_options.contents.savedata_length = 0 + + return tox_options + +def oArgparse(lArgv): + global THRESHOLD + parser = ts.oMainArgparser() + parser.add_argument('--norequest',type=str, default='False', + choices=['True','False'], + help='Use _norequest during testing') + parser.add_argument('--test_timeout',type=int, default=THRESHOLD, + help='Test timeout during testing') + parser.add_argument('profile', type=str, nargs='?', default=None, + help='Path to Tox profile') + oArgs = parser.parse_args(lArgv) + ts.clean_booleans(oArgs) + THRESHOLD = oArgs.test_timeout + + if hasattr(oArgs, 'sleep'): + if oArgs.sleep == 'qt': + pass # broken or gevent.sleep(idle_period) + elif oArgs.sleep == 'gevent': + pass # broken or gevent.sleep(idle_period) + else: + oArgs.sleep = 'time' + + return oArgs + +def main(lArgs=None) -> int: + global oTOX_OARGS + global bIS_LOCAL + global THRESHOLD + if lArgs is None: lArgs = sys.argv[1:] + oArgs = oArgparse(lArgs) + bIS_LOCAL = oArgs.network in ['newlocal', 'localnew', 'local'] + THRESHOLD = oArgs.test_timeout + + oTOX_OARGS = oArgs + setattr(oTOX_OARGS, 'bIS_LOCAL', bIS_LOCAL) + bIS_LOCAL = True + setattr(oTOX_OARGS, 'bIS_LOCAL', bIS_LOCAL) + # oTOX_OPTIONS = ToxOptions() + global oTOX_OPTIONS + oTOX_OPTIONS = oTestsToxOptions(oArgs) + if coloredlogs: + # https://pypi.org/project/coloredlogs/ + coloredlogs.install(level=oArgs.loglevel, + logger=LOG, + # %(asctime)s,%(msecs)03d %(hostname)s [%(process)d] + fmt='%(name)s %(levelname)s %(message)s' + ) + else: + logging.basicConfig(level=oArgs.loglevel) # logging.INFO + + return iMain(oArgs) + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:] )) diff --git a/src/toxygen_wrapper/tox.py b/src/toxygen_wrapper/tox.py new file mode 100644 index 0000000..a54af56 --- /dev/null +++ b/src/toxygen_wrapper/tox.py @@ -0,0 +1,3380 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- + +# ctypes wrapping of libtoxcore + +# WIP: - all functions are being changed to accept strings or byres for variables +# the library will use as bytes, and return sstrings not bytes for things +# you will use as strings. YMMV. + +from ctypes import * +from datetime import datetime +from typing import Union, Callable + +try: + from tox_wrapper.libtox import LibToxCore + from tox_wrapper.toxav import ToxAV + from tox_wrapper.toxcore_enums_and_consts import * + import tox_wrapper.toxcore_enums_and_consts as enum +except: + from libtox import LibToxCore + from toxav import ToxAV + from toxcore_enums_and_consts import * + import toxcore_enums_and_consts as enum + +_c_uint32 = POINTER(c_uint32) + +# callbacks can be called in any thread so were being careful +# tox.py can be called by callbacks +def LOG_ERROR(a) -> None: + print('EROR> '+a) +def LOG_WARN(a) -> None: + print('WARN> '+a) +def LOG_INFO(a) -> None: + bVERBOSE = hasattr(__builtins__, 'app') and app.oArgs.loglevel <= 20 + if bVERBOSE: print('INFO> '+a) +def LOG_DEBUG(a) -> None: + bVERBOSE = hasattr(__builtins__, 'app') and app.oArgs.loglevel <= 10 + if bVERBOSE: print('DBUG> '+a) +def LOG_TRACE(a) -> None: + bVERBOSE = hasattr(__builtins__, 'app') and app.oArgs.loglevel < 10 + if bVERBOSE: print('TRAC> '+a) + +UINT32_MAX = 2 ** 32 -1 +class ToxError(RuntimeError): pass +TOX_MAX_STATUS_MESSAGE_LENGTH = 1007 + +global aTIMES +aTIMES=dict() +def bTooSoon(key, sSlot, fSec=10.0) -> bool: + # rate limiting + global aTIMES + if sSlot not in aTIMES: + aTIMES[sSlot] = dict() + OTIME = aTIMES[sSlot] + now = datetime.now() + if key not in OTIME: + OTIME[key] = now + return False + delta = now - OTIME[key] + OTIME[key] = now + if delta.total_seconds() < fSec: return True + return False + + +class ToxOptions(Structure): + _fields_ = [ + ('ipv6_enabled', c_bool), + ('udp_enabled', c_bool), + ('local_discovery_enabled', c_bool), + ('dht_announcements_enabled', c_bool), + ('proxy_type', c_int), + ('proxy_host', c_char_p), + ('proxy_port', c_uint16), + ('start_port', c_uint16), + ('end_port', c_uint16), + ('tcp_port', c_uint16), + ('hole_punching_enabled', c_bool), + ('savedata_type', c_int), + ('savedata_data', c_char_p), + ('savedata_length', c_size_t), + ('log_callback', c_void_p), + ('log_user_data', c_void_p), + ('experimental_thread_safety', c_bool), + ('operating_system', c_void_p), + ] + + +class GroupChatSelfPeerInfo(Structure): + _fields_ = [ + ('nick', c_char_p), + ('nick_length', c_uint8), + ('user_status', c_int) + ] + +def string_to_bin_charp(tox_id): + if tox_id is None: return None + assert type(tox_id) == str, f"{type(tox_id)} != str" + return c_char_p(bytes.fromhex(tox_id)) + + +def bin_to_string(raw_id, length) -> str: + assert isinstance(raw_id, bytes) or isinstance(raw_id, Array), \ + f"{type(raw_id)} != bytes" + res = ''.join('{:02x}'.format(ord(raw_id[i])) for i in range(length)) + return res.upper() + +def sGetError(value, a) -> str: + # dict(enumerate(a))[value] + for k,v in a.items(): + if v == value: + s = k + return s + return '' + +class Tox: + libtoxcore = LibToxCore() + + def __init__(self, tox_options=None, tox_pointer=None, app=None): + """Creates and initialises a new Tox instance with the options passed. + + This function will bring the instance into a valid state. + Running the event loop with a new instance will operate correctly. + + :param tox_options: An options object. If this parameter is None, the default options are used. + :param tox_pointer: Tox instance pointer. If this parameter is not None, tox_options will be ignored. + + """ + self._app = app # QtWidgets.QApplication.instance() + if tox_pointer is not None: + self._tox_pointer = tox_pointer + else: + tox_err_new = c_int() + f = Tox.libtoxcore.tox_new + f.restype = POINTER(c_void_p) + self._tox_pointer = f(tox_options, byref(tox_err_new)) + tox_err_new = tox_err_new.value + if tox_err_new == TOX_ERR_NEW['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if tox_err_new == TOX_ERR_NEW['MALLOC']: + raise MemoryError('The function was unable to allocate enough ' + 'memory to store the internal structures for the Tox object.') + if tox_err_new == TOX_ERR_NEW['PORT_ALLOC']: + raise ToxError('The function was unable to bind to a port. This may mean that all ports have ' + 'already been bound, e.g. by other Tox instances, or it may mean a permission error.' + ' You may be able to gather more information from errno.') + if tox_err_new == TOX_ERR_NEW['TCP_SERVER_ALLOC']: + raise ToxError('The function was unable to bind the tcp server port.') + if tox_err_new == TOX_ERR_NEW['PROXY_BAD_TYPE']: + raise ArgumentError('proxy_type was invalid.') + if tox_err_new == TOX_ERR_NEW['PROXY_BAD_HOST']: + raise ArgumentError('proxy_type was valid but the proxy_host passed had an invalid format or was NULL.') + if tox_err_new == TOX_ERR_NEW['PROXY_BAD_PORT']: + raise ArgumentError('proxy_type was valid, but the proxy_port was invalid.') + if tox_err_new == TOX_ERR_NEW['PROXY_NOT_FOUND']: + raise ArgumentError('The proxy address passed could not be resolved.') + if tox_err_new == TOX_ERR_NEW['LOAD_ENCRYPTED']: + raise ArgumentError('The byte array to be loaded contained an encrypted save.') + if tox_err_new == TOX_ERR_NEW['LOAD_BAD_FORMAT']: + raise ArgumentError('The data format was invalid. This can happen when loading data that was saved by' + ' an older version of Tox, or when the data has been corrupted. When loading from' + ' badly formatted data, some data may have been loaded, and the rest is discarded.' + ' Passing an invalid length parameter also causes this error.') + + self.self_connection_status_cb = None + self.self_logger_cb = None + self.friend_name_cb = None + self.friend_status_message_cb = None + self.friend_status_cb = None + self.friend_connection_status_cb = None + self.friend_request_cb = None + self.friend_read_receipt_cb = None + self.friend_typing_cb = None + self.friend_message_cb = None + self.file_recv_control_cb = None + self.file_chunk_request_cb = None + self.file_recv_cb = None + self.file_recv_chunk_cb = None + self.friend_lossy_packet_cb = None + self.friend_lossless_packet_cb = None + self.group_moderation_cb = None + self.group_join_fail_cb = None + self.group_self_join_cb = None + self.group_invite_cb = None + self.group_custom_packet_cb = None + self.group_private_message_cb = None + self.group_message_cb = None + + self.group_password_cb = None + self.group_peer_limit_cb = None + self.group_privacy_state_cb = None + self.group_topic_cb = None + self.group_peer_status_cb = None + self.group_peer_name_cb = None + self.group_peer_exit_cb = None + self.group_peer_join_cb = None + self.AV = ToxAV(self._tox_pointer) + + def kill(self) -> None: + if hasattr(self, 'AV'): del self.AV + LOG_INFO(f"tox.kill") + try: + Tox.libtoxcore.tox_kill(self._tox_pointer) + except Exception as e: + LOG_ERROR(f"tox.kill {e!s}") + else: + LOG_DEBUG(f"tox.kill") + return None + + # Startup options + + @staticmethod + def options_default(tox_options) -> None: + """Initialises a Tox_Options object with the default options. + + The result of this function is independent of the original + options. All values will be overwritten, no values will be read + (so it is permissible to pass an uninitialised object). + + If options is NULL, this function has no effect. + + :param tox_options: A pointer to options object to be filled with default options. + return value: None + """ + LOG_DEBUG(f"tox.options_default") + Tox.libtoxcore.tox_options_default(tox_options) + return None + + @staticmethod + def options_new(): # a pointer + """Allocates a new Tox_Options object and initialises it with + + the default options. This function can be used to preserve long + term ABI compatibility by giving the responsibility of + allocation and deallocation to the Tox library. + + Objects returned from this function must be freed using the + tox_options_free function. + + :return: A pointer to new ToxOptions object with default options or raise MemoryError. + """ + tox_err_options_new = c_int() + f = Tox.libtoxcore.tox_options_new + f.restype = POINTER(ToxOptions) + result = f(byref(tox_err_options_new)) + result._options_pointer = result + tox_err_options_new = tox_err_options_new.value + if tox_err_options_new == TOX_ERR_OPTIONS_NEW['OK']: + return result + if tox_err_options_new == TOX_ERR_OPTIONS_NEW['MALLOC']: + raise MemoryError('The function failed to allocate enough memory for the options struct.') + raise ToxError('The function did not return OK for the options struct.') + + @staticmethod + def options_free(tox_options) -> None: + """ + Releases all resources associated with an options objects. + + Passing a pointer that was not returned by tox_options_new results in undefined behaviour. + + :param tox_options: A pointer to new ToxOptions object + """ + LOG_DEBUG(f"tox.options_free") + Tox.libtoxcore.tox_options_free(tox_options) + + # Creation and destruction + + def get_savedata_size(self) -> int: + """ + Calculates the number of bytes required to store the tox instance with tox_get_savedata. + This function cannot fail. The result is always greater than 0. + + :return: number of bytes + """ + return int(Tox.libtoxcore.tox_get_savedata_size(self._tox_pointer)) + + def get_savedata(self, savedata: Union[Array, None]=None) -> bytes: + """ + Store all information associated with the tox instance to a byte array. + + :param savedata: pointer (c_char_p) to a memory region large enough to store the tox instance data. + Call tox_get_savedata_size to find the number of bytes required. If this parameter is None, this function + allocates memory for the tox instance data. + :return: pointer (c_char_p) to a memory region with the tox instance data + """ + if savedata is None: + savedata_size = self.get_savedata_size() + savedata = create_string_buffer(savedata_size) + else: + isinstance(savedata, Array), type(savedata) + LOG_DEBUG(f"tox.get_savedata") + Tox.libtoxcore.tox_get_savedata(self._tox_pointer, savedata) + return bytes(savedata[:]) + + # Connection lifecycle and event loop + + def bootstrap(self, address: Union[str,bytes], port: int, public_key: Union[bytes,str]) -> bool: + """Sends a "get nodes" request to the given bootstrap node with IP, port, and public key to setup connections. + + This function will attempt to connect to the node using UDP. + You must use this function even if Tox_Options.udp_enabled was + set to false. + + :param address: The hostname or IP address (IPv4 or IPv6) of the node. + :param port: The port on the host on which the bootstrap Tox instance is listening. + :param public_key: The long term public key of the bootstrap node (TOX_PUBLIC_KEY_SIZE bytes). + :return: True on success. + + """ + LOG_TRACE(f"tox_bootstrap={address}") + if type(address) == str: + address = bytes(address, 'utf-8') + if type(public_key) == bytes: + public_key = str(public_key, 'utf-8') + tox_err_bootstrap = c_int() + try: + result = Tox.libtoxcore.tox_bootstrap(self._tox_pointer, + c_char_p(address), + c_uint16(port), + string_to_bin_charp(public_key), + byref(tox_err_bootstrap)) + except Exception as e: + # Fatal Python error: Segmentation fault + LOG_ERROR(f"libtoxcore.tox_bootstrap {e}") + # dunno + raise + + tox_err_bootstrap = tox_err_bootstrap.value + if tox_err_bootstrap == TOX_ERR_BOOTSTRAP['OK']: + return bool(result) + if tox_err_bootstrap == TOX_ERR_BOOTSTRAP['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if tox_err_bootstrap == TOX_ERR_BOOTSTRAP['BAD_HOST']: + raise ArgumentError('The address could not be resolved to an IP ' + 'address, or the address passed was invalid.') + if tox_err_bootstrap == TOX_ERR_BOOTSTRAP['BAD_PORT']: + raise ArgumentError('The port passed was invalid. The valid port range is (1, 65535).') + # me - this seems wrong - should be False + return False + + def add_tcp_relay(self, address: Union[str,bytes], port: int, public_key: Union[bytes,str]) -> bool: + """Adds additional host:port pair as TCP relay. + + This function can be used to initiate TCP connections to + different ports on the same bootstrap node, or to add TCP + relays without using them as bootstrap nodes. + + :param address: The hostname or IP address (IPv4 or IPv6) of the TCP relay. + :param port: The port on the host on which the TCP relay is listening. + :param public_key: The long term public key of the TCP relay (TOX_PUBLIC_KEY_SIZE bytes). + :return: True on success. + + """ + LOG_TRACE(f"tox_add_tcp_relay address={address}") + if type(address) == str: + address = bytes(address, 'utf-8') + if type(public_key) == bytes: + public_key = str(public_key, 'utf-8') + tox_err_bootstrap = c_int() + result = Tox.libtoxcore.tox_add_tcp_relay(self._tox_pointer, + c_char_p(address), + c_uint16(port), + string_to_bin_charp(public_key), + byref(tox_err_bootstrap)) + tox_err_bootstrap = tox_err_bootstrap.value + if tox_err_bootstrap == TOX_ERR_BOOTSTRAP['OK']: + return bool(result) + if tox_err_bootstrap == TOX_ERR_BOOTSTRAP['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if tox_err_bootstrap == TOX_ERR_BOOTSTRAP['BAD_HOST']: + raise ArgumentError('The address could not be resolved to an IP ' + 'address, or the IP address passed was invalid.') + if tox_err_bootstrap == TOX_ERR_BOOTSTRAP['BAD_PORT']: + raise ArgumentError('The port passed was invalid. The valid port range is (1, 65535).') + raise ToxError('The function did not return OK') + + def self_get_connection_status(self) -> int: + """ + Return whether we are connected to the DHT. + The return value is equal to the last value received through the + `self_connection_status` callback. + + :return: TOX_CONNECTION + """ + iRet = Tox.libtoxcore.tox_self_get_connection_status(self._tox_pointer) + if iRet > 2: + LOG_ERROR(f"self_get_connection_status {iRet} > 2") + return 0 + LOG_TRACE(f"self_get_connection_status {iRet}") + return int(iRet) + + def callback_self_connection_status(self, callback: Union[Callable,None]) -> None: + """Set the callback for the `self_connection_status` event. + Pass None to unset. + + This event is triggered whenever there is a change in the DHT + connection state. When disconnected, a client may choose to + call tox_bootstrap again, to reconnect to the DHT. Note that + this state may frequently change for short amounts of + time. Clients should therefore not immediately bootstrap on + receiving a disconnect. + + :param callback: Python function. Should take + pointer (c_void_p) to Tox object, + TOX_CONNECTION (c_int), + pointer (c_void_p) to user_data + + """ + if callback is None: + Tox.libtoxcore.tox_callback_self_connection_status(self._tox_pointer, + POINTER(None)()) + self.self_connection_status_cb = None + return + + c_callback = CFUNCTYPE(None, c_void_p, c_int, c_void_p) + self.self_connection_status_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_self_connection_status") + Tox.libtoxcore.tox_callback_self_connection_status(self._tox_pointer, + self.self_connection_status_cb) + + def iteration_interval(self) -> int: + """ + Return the time in milliseconds before tox_iterate() should be + called again for optimal performance. + :return: time in milliseconds + + """ + return int(Tox.libtoxcore.tox_iteration_interval(self._tox_pointer)) + + def iterate(self, user_data: Union[bytes,None] = None) -> None: # void + """ + The main loop that needs to be run in intervals of tox_iteration_interval() milliseconds. + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + try: + LOG_TRACE(f"tox_iterate") + # void pointer + Tox.libtoxcore.tox_iterate(self._tox_pointer, c_char_p(user_data)) + except Exception as e: + # Fatal Python error: Segmentation fault + LOG_ERROR(f"iterate {e!s}") + else: + LOG_TRACE(f"iterate") + + # Internal client information (Tox address/id) + + def self_get_toxid(self, address: Union[Array, None]=None) -> str: + return self.self_get_address(address) + + def self_get_address(self, address: Union[Array, None]=None) -> str: + """ + Writes the Tox friend address of the client to a byte array. The address is not in human-readable format. If a + client wants to display the address, formatting is required. + + :param address: pointer (c_char_p) to a memory region of at least TOX_ADDRESS_SIZE bytes. If this parameter is + None, this function allocates memory for address. + :return: Tox friend address + """ + if address is None: + address = create_string_buffer(TOX_ADDRESS_SIZE) + else: + isinstance(address, Array), type(address) + LOG_INFO(f"tox.self_get_address") + Tox.libtoxcore.tox_self_get_address(self._tox_pointer, address) + return bin_to_string(address, TOX_ADDRESS_SIZE) + + def self_set_nospam(self, nospam: int) -> None: + """ + Set the 4-byte nospam part of the address. + + :param nospam: Any 32 bit unsigned integer. + """ + LOG_DEBUG(f"tox.self_set_nospam") + Tox.libtoxcore.tox_self_set_nospam(self._tox_pointer, c_uint32(nospam)) + + def self_get_nospam(self) -> int: + """ + Get the 4-byte nospam part of the address. + + :return: nospam part of the address + """ + return int(Tox.libtoxcore.tox_self_get_nospam(self._tox_pointer)) + + def self_get_public_key(self, public_key: Union[Array, None] = None) -> str: + """ + Copy the Tox Public Key (long term) from the Tox object. + + :param public_key: A memory region of at least TOX_PUBLIC_KEY_SIZE bytes. If this parameter is NULL, this + function allocates memory for Tox Public Key. + :return: Tox Public Key + """ + if public_key is None: + public_key = create_string_buffer(TOX_PUBLIC_KEY_SIZE) + else: + isinstance(public_key, Array), type(public_key) + LOG_DEBUG(f"tox.self_get_public_key") + Tox.libtoxcore.tox_self_get_public_key(self._tox_pointer, public_key) + return bin_to_string(public_key, TOX_PUBLIC_KEY_SIZE) + + def self_get_secret_key(self, secret_key: Union[Array, None] = None) -> str: + """ + Copy the Tox Secret Key from the Tox object. + + :param secret_key: pointer (c_char_p) to a memory region of at least TOX_SECRET_KEY_SIZE bytes. If this + parameter is NULL, this function allocates memory for Tox Secret Key. + :return: Tox Secret Key + """ + if secret_key is None: + secret_key = create_string_buffer(TOX_SECRET_KEY_SIZE) + else: + isinstance(secret_key, Array), type(secret_key) + LOG_DEBUG(f"tox.self_get_secret_key") + Tox.libtoxcore.tox_self_get_secret_key(self._tox_pointer, secret_key) + return bin_to_string(secret_key, TOX_SECRET_KEY_SIZE) + + # User-visible client information (nickname/status) + + def self_set_name(self, name: Union[bytes,str]) -> bool: + """ + Set the nickname for the Tox client. + + Nickname length cannot exceed TOX_MAX_NAME_LENGTH. If length is 0, the name parameter is ignored + (it can be None), and the nickname is set back to empty. + :param name: New nickname. + :return: True on success. + """ + tox_err_set_info = c_int() + if type(name) == str: + name = bytes(name, 'utf-8') + LOG_DEBUG(f"tox.self_set_name") + result = Tox.libtoxcore.tox_self_set_name(self._tox_pointer, + name, # sic + c_size_t(len(name)), + byref(tox_err_set_info)) + tox_err_set_info = tox_err_set_info.value + if tox_err_set_info == TOX_ERR_SET_INFO['OK']: + return bool(result) + elif tox_err_set_info == TOX_ERR_SET_INFO['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + elif tox_err_set_info == TOX_ERR_SET_INFO['TOO_LONG']: + raise ArgumentError('Information length exceeded maximum permissible size.') + raise ToxError('The function did not return OK') + + def self_get_name_size(self) -> int: + """ + Return the length of the current nickname as passed to tox_self_set_name. + + If no nickname was set before calling this function, the name is empty, and this function returns 0. + + :return: length of the current nickname + """ + retval = Tox.libtoxcore.tox_self_get_name_size(self._tox_pointer) + return int(retval) + + def self_get_name(self, name: Union[Array,None] = None) -> str: + """ + Write the nickname set by tox_self_set_name to a byte array. + + If no nickname was set before calling this function, the name is empty, and this function has no effect. + + Call tox_self_get_name_size to find out how much memory to allocate for the result. + + :param name: pointer (c_char_p) to a memory region location large enough to hold the nickname. If this parameter + is NULL, the function allocates memory for the nickname. + :return: nickname + """ + if name is None: + name = create_string_buffer(self.self_get_name_size()) + else: + isinstance(name, Array), type(name) + LOG_DEBUG(f"tox.self_get_name") + Tox.libtoxcore.tox_self_get_name(self._tox_pointer, name) + return str(name.value, 'utf-8', errors='ignore') + + def self_set_status_message(self, status_message: Union[bytes,str]) -> bool: + """Set the client's status message. + + Status message length cannot exceed TOX_MAX_STATUS_MESSAGE_LENGTH. + If length is 0, the status parameter is ignored, and the user status is + set back to empty. + + :param status_message: new status message + :return: True on success. + """ + tox_err_set_info = c_int() + if len(status_message) > TOX_MAX_STATUS_MESSAGE_LENGTH: + status_message = status_message[:TOX_MAX_STATUS_MESSAGE_LENGTH] + if type(status_message) == str: + status_message = bytes(status_message, 'utf-8') + LOG_DEBUG(f"tox.self_set_status_message") + result = Tox.libtoxcore.tox_self_set_status_message(self._tox_pointer, + status_message, # sic + c_size_t(len(status_message)), + byref(tox_err_set_info)) + tox_err_set_info = tox_err_set_info.value + if tox_err_set_info == TOX_ERR_SET_INFO['OK']: + return bool(result) + if tox_err_set_info == TOX_ERR_SET_INFO['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if tox_err_set_info == TOX_ERR_SET_INFO['TOO_LONG']: + raise ArgumentError('Information length exceeded maximum permissible size.') + raise ToxError('The function did not return OK.') + + def self_get_status_message_size(self) -> int: + """ + Return the length of the current status message as passed to tox_self_set_status_message. + + If no status message was set before calling this function, the status is empty, and this function returns 0. + + :return: length of the current status message + """ + return Tox.libtoxcore.tox_self_get_status_message_size(self._tox_pointer) + + def self_get_status_message(self, status_message: Union[Array,None] = None) -> str: + """ + Write the status message set by tox_self_set_status_message to a byte array. + + If no status message was set before calling this function, the status is empty, and this function has no effect. + + Call tox_self_get_status_message_size to find out how much memory to allocate for the result. + + :param status_message: pointer (c_char_p) to a valid memory location large enough to hold the status message. + If this parameter is None, the function allocates memory for the status message. + :return: status message + """ + if status_message is None: + status_message = create_string_buffer(self.self_get_status_message_size()) + else: + isinstance(status_message, Array), type(status_message) + LOG_DEBUG(f"tox.self_get_status_message") + Tox.libtoxcore.tox_self_get_status_message(self._tox_pointer, status_message) + return str(status_message.value, 'utf-8', errors='ignore') + + def self_set_status(self, status: int) -> None: + """ + Set the client's user status. + + :param status: One of the user statuses listed in the enumeration TOX_USER_STATUS. + """ + if bTooSoon('self', 'tox_self_set_status', 5.0): return None + LOG_DEBUG(f"tox.self_set_status {status}") + Tox.libtoxcore.tox_self_set_status(self._tox_pointer, c_uint32(status)) + return None + + def self_get_status(self) -> int: + """ + Returns the client's user status. + + :return: client's user status + """ + LOG_TRACE(f"tox_get_status") + result = Tox.libtoxcore.tox_self_get_status(self._tox_pointer) + return int(result) + + # Friend list management + + def friend_add(self, address: Union[bytes,str], message: Union[bytes,str]) -> int: + """Add a friend to the friend list and send a friend request. + + A friend request message must be at least 1 byte long and at + most TOX_MAX_FRIEND_REQUEST_LENGTH. + + Friend numbers are unique identifiers used in all functions + that operate on friends. Once added, a friend number is stable + for the lifetime of the Tox object. After saving the state and + reloading it, the friend numbers may not be the same as + before. Deleting a friend creates a gap in the friend number + set, which is filled by the next adding of a friend. Any + pattern in friend numbers should not be relied on. + + If more than INT32_MAX friends are added, this function causes + undefined behaviour. + + :param address: The address of the friend (returned by tox_self_get_address of the friend you wish to add) it + must be TOX_ADDRESS_SIZE bytes. + :param message: The message that will be sent along with the friend request. + :return: the friend number on success, UINT32_MAX on failure. + + """ + tox_err_friend_add = c_int() + LOG_DEBUG(f"tox.friend_add") + if type(address) == bytes: + address = str(address, 'utf-8') + if type(message) == str: + message = bytes(message, 'utf-8') + result = Tox.libtoxcore.tox_friend_add(self._tox_pointer, + string_to_bin_charp(address), + message, # sic + c_size_t(len(message)), + byref(tox_err_friend_add)) + tox_err_friend_add = tox_err_friend_add.value + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['OK']: + return int(result) + + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['TOO_LONG']: + raise ArgumentError('The length of the friend request message exceeded TOX_MAX_FRIEND_REQUEST_LENGTH.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['NO_MESSAGE']: + raise ArgumentError('The friend request message was empty. This, and the TOO_LONG code will never be' + ' returned from tox_friend_add_norequest.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['OWN_KEY']: + raise ArgumentError('The friend address belongs to the sending client.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['ALREADY_SENT']: + raise ArgumentError('A friend request has already been sent, or the address belongs to a friend that is' + ' already on the friend list.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['BAD_CHECKSUM']: + raise ArgumentError('The friend address checksum failed.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['SET_NEW_NOSPAM']: + raise ArgumentError('The friend was already there, but the nospam value was different.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['MALLOC']: + raise MemoryError('A memory allocation failed when trying to increase the friend list size.') + raise ToxError('The function did not return OK for the friend add.') + + def friend_add_norequest(self, public_key: Union[bytes,str]) -> int: + """Add a friend without sending a friend request. + + This function is used to add a friend in response to a friend + request. If the client receives a friend request, it can be + reasonably sure that the other client added this client as a + friend, eliminating the need for a friend request. + + This function is also useful in a situation where both + instances are controlled by the same entity, so that this + entity can perform the mutual friend adding. In this case, + there is no need for a friend request, either. + + :param public_key: A byte array of length TOX_PUBLIC_KEY_SIZE containing the Public Key (not the Address) of the + friend to add. + :return: the friend number on success, UINT32_MAX on failure. + + """ + tox_err_friend_add = c_int() + LOG_DEBUG(f"tox.friend_add_norequest") + if type(public_key) == bytes: + public_key = str(public_key, 'utf-8') + result = Tox.libtoxcore.tox_friend_add_norequest(self._tox_pointer, + string_to_bin_charp(public_key), + byref(tox_err_friend_add)) + tox_err_friend_add = tox_err_friend_add.value + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['OK']: + return int(result) + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['TOO_LONG']: + raise ArgumentError('The length of the friend request message exceeded TOX_MAX_FRIEND_REQUEST_LENGTH.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['NO_MESSAGE']: + raise ArgumentError('The friend request message was empty. This, and the TOO_LONG code will never be' + ' returned from tox_friend_add_norequest.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['OWN_KEY']: + raise ArgumentError('The friend address belongs to the sending client.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['ALREADY_SENT']: + raise ArgumentError('A friend request has already been sent, or the address belongs to a friend that is' + ' already on the friend list.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['BAD_CHECKSUM']: + raise ArgumentError('The friend address checksum failed.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['SET_NEW_NOSPAM']: + raise ArgumentError('The friend was already there, but the nospam value was different.') + if tox_err_friend_add == TOX_ERR_FRIEND_ADD['MALLOC']: + raise MemoryError('A memory allocation failed when trying to increase the friend list size.') + raise ToxError('The function did not return OK for the friend add.') + + def friend_delete(self, friend_number: int) -> bool: + """ + Remove a friend from the friend list. + + This does not notify the friend of their deletion. After calling this function, this client will appear offline + to the friend and no communication can occur between the two. + + :param friend_number: Friend number for the friend to be deleted. + :return: True on success. + """ + tox_err_friend_delete = c_int() + LOG_DEBUG(f"tox.friend_delete") + result = Tox.libtoxcore.tox_friend_delete(self._tox_pointer, + c_uint32(friend_number), + byref(tox_err_friend_delete)) + tox_err_friend_delete = tox_err_friend_delete.value + if tox_err_friend_delete == TOX_ERR_FRIEND_DELETE['OK']: + return bool(result) + elif tox_err_friend_delete == TOX_ERR_FRIEND_DELETE['FRIEND_NOT_FOUND']: + raise ArgumentError('There was no friend with the given friend number. No friends were deleted.') + raise ToxError('The function did not return OK for the friend add.') + + # Friend list queries + + def friend_by_public_key(self, public_key: Union[str,bytes]) -> int: + """ + Return the friend number associated with that Public Key. + + :param public_key: A byte array containing the Public Key. + :return: friend number + """ + tox_err_friend_by_public_key = c_int() + LOG_DEBUG(f"tox.friend_by_public_key") + if type(public_key) == bytes: + public_key = str(public_key, 'utf-8') + result = Tox.libtoxcore.tox_friend_by_public_key(self._tox_pointer, + string_to_bin_charp(public_key), + byref(tox_err_friend_by_public_key)) + tox_err_friend_by_public_key = tox_err_friend_by_public_key.value + if tox_err_friend_by_public_key == TOX_ERR_FRIEND_BY_PUBLIC_KEY['OK']: + return int(result) + if tox_err_friend_by_public_key == TOX_ERR_FRIEND_BY_PUBLIC_KEY['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if tox_err_friend_by_public_key == TOX_ERR_FRIEND_BY_PUBLIC_KEY['NOT_FOUND']: + raise ArgumentError('No friend with the given Public Key exists on the friend list.') + raise ToxError('The function did not return OK for the friend by public key.') + + def friend_exists(self, friend_number: int) -> bool: + """ + Checks if a friend with the given friend number exists and returns true if it does. + """ + assert type(friend_number) == int + # bool() -> TypeError: 'str' object cannot be interpreted as an integer + return bool(Tox.libtoxcore.tox_friend_exists(self._tox_pointer, c_uint32(friend_number))) + + def self_get_friend_list_size(self) -> int: + """ + Return the number of friends on the friend list. + + This function can be used to determine how much memory to allocate for tox_self_get_friend_list. + + :return: number of friends + """ + return Tox.libtoxcore.tox_self_get_friend_list_size(self._tox_pointer) + + def self_get_friend_list(self, friend_list: Union[list[int],None]=None) -> list: + """ + Copy a list of valid friend numbers into an array. + + Call tox_self_get_friend_list_size to determine the number of elements to allocate. + + :param friend_list: pointer (c_char_p) to a memory region with enough space to hold the friend list. If this + parameter is None, this function allocates memory for the friend list. + :return: friend list + """ + friend_list_size = self.self_get_friend_list_size() + if friend_list is None: + friend_list = create_string_buffer(sizeof(c_uint32) * friend_list_size) + friend_list = POINTER(c_uint32)(friend_list) + else: + isinstance(friend_list_size, Array), type(friend_list_size) + LOG_TRACE(f"tox_self_get_friend_list") + Tox.libtoxcore.tox_self_get_friend_list(self._tox_pointer, friend_list) + return friend_list[0:friend_list_size] + + def friend_get_public_key(self, friend_number: int, public_key: Union[Array,None] = None) -> str: + """ + Copies the Public Key associated with a given friend number to a byte array. + + :param friend_number: The friend number you want the Public Key of. + :param public_key: pointer (c_char_p) to a memory region of at least TOX_PUBLIC_KEY_SIZE bytes. If this + parameter is None, this function allocates memory for Tox Public Key. + :return: Tox Public Key + """ + if public_key is None: + public_key = create_string_buffer(TOX_PUBLIC_KEY_SIZE) + else: + isinstance(public_key, Array), type(public_key) + tox_err_friend_get_public_key = c_int() + LOG_TRACE(f"tox_friend_get_public_key") + Tox.libtoxcore.tox_friend_get_public_key(self._tox_pointer, + c_uint32(friend_number), + public_key, + byref(tox_err_friend_get_public_key)) + tox_err_friend_get_public_key = tox_err_friend_get_public_key.value + if tox_err_friend_get_public_key == TOX_ERR_FRIEND_GET_PUBLIC_KEY['OK']: + return bin_to_string(public_key, TOX_PUBLIC_KEY_SIZE) + elif tox_err_friend_get_public_key == TOX_ERR_FRIEND_GET_PUBLIC_KEY['FRIEND_NOT_FOUND']: + raise ArgumentError('No friend with the given number exists on the friend list.') + raise ToxError('The function did not return OK') + + def friend_get_last_online(self, friend_number: int) -> int: + """ + Return a unix-time timestamp of the last time the friend associated with a given friend number was seen online. + This function will return UINT64_MAX on error. + + :param friend_number: The friend number you want to query. + :return: unix-time timestamp + """ + tox_err_last_online = c_int() + LOG_DEBUG(f"tox.friend_get_last_online") + result = Tox.libtoxcore.tox_friend_get_last_online(self._tox_pointer, + c_uint32(friend_number), + byref(tox_err_last_online)) + tox_err_last_online = tox_err_last_online.value + if tox_err_last_online == TOX_ERR_FRIEND_GET_LAST_ONLINE['OK']: + return int(result) + elif tox_err_last_online == TOX_ERR_FRIEND_GET_LAST_ONLINE['FRIEND_NOT_FOUND']: + raise ArgumentError('No friend with the given number exists on the friend list.') + raise ToxError('The function did not return OK') + + # Friend-specific state queries (can also be received through callbacks) + + def friend_get_name_size(self, friend_number: int) -> int: + """ + Return the length of the friend's name. If the friend number is invalid, the return value is unspecified. + + The return value is equal to the `length` argument received by the last `friend_name` callback. + """ + tox_err_friend_query = c_int() + LOG_TRACE(f"tox_friend_get_name_size") + result = Tox.libtoxcore.tox_friend_get_name_size(self._tox_pointer, + c_uint32(friend_number), + byref(tox_err_friend_query)) + tox_err_friend_query = tox_err_friend_query.value + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['OK']: + return int(result) + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['NULL']: + raise ArgumentError('The pointer parameter for storing the query result (name, message) was NULL. Unlike' + ' the `_self_` variants of these functions, which have no effect when a parameter is' + ' NULL, these functions return an error in that case.') + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number did not designate a valid friend.') + raise ToxError('The function did not return OK') + + def friend_get_name(self, friend_number: int, name=None) -> str: + """Write the name of the friend designated by the given friend number to a byte array. + + Call tox_friend_get_name_size to determine the allocation size + for the `name` parameter. + + The data written to `name` is equal to the data received by the + last `friend_name` callback. + + :param friend_number: number of friend + :param name: pointer (c_char_p) to a valid memory region large enough to store the friend's name. + :return: name of the friend + """ + if name is None: + name = create_string_buffer(self.friend_get_name_size(friend_number)) + else: + isinstance(name, Array), type(name) + tox_err_friend_query = c_int() + LOG_DEBUG(f"tox.friend_get_name") + Tox.libtoxcore.tox_friend_get_name(self._tox_pointer, + c_uint32(friend_number), + name, + byref(tox_err_friend_query)) + tox_err_friend_query = tox_err_friend_query.value + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['OK']: + return str(name.value, 'utf-8', errors='ignore') + elif tox_err_friend_query == TOX_ERR_FRIEND_QUERY['NULL']: + raise ArgumentError('The pointer parameter for storing the query result (name, message) was NULL. Unlike' + ' the `_self_` variants of these functions, which have no effect when a parameter is' + ' NULL, these functions return an error in that case.') + elif tox_err_friend_query == TOX_ERR_FRIEND_QUERY['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number did not designate a valid friend.') + raise ToxError('The function did not return OK') + + def callback_friend_name(self, callback: Union[Callable,None]) -> None: + """ + Set the callback for the `friend_name` event. Pass None to unset. + + This event is triggered when a friend changes their name. + + :param callback: Python function. Should take pointer (c_void_p) to Tox object, + The friend number (c_uint32) of the friend whose name changed, + A byte array (c_char_p) containing the same data as tox_friend_get_name would write to its `name` parameter, + A value (c_size_t) equal to the return value of tox_friend_get_name_size, + pointer (c_void_p) to user_data + """ + LOG_DEBUG(f"tox.callback_friend_name") + if callback is None: + Tox.libtoxcore.tox_callback_friend_name(self._tox_pointer, + POINTER(None)()) + self.friend_name_cb = None + return + + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_char_p, c_size_t, c_void_p) + self.friend_name_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_friend_name") + Tox.libtoxcore.tox_callback_friend_name(self._tox_pointer, self.friend_name_cb) + + def friend_get_status_message_size(self, friend_number: int) -> int: + """ + Return the length of the friend's status message. If the friend number is invalid, the return value is SIZE_MAX. + + :return: length of the friend's status message + """ + tox_err_friend_query = c_int() + LOG_TRACE(f"tox_friend_get_status_message_size") + result = Tox.libtoxcore.tox_friend_get_status_message_size(self._tox_pointer, + c_uint32(friend_number), + byref(tox_err_friend_query)) + tox_err_friend_query = tox_err_friend_query.value + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['OK']: + return int(result) + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['NULL']: + raise ArgumentError('The pointer parameter for storing the query result (name, message) was NULL. Unlike' + ' the `_self_` variants of these functions, which have no effect when a parameter is' + ' NULL, these functions return an error in that case.') + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number did not designate a valid friend.') + raise ToxError('The function did not return OK') + + def friend_get_status_message(self, friend_number: int, status_message=None) -> str: + """Write the status message of the friend designated by the given friend number to a byte array. + + Call tox_friend_get_status_message_size to determine the + allocation size for the `status_name` parameter. + + The data written to `status_message` is equal to the data + received by the last `friend_status_message` callback. + + :param friend_number: + :param status_message: pointer (c_char_p) to a valid memory region large enough to store the friend's status + message. + :return: status message of the friend + """ + if status_message is None: + status_message = create_string_buffer(self.friend_get_status_message_size(friend_number)) + else: + isinstance(status_message, Array), type(status_message) + tox_err_friend_query = c_int() + LOG_DEBUG(f"tox.friend_get_status_message") + Tox.libtoxcore.tox_friend_get_status_message(self._tox_pointer, + c_uint32(friend_number), + status_message, + byref(tox_err_friend_query)) + tox_err_friend_query = tox_err_friend_query.value + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['OK']: + # 'utf-8' codec can't decode byte 0xb7 in position 2: invalid start byte + return str(status_message.value, 'utf-8', errors='ignore') + elif tox_err_friend_query == TOX_ERR_FRIEND_QUERY['NULL']: + raise ArgumentError('The pointer parameter for storing the query result (name, message) was NULL. Unlike' + ' the `_self_` variants of these functions, which have no effect when a parameter is' + ' NULL, these functions return an error in that case.') + elif tox_err_friend_query == TOX_ERR_FRIEND_QUERY['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number did not designate a valid friend.') + raise ToxError('The function did not return OK') + + def callback_friend_status_message(self, callback: Union[Callable,None]) -> None: + """ + Set the callback for the `friend_status_message` event. Pass NULL to unset. + + This event is triggered when a friend changes their status message. + + :param callback: Python function. Should take pointer (c_void_p) to Tox object, + The friend number (c_uint32) of the friend whose status message changed, + A byte array (c_char_p) containing the same data as tox_friend_get_status_message would write to its + `status_message` parameter, + A value (c_size_t) equal to the return value of tox_friend_get_status_message_size, + pointer (c_void_p) to user_data + """ + LOG_DEBUG(f"tox.callback_friend_status_message") + if callback is None: + Tox.libtoxcore.tox_callback_friend_status_message(self._tox_pointer, + POINTER(None)()) + self.friend_status_message_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_char_p, c_size_t, c_void_p) + self.friend_status_message_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_friend_status_message") + Tox.libtoxcore.tox_callback_friend_status_message(self._tox_pointer, + self.friend_status_message_cb) + + def friend_get_status(self, friend_number: int) -> int: + """ + Return the friend's user status (away/busy/...). If the friend number is invalid, the return value is + unspecified. + + The status returned is equal to the last status received through the `friend_status` callback. + + :return: TOX_USER_STATUS + """ + tox_err_friend_query = c_int() + LOG_DEBUG(f"tox.friend_get_status") + result = Tox.libtoxcore.tox_friend_get_status(self._tox_pointer, + c_uint32(friend_number), + byref(tox_err_friend_query)) + tox_err_friend_query = tox_err_friend_query.value + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['OK']: + return int(result) + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['NULL']: + raise ArgumentError('The pointer parameter for storing the query result (name, message) was NULL. Unlike' + ' the `_self_` variants of these functions, which have no effect when a parameter is' + ' NULL, these functions return an error in that case.') + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number did not designate a valid friend.') + raise ToxError('The function did not return OK.') + + def callback_friend_status(self, callback: Union[Callable,None]) -> None: + """ + Set the callback for the `friend_status` event. Pass None to unset. + + This event is triggered when a friend changes their user status. + + :param callback: Python function. Should take pointer (c_void_p) to Tox object, + :param The friend number (c_uint32) of the friend whose user status changed, + :param The new user status (TOX_USER_STATUS), + :param user_data: pointer (c_void_p) to user data + """ + LOG_DEBUG(f"tox.callback_friend_status") + if callback is None: + Tox.libtoxcore.tox_callback_friend_status(self._tox_pointer, + POINTER(None)()) + self.friend_status_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_int, c_void_p) + self.friend_status_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_friend_status") + Tox.libtoxcore.tox_callback_friend_status(self._tox_pointer, self.friend_status_cb) + return None + + def friend_get_connection_status(self, friend_number: int) -> int: + """ + Check whether a friend is currently connected to this client. + + The result of this function is equal to the last value received by the `friend_connection_status` callback. + + :param friend_number: The friend number for which to query the connection status. + :return: the friend's connection status (TOX_CONNECTION) as it was received through the + `friend_connection_status` event. + """ + tox_err_friend_query = c_int() + LOG_DEBUG(f"tox.friend_get_connection_status") + result = Tox.libtoxcore.tox_friend_get_connection_status(self._tox_pointer, + c_uint32(friend_number), + byref(tox_err_friend_query)) + tox_err_friend_query = tox_err_friend_query.value + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['OK']: + return int(result) + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['NULL']: + raise ArgumentError('The pointer parameter for storing the query result (name, message) was NULL. Unlike' + ' the `_self_` variants of these functions, which have no effect when a parameter is' + ' NULL, these functions return an error in that case.') + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number did not designate a valid friend.') + raise ToxError('The function did not return OK for friend get connection status.') + + def callback_friend_connection_status(self, callback: Union[Callable,None]) -> None: + """Set the callback for the `friend_connection_status` event. Pass NULL to unset. + + This event is triggered when a friend goes offline after having been online, or when a friend goes online. + + This callback is not called when adding friends. It is assumed + that when adding friends, their connection status is initially + offline. + + :param callback: Python function. Should take pointer (c_void_p) to Tox object, + The friend number (c_uint32) of the friend whose connection status changed, + The result of calling tox_friend_get_connection_status (TOX_CONNECTION) on the passed friend_number, + pointer (c_void_p) to user_data + """ + LOG_DEBUG(f"tox.callback_friend_connection_status") + if callback is None: + Tox.libtoxcore.tox_callback_friend_connection_status(self._tox_pointer, + POINTER(None)()) + self.friend_connection_status_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_int, c_void_p) + self.friend_connection_status_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_friend_connection_status") + Tox.libtoxcore.tox_callback_friend_connection_status(self._tox_pointer, + self.friend_connection_status_cb) + return None + + def friend_get_typing(self, friend_number: int) -> bool: + """ + Check whether a friend is currently typing a message. + + :param friend_number: The friend number for which to query the typing status. + :return: true if the friend is typing. + """ + tox_err_friend_query = c_int() + LOG_DEBUG(f"tox.friend_get_typing") + result = Tox.libtoxcore.tox_friend_get_typing(self._tox_pointer, + c_uint32(friend_number), + byref(tox_err_friend_query)) + tox_err_friend_query = tox_err_friend_query.value + if tox_err_friend_query == TOX_ERR_FRIEND_QUERY['OK']: + return bool(result) + elif tox_err_friend_query == TOX_ERR_FRIEND_QUERY['NULL']: + raise ArgumentError('The pointer parameter for storing the query result (name, message) was NULL. Unlike' + ' the `_self_` variants of these functions, which have no effect when a parameter is' + ' NULL, these functions return an error in that case.') + elif tox_err_friend_query == TOX_ERR_FRIEND_QUERY['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number did not designate a valid friend.') + raise ToxError('The function did not return OK') + + def callback_friend_typing(self, callback: Union[Callable,None]) -> None: + """ + Set the callback for the `friend_typing` event. Pass NULL to unset. + + This event is triggered when a friend starts or stops typing. + + :param callback: Python function. Should take pointer (c_void_p) to Tox object, + The friend number (c_uint32) of the friend who started or stopped typing, + The result of calling tox_friend_get_typing (c_bool) on the passed friend_number, + pointer (c_void_p) to user_data + """ + LOG_DEBUG(f"tox.callback_friend_typing") + if callback is None: + Tox.libtoxcore.tox_callback_friend_typing(self._tox_pointer, + POINTER(None)()) + self.friend_typing_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_bool, c_void_p) + self.friend_typing_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_friend_typing") + Tox.libtoxcore.tox_callback_friend_typing(self._tox_pointer, self.friend_typing_cb) + + # Sending private messages + + def self_set_typing(self, friend_number: int, typing: bool) -> bool: + """ + Set the client's typing status for a friend. + + The client is responsible for turning it on or off. + + :param friend_number: The friend to which the client is typing a message. + :param typing: The typing status. True means the client is typing. + :return: True on success. + """ + tox_err_set_typing = c_int() + LOG_DEBUG(f"tox.self_set_typing") + result = Tox.libtoxcore.tox_self_set_typing(self._tox_pointer, c_uint32(friend_number), + c_bool(typing), byref(tox_err_set_typing)) + tox_err_set_typing = tox_err_set_typing.value + if tox_err_set_typing == TOX_ERR_SET_TYPING['OK']: + return bool(result) + if tox_err_set_typing == TOX_ERR_SET_TYPING['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend number did not designate a valid friend.') + raise ToxError('The function did not return OK for set typing.') + + def friend_send_message(self, friend_number: int, message_type: int, message: Union[str,bytes]) -> int: + """Send a text chat message to an online friend. + + This function creates a chat message packet and pushes it into the send queue. + + The message length may not exceed + TOX_MAX_MESSAGE_LENGTH. Larger messages must be split by the + client and sent as separate messages. Other clients can then + reassemble the fragments. Messages may not be empty. + + The return value of this function is the message ID. If a read + receipt is received, the triggered `friend_read_receipt` event + will be passed this message ID. + + Message IDs are unique per friend. The first message ID is 0. + Message IDs are incremented by 1 each time a message is sent. + If UINT32_MAX messages were sent, the next message ID is 0. + + :param friend_number: The friend number of the friend to send the message to. + :param message_type: Message type (TOX_MESSAGE_TYPE). + :param message: A non-None message text. + :return: message ID + + """ + if message and type(message) == str: + message = bytes(message, 'utf-8') + tox_err_friend_send_message = c_int() + LOG_DEBUG(f"tox.friend_send_message") + result = Tox.libtoxcore.tox_friend_send_message(self._tox_pointer, + c_uint32(friend_number), + c_int(message_type), + message, # sicNO + c_size_t(len(message)), + byref(tox_err_friend_send_message)) + tox_err_friend_send_message = tox_err_friend_send_message.value + if tox_err_friend_send_message == TOX_ERR_FRIEND_SEND_MESSAGE['OK']: + return int(result) + if tox_err_friend_send_message == TOX_ERR_FRIEND_SEND_MESSAGE['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if tox_err_friend_send_message == TOX_ERR_FRIEND_SEND_MESSAGE['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend number did not designate a valid friend.') + if tox_err_friend_send_message == TOX_ERR_FRIEND_SEND_MESSAGE['FRIEND_NOT_CONNECTED']: + raise ArgumentError('This client is currently not connected to the friend.') + if tox_err_friend_send_message == TOX_ERR_FRIEND_SEND_MESSAGE['SENDQ']: + raise MemoryError('An allocation error occurred while increasing the send queue size.') + if tox_err_friend_send_message == TOX_ERR_FRIEND_SEND_MESSAGE['TOO_LONG']: + raise ArgumentError('Message length exceeded TOX_MAX_MESSAGE_LENGTH.') + if tox_err_friend_send_message == TOX_ERR_FRIEND_SEND_MESSAGE['EMPTY']: + raise ArgumentError('Attempted to send a zero-length message.') + raise ToxError('The function did not return OK for friend send message.') + + def callback_friend_read_receipt(self, callback: Union[Callable,None]) -> None: + """ + Set the callback for the `friend_read_receipt` event. Pass None to unset. + + This event is triggered when the friend receives the message sent with tox_friend_send_message with the + corresponding message ID. + + :param callback: Python function. Should take pointer (c_void_p) to Tox object, + The friend number (c_uint32) of the friend who received the message, + The message ID (c_uint32) as returned from tox_friend_send_message corresponding to the message sent, + pointer (c_void_p) to user_data + :param user_data: pointer (c_void_p) to user data + """ + LOG_DEBUG(f"tox.callback_friend_read_receipt") + if callback is None: + Tox.libtoxcore.tox_callback_friend_read_receipt(self._tox_pointer, + POINTER(None)()) + self.friend_read_receipt_cb = None + return + + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_void_p) + self.friend_read_receipt_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_friend_read_receipt") + Tox.libtoxcore.tox_callback_friend_read_receipt(self._tox_pointer, + self.friend_read_receipt_cb) + + # Receiving private messages and friend requests + + def callback_friend_request(self, callback: Union[Callable,None]) -> None: + """ + Set the callback for the `friend_request` event. Pass None to unset. + + This event is triggered when a friend request is received. + + :param callback: Python function. Should take + pointer (c_void_p) to Tox object, + The Public Key (c_uint8 array) of the user who sent the friend request, + The message (c_char_p) they sent along with the request, + The size (c_size_t) of the message byte array, + pointer (c_void_p) to user_data + :param user_data: pointer (c_void_p) to user data + """ + if callback is None: + Tox.libtoxcore.tox_callback_friend_request(self._tox_pointer, + POINTER(None)()) + self.friend_request_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, POINTER(c_uint8), c_char_p, c_size_t, c_void_p) + self.friend_request_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_friend_request") + Tox.libtoxcore.tox_callback_friend_request(self._tox_pointer, self.friend_request_cb) + + def callback_friend_message(self, callback: Union[Callable,None]) -> None: + """ + Set the callback for the `friend_message` event. Pass None to unset. + + This event is triggered when a message from a friend is received. + + :param callback: Python function. Should take + pointer (c_void_p) to Tox object, + The friend number (c_uint32) of the friend who sent the message, + Message type (TOX_MESSAGE_TYPE), + The message data (c_char_p) they sent, + The size (c_size_t) of the message byte array. + pointer (c_void_p) to user_data + """ + LOG_DEBUG(f"tox.callback_friend_message") + if callback is None: + Tox.libtoxcore.tox_callback_friend_message(self._tox_pointer, + POINTER(None)()) + self.friend_message_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_int, c_char_p, c_size_t, c_void_p) + self.friend_message_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_friend_message") + Tox.libtoxcore.tox_callback_friend_message(self._tox_pointer, self.friend_message_cb) + + # File transmission: common between sending and receiving + + @staticmethod + def hash(data: Union[str,bytes], hash=None) -> str: + """Generates a cryptographic hash of the given data. + + This function may be used by clients for any purpose, but is + provided primarily for validating cached avatars. This use is + highly recommended to avoid unnecessary avatar updates. + + If hash is NULL or data is NULL while length is not 0 the function returns false, otherwise it returns true. + + This function is a tox_wrapper to internal message-digest functions. + + :param hash: A valid memory location the hash data. It must be at least TOX_HASH_LENGTH bytes in size. + :param data: Data to be hashed or NULL. +#? :return: true if hash was not NULL. + :return: the hash as a string. + """ + if hash is None: + hash = create_string_buffer(TOX_HASH_LENGTH) + else: + assert isinstance(hash, Array), f"{type(hash)}" + if type(data) == str: + data = bytes(data, 'utf-8') # f"{type(data)} != bytes" + Tox.libtoxcore.tox_hash(hash, c_char_p(data), c_size_t(len(data))) + return bin_to_string(hash, TOX_HASH_LENGTH) + + def file_control(self, friend_number: int, file_number: int, control: int) -> bool: + """ + Sends a file control command to a friend for a given file transfer. + + :param friend_number: The friend number of the friend the file is being transferred to or received from. + :param file_number: The friend-specific identifier for the file transfer. + :param control: The control (TOX_FILE_CONTROL) command to send. + :return: True on success. + """ + tox_err_file_control = c_int() + LOG_DEBUG(f"tox.file_control") + result = Tox.libtoxcore.tox_file_control(self._tox_pointer, + c_uint32(friend_number), + c_uint32(file_number), + c_int(control), byref(tox_err_file_control)) + tox_err_file_control = tox_err_file_control.value + if tox_err_file_control == TOX_ERR_FILE_CONTROL['OK']: + return bool(result) + if tox_err_file_control == TOX_ERR_FILE_CONTROL['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number passed did not designate a valid friend.') + if tox_err_file_control == TOX_ERR_FILE_CONTROL['FRIEND_NOT_CONNECTED']: + raise ArgumentError('This client is currently not connected to the friend.') + if tox_err_file_control == TOX_ERR_FILE_CONTROL['NOT_FOUND']: + raise ArgumentError('No file transfer with the given file number was found for the given friend.') + if tox_err_file_control == TOX_ERR_FILE_CONTROL['NOT_PAUSED']: + raise ToxError('A RESUME control was sent, but the file transfer is running normally.') + if tox_err_file_control == TOX_ERR_FILE_CONTROL['DENIED']: + raise ToxError('A RESUME control was sent, but the file transfer was paused by the other party. Only ' + 'the party that paused the transfer can resume it.') + if tox_err_file_control == TOX_ERR_FILE_CONTROL['ALREADY_PAUSED']: + raise ToxError('A PAUSE control was sent, but the file transfer was already paused.') + if tox_err_file_control == TOX_ERR_FILE_CONTROL['SENDQ']: + raise ToxError('Packet queue is full.') + raise ToxError('The function did not return OK for file control.') + + def callback_file_recv_control(self, callback: Union[Callable,None]) -> None: + """Set the callback for the `file_recv_control` event. Pass NULL to unset. + + This event is triggered when a file control command is received + from a friend. + + :param callback: Python function. + When receiving TOX_FILE_CONTROL_CANCEL, the client should + release the resources associated with the file number and + consider the transfer failed. + + Should take pointer (c_void_p) to Tox object, + The friend number (c_uint32) of the friend who is sending the file. + The friend-specific file number (c_uint32) the data received is associated with. + The file control (TOX_FILE_CONTROL) command received. + pointer (c_void_p) to user_data + """ + if callback is None: + Tox.libtoxcore.tox_callback_file_recv_control(self._tox_pointer, + POINTER(None)()) + self.file_recv_control_cb = None + return + + LOG_DEBUG(f"tox.callback_file_recv_control") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_int, c_void_p) + self.file_recv_control_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_file_recv_control") + Tox.libtoxcore.tox_callback_file_recv_control(self._tox_pointer, + self.file_recv_control_cb) + + def file_seek(self, friend_number: int, file_number: int, position: int) -> bool: + """ + Sends a file seek control command to a friend for a given file transfer. + + This function can only be called to resume a file transfer right before TOX_FILE_CONTROL_RESUME is sent. + + :param friend_number: The friend number of the friend the file is being received from. + :param file_number: The friend-specific identifier for the file transfer. + :param position: The position that the file should be seeked to. + :return: True on success. + """ + tox_err_file_seek = c_int() + LOG_DEBUG(f"tox.file_control") + result = Tox.libtoxcore.tox_file_control(self._tox_pointer, + c_uint32(friend_number), + c_uint32(file_number), + c_uint64(position), + byref(tox_err_file_seek)) + tox_err_file_seek = tox_err_file_seek.value + if tox_err_file_seek == TOX_ERR_FILE_SEEK['OK']: + return bool(result) + if tox_err_file_seek == TOX_ERR_FILE_SEEK['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number passed did not designate a valid friend.') + if tox_err_file_seek == TOX_ERR_FILE_SEEK['FRIEND_NOT_CONNECTED']: + raise ArgumentError('This client is currently not connected to the friend.') + if tox_err_file_seek == TOX_ERR_FILE_SEEK['NOT_FOUND']: + raise ArgumentError('No file transfer with the given file number was found for the given friend.') + if tox_err_file_seek == TOX_ERR_FILE_SEEK['SEEK_DENIED']: + raise IOError('File was not in a state where it could be seeked.') + if tox_err_file_seek == TOX_ERR_FILE_SEEK['INVALID_POSITION']: + raise ArgumentError('Seek position was invalid') + if tox_err_file_seek == TOX_ERR_FILE_SEEK['SENDQ']: + raise ToxError('Packet queue is full.') + raise ToxError('The function did not return OK') + + def file_get_file_id(self, friend_number: int, file_number: int, file_id=None) -> str: + """ + Copy the file id associated to the file transfer to a byte array. + + :param friend_number: The friend number of the friend the file is being transferred to or received from. + :param file_number: The friend-specific identifier for the file transfer. + :param file_id: A pointer (c_char_p) to memory region of at least TOX_FILE_ID_LENGTH bytes. If this parameter is + None, this function has no effect. + :return: file id. + """ + if file_id is None: + file_id = create_string_buffer(TOX_FILE_ID_LENGTH) + else: + isinstance(file_id, Array), type(file_id) + tox_err_file_get = c_int() + LOG_DEBUG(f"tox.file_get_file_id") + Tox.libtoxcore.tox_file_get_file_id(self._tox_pointer, + c_uint32(friend_number), + c_uint32(file_number), + file_id, + byref(tox_err_file_get)) + error = tox_err_file_get + if error.value == TOX_ERR_FILE_GET['OK']: + return bin_to_string(file_id, TOX_FILE_ID_LENGTH) + s = sGetError(error.value, TOX_ERR_FILE_GET) + LOG_ERROR(f"group_new err={error.value} {s}") + # have seen ArgumentError: group_new 3 NOT_FOUND + raise ArgumentError(f"group_new err={error.value} {s}") + + # File transmission: sending + + def file_send(self, friend_number: int, kind: int, file_size: int, file_id: int, filename: Union[bytes,str]) -> int: + """Send a file transmission request. + + Maximum filename length is TOX_MAX_FILENAME_LENGTH bytes. The + filename should generally just be a file name, not a path with + directory names. + + If a non-UINT64_MAX file size is provided, it can be used by + both sides to determine the sending progress. File size can be + set to UINT64_MAX for streaming data of unknown size. + + File transmission occurs in chunks, which are requested + through the `file_chunk_request` event. + + When a friend goes offline, all file transfers associated with + the friend are purged from core. + + If the file contents change during a transfer, the behaviour + is unspecified in general. What will actually happen depends + on the mode in which the file was modified and how the client + determines the file size. + + - If the file size was increased + - and sending mode was streaming (file_size = UINT64_MAX), the behaviour will be as expected. + - and sending mode was file (file_size != UINT64_MAX), the file_chunk_request callback will receive length = + 0 when Core thinks the file transfer has finished. If the client remembers the file size as it was when + sending the request, it will terminate the transfer normally. If the client re-reads the size, it will think + the friend cancelled the transfer. + - If the file size was decreased + - and sending mode was streaming, the behaviour is as expected. + - and sending mode was file, the callback will return 0 at the new (earlier) end-of-file, signalling to the + friend that the transfer was cancelled. + - If the file contents were modified + - at a position before the current read, the two files (local and remote) will differ after the transfer + terminates. + - at a position after the current read, the file transfer will succeed as expected. + - In either case, both sides will regard the transfer as complete and successful. + + :param friend_number: The friend number of the friend the file send request should be sent to. + :param kind: The meaning of the file to be sent. + :param file_size: Size in bytes of the file the client wants to send, UINT64_MAX if unknown or streaming. + :param file_id: A file identifier of length TOX_FILE_ID_LENGTH that can be used to uniquely identify file + transfers across core restarts. If NULL, a random one will be generated by core. It can then be obtained by + using tox_file_get_file_id(). + :param filename: Name of the file. Does not need to be the actual name. This name will be sent along with the + file send request. + :return: A file number used as an identifier in subsequent callbacks. This number is per friend. File numbers + are reused after a transfer terminates. On failure, this function returns UINT32_MAX. Any pattern in file + numbers should not be relied on. + """ + LOG_DEBUG(f"tox.file_send") + tox_err_file_send = c_int() + if type(filename) == str: + filename = bytes(filename, 'utf-8') + result = self.libtoxcore.tox_file_send(self._tox_pointer, + c_uint32(friend_number), + c_uint32(kind), + c_uint64(file_size), + string_to_bin_charp(file_id), + filename, # sic + c_size_t(len(filename)), + byref(tox_err_file_send)) + err_file = tox_err_file_send.value + if err_file == TOX_ERR_FILE_SEND['OK']: + # UINT32_MAX + return int(result) + if err_file == TOX_ERR_FILE_SEND['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if err_file == TOX_ERR_FILE_SEND['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number passed did not designate a valid friend.') + if err_file == TOX_ERR_FILE_SEND['FRIEND_NOT_CONNECTED']: + raise ArgumentError('This client is currently not connected to the friend.') + if err_file == TOX_ERR_FILE_SEND['NAME_TOO_LONG']: + raise ArgumentError('Filename length exceeded TOX_MAX_FILENAME_LENGTH bytes.') + if err_file == TOX_ERR_FILE_SEND['TOO_MANY']: + raise ToxError('Too many ongoing transfers. The maximum number of concurrent file transfers is 256 per' + 'friend per direction (sending and receiving).') + raise ToxError('The function did not return OK') + + def file_send_chunk(self, friend_number: int, file_number: int, position, data: Union[Array,bytes]) -> int: + """ + Send a chunk of file data to a friend. + + This function is called in response to the `file_chunk_request` callback. The length parameter should be equal + to the one received though the callback. If it is zero, the transfer is assumed complete. For files with known + size, Core will know that the transfer is complete after the last byte has been received, so it is not necessary + (though not harmful) to send a zero-length chunk to terminate. For streams, core will know that the transfer is + finished if a chunk with length less than the length requested in the callback is sent. + + :param friend_number: The friend number of the receiving friend for this file. + :param file_number: The file transfer identifier returned by tox_file_send. + :param position: The file or stream position from which to continue reading. + :param data: Chunk of file data + :return: true on success. + """ + LOG_DEBUG(f"tox.file_send_chunk") + tox_err_file_send_chunk = c_int() + + isinstance(data, Array) or isinstance(data, bytes), "file_send_chunk type(data)" + + result = self.libtoxcore.tox_file_send_chunk(self._tox_pointer, + c_uint32(friend_number), c_uint32(file_number), + c_uint64(position), + data, # sic + c_size_t(len(data)), + byref(tox_err_file_send_chunk)) + tox_err_file_send_chunk = tox_err_file_send_chunk.value + if tox_err_file_send_chunk == TOX_ERR_FILE_SEND_CHUNK['OK']: + return bool(result) + if tox_err_file_send_chunk == TOX_ERR_FILE_SEND_CHUNK['NULL']: + raise ArgumentError('The length parameter was non-zero, but data was NULL.') + if tox_err_file_send_chunk == TOX_ERR_FILE_SEND_CHUNK['FRIEND_NOT_FOUND']: + ArgumentError('The friend_number passed did not designate a valid friend.') + elif tox_err_file_send_chunk == TOX_ERR_FILE_SEND_CHUNK['FRIEND_NOT_CONNECTED']: + raise ArgumentError('This client is currently not connected to the friend.') + if tox_err_file_send_chunk == TOX_ERR_FILE_SEND_CHUNK['NOT_FOUND']: + raise ArgumentError('No file transfer with the given file number was found for the given friend.') + if tox_err_file_send_chunk == TOX_ERR_FILE_SEND_CHUNK['NOT_TRANSFERRING']: + raise ArgumentError('File transfer was found but isn\'t in a transferring state: (paused, done, broken, ' + 'etc...) (happens only when not called from the request chunk callback).') + if tox_err_file_send_chunk == TOX_ERR_FILE_SEND_CHUNK['INVALID_LENGTH']: + raise ArgumentError('Attempted to send more or less data than requested. The requested data size is ' + 'adjusted according to maximum transmission unit and the expected end of the file. ' + 'Trying to send less or more than requested will return this error.') + if tox_err_file_send_chunk == TOX_ERR_FILE_SEND_CHUNK['SENDQ']: + raise ToxError('Packet queue is full.') + if tox_err_file_send_chunk == TOX_ERR_FILE_SEND_CHUNK['WRONG_POSITION']: + raise ArgumentError('Position parameter was wrong.') + raise ToxError('The function did not return OK') + + def callback_file_chunk_request(self, callback: Union[Callable,None]) -> None: + """Set the callback for the `file_chunk_request` event. Pass None to unset. + + This event is triggered when Core is ready to send more file data. + + :param callback: Python function. + If the length parameter is 0, the file transfer is finished, and + the client's resources associated with the file number should be + released. After a call with zero length, the file number can be + reused for future file transfers. + + If the requested position is not equal to the client's idea of + the current file or stream position, it will need to seek. In + case of read-once streams, the client should keep the last read + chunk so that a seek back can be supported. A seek-back only + ever needs to read from the last requested chunk. This happens + when a chunk was requested, but the send failed. A seek-back + request can occur an arbitrary number of times for any given + chunk. + + In response to receiving this callback, the client should call + the function `tox_file_send_chunk` with the requested chunk. If + the number of bytes sent through that function is zero, the file + transfer is assumed complete. A client must send the full length + of data requested with this callback. + + Should take pointer (c_void_p) to Tox object, + The friend number (c_uint32) of the receiving friend for this file. + The file transfer identifier (c_uint32) returned by tox_file_send. + The file or stream position (c_uint64) from which to continue reading. + The number of bytes (c_size_t) requested for the current chunk. + pointer (c_void_p) to user_data + """ + if callback is None: + Tox.libtoxcore.tox_callback_file_chunk_request(self._tox_pointer, + POINTER(None)()) + self.file_chunk_request_cb = None + return + LOG_DEBUG(f"tox.callback_file_chunk_request") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_uint64, c_size_t, c_void_p) + self.file_chunk_request_cb = c_callback(callback) + self.libtoxcore.tox_callback_file_chunk_request(self._tox_pointer, self.file_chunk_request_cb) + + # File transmission: receiving + + def callback_file_recv(self, callback: Union[Callable,None]) -> None: + """Set the callback for the `file_recv` event. Pass None to unset. + + This event is triggered when a file transfer request is + received. + + :param callback: Python function. + The client should acquire resources to be associated with the + file transfer. Incoming file transfers start in the PAUSED + state. After this callback returns, a transfer can be rejected + by sending a TOX_FILE_CONTROL_CANCEL control command before any + other control commands. It can be accepted by sending + TOX_FILE_CONTROL_RESUME. + + Should take pointer (c_void_p) to Tox object, + The friend number (c_uint32) of the friend who is sending the file transfer request. + The friend-specific file number (c_uint32) the data received is associated with. + The meaning of the file (c_uint32) to be sent. + Size in bytes (c_uint64) of the file the client wants to send, UINT64_MAX if unknown or streaming. + Name of the file (c_char_p). Does not need to be the actual name. This name will be sent along with the file + send request. + Size in bytes (c_size_t) of the filename. + pointer (c_void_p) to user_data + """ + if callback is None: + Tox.libtoxcore.tox_callback_file_recv(self._tox_pointer, + POINTER(None)()) + self.file_recv_cb = None + return + + LOG_DEBUG(f"tox.callback_file_recv") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_uint32, c_uint64, c_char_p, c_size_t, c_void_p) + self.file_recv_cb = c_callback(callback) + self.libtoxcore.tox_callback_file_recv(self._tox_pointer, self.file_recv_cb) + + def callback_file_recv_chunk(self, callback: Union[Callable,None]) -> None: + """Set the callback for the `file_recv_chunk` event. Pass NULL to unset. + + This event is first triggered when a file transfer request is + received, and subsequently when a chunk of file data for an + accepted request was received. + + :param callback: Python function. + When length is 0, the transfer is finished and the client should + release the resources it acquired for the transfer. After a call + with length = 0, the file number can be reused for new file + transfers. + + If position is equal to file_size (received in the file_receive + callback) when the transfer finishes, the file was received + completely. Otherwise, if file_size was UINT64_MAX, streaming + ended successfully when length is 0. + + Should take pointer (c_void_p) to Tox object, The friend number + (c_uint32) of the friend who is sending the file. The + friend-specific file number (c_uint32) the data received is + associated with. The file position (c_uint64) of the first byte + in data. A byte array (c_char_p) containing the received chunk. + The length (c_size_t) of the received chunk. pointer (c_void_p) + to user_data + """ + if callback is None: + Tox.libtoxcore.tox_callback_file_recv_chunk(self._tox_pointer, + POINTER(None)()) + self.file_recv_chunk_cb = None + return + + LOG_DEBUG(f"tox.callback_file_recv_chunk") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_uint64, POINTER(c_uint8), c_size_t, c_void_p) + self.file_recv_chunk_cb = c_callback(callback) + self.libtoxcore.tox_callback_file_recv_chunk(self._tox_pointer, self.file_recv_chunk_cb) + + # Low-level custom packet sending and receiving + + def friend_send_lossy_packet(self, friend_number: int, data: Union[Array,bytes]) -> bool: + """ + Send a custom lossy packet to a friend. + The first byte of data must be in the range 200-254. Maximum length of a + custom packet is TOX_MAX_CUSTOM_PACKET_SIZE. + + Lossy packets behave like UDP packets, meaning they might never reach the + other side or might arrive more than once (if someone is messing with the + connection) or might arrive in the wrong order. + + Unless latency is an issue, it is recommended that you use lossless custom packets instead. + + :param friend_number: The friend number of the friend this lossy packet + :param data: python string containing the packet data + :return: True on success. + """ + LOG_DEBUG(f"friend_send_lossy_packet") + isinstance(data, bytes) or isinstance(data, Array), f"{type(data)}" + tox_err_friend_custom_packet = c_int() + result = self.libtoxcore.tox_friend_send_lossy_packet(self._tox_pointer, + c_uint32(friend_number), + data, # sic + c_size_t(len(data)), + byref(tox_err_friend_custom_packet)) + tox_err_friend_custom_packet = tox_err_friend_custom_packet.value + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['OK']: + return bool(result) + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend number did not designate a valid friend.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['FRIEND_NOT_CONNECTED']: + raise ArgumentError('This client is currently not connected to the friend.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['INVALID']: + raise ArgumentError('The first byte of data was not in the specified range for the packet type.' + 'This range is 200-254 for lossy, and 160-191 for lossless packets.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['EMPTY']: + raise ArgumentError('Attempted to send an empty packet.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['TOO_LONG']: + raise ArgumentError('Packet data length exceeded TOX_MAX_CUSTOM_PACKET_SIZE.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['SENDQ']: + raise ToxError('Packet queue is full.') + raise ToxError('The function did not return OK') + + def friend_send_lossless_packet(self, friend_number: int, data: Union[Array,bytes]) -> int: + """ + Send a custom lossless packet to a friend. + The first byte of data must be in the range 160-191. Maximum length of a + custom packet is TOX_MAX_CUSTOM_PACKET_SIZE. + + Lossless packet behaviour is comparable to TCP (reliability, arrive in order) + but with packets instead of a stream. + + :param friend_number: The friend number of the friend this lossless packet + :param data: python string containing the packet data + :return: True on success. + """ + LOG_DEBUG(f"friend_send_lossless_packet") + tox_err_friend_custom_packet = c_int() + isinstance(data, bytes) or isinstance(data, Array), f"{type(data)}" + result = self.libtoxcore.tox_friend_send_lossless_packet(self._tox_pointer, + c_uint32(friend_number), + data, # sic + c_size_t(len(data)), + byref(tox_err_friend_custom_packet)) + tox_err_friend_custom_packet = tox_err_friend_custom_packet.value + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['OK']: + return bool(result) + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend number did not designate a valid friend.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['FRIEND_NOT_CONNECTED']: + raise ArgumentError('This client is currently not connected to the friend.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['INVALID']: + raise ArgumentError('The first byte of data was not in the specified range for the packet type.' + 'This range is 200-254 for lossy, and 160-191 for lossless packets.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['EMPTY']: + raise ArgumentError('Attempted to send an empty packet.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['TOO_LONG']: + raise ArgumentError('Packet data length exceeded TOX_MAX_CUSTOM_PACKET_SIZE.') + if tox_err_friend_custom_packet == TOX_ERR_FRIEND_CUSTOM_PACKET['SENDQ']: + raise ToxError('Packet queue is full.') + raise ToxError('The function did not return OK') + + def callback_friend_lossy_packet(self, callback: Union[Callable,None]) -> None: + """ + Set the callback for the `friend_lossy_packet` event. Pass NULL to unset. + + :param callback: Python function. + Should take pointer (c_void_p) to Tox object, + friend_number (c_uint32) - The friend number of the friend who sent a lossy packet, + A byte array (c_uint8 array) containing the received packet data, + length (c_size_t) - The length of the packet data byte array, + pointer (c_void_p) to user_data + """ + if callback is None: + self.libtoxcore.tox_callback_friend_lossy_packet(self._tox_pointer, POINTER(None)()) + self.friend_lossy_packet_cb = None + return + + LOG_DEBUG(f"callback_friend_lossy_packet") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, POINTER(c_uint8), c_size_t, c_void_p) + self.friend_lossy_packet_cb = c_callback(callback) + self.libtoxcore.tox_callback_friend_lossy_packet(self._tox_pointer, self.friend_lossy_packet_cb) + + def callback_friend_lossless_packet(self, callback: Union[Callable,None]) -> None: + """ + Set the callback for the `friend_lossless_packet` event. Pass NULL to unset. + + :param callback: Python function. + Should take pointer (c_void_p) to Tox object, + friend_number (c_uint32) - The friend number of the friend who sent a lossless packet, + A byte array (c_uint8 array) containing the received packet data, + length (c_size_t) - The length of the packet data byte array, + pointer (c_void_p) to user_data + """ + if callback is None: + self.friend_lossless_packet_cb = None + self.libtoxcore.tox_callback_friend_lossless_packet(self._tox_pointer, POINTER(None)()) + return + + LOG_DEBUG(f"callback_friend_lossless_packet") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, POINTER(c_uint8), c_size_t, c_void_p) + self.friend_lossless_packet_cb = c_callback(callback) + self.libtoxcore.tox_callback_friend_lossless_packet(self._tox_pointer, self.friend_lossless_packet_cb) + + # Low-level network information + # def self_get_keys(self): pass + + def self_get_dht_id(self, dht_id=None) -> str: + """Writes the temporary DHT public key of this instance to a byte array. + + This can be used in combination with an externally accessible + IP address and the bound port (from tox_self_get_udp_port) to + run a temporary bootstrap node. + + Be aware that every time a new instance is created, the DHT + public key changes, meaning this cannot be used to run a + permanent bootstrap node. + + :param dht_id: pointer (c_char_p) to a memory region of at least TOX_PUBLIC_KEY_SIZE bytes. If this parameter is + None, this function allocates memory for dht_id. + :return: dht_id + + """ + if dht_id is None: + dht_id = create_string_buffer(TOX_PUBLIC_KEY_SIZE) + else: + isinstance(dht_id, Array), type(dht_id) + LOG_DEBUG(f"tox.self_get_dht_id") + Tox.libtoxcore.tox_self_get_dht_id(self._tox_pointer, dht_id) + return bin_to_string(dht_id, TOX_PUBLIC_KEY_SIZE) + + def self_get_udp_port(self) -> int: + """ + Return the UDP port this Tox instance is bound to. + """ + tox_err_get_port = c_int() + LOG_DEBUG(f"tox.self_get_udp_port") + result = Tox.libtoxcore.tox_self_get_udp_port(self._tox_pointer, byref(tox_err_get_port)) + tox_err_get_port = tox_err_get_port.value + if tox_err_get_port == TOX_ERR_GET_PORT['OK']: + return int(result) + if tox_err_get_port == TOX_ERR_GET_PORT['NOT_BOUND']: + raise ToxError('The instance was not bound to any port.') + raise ToxError('The function did not return OK') + + def self_get_tcp_port(self) -> int: + """ + Return the TCP port this Tox instance is bound to. This is only relevant if the instance is acting as a TCP + relay. + """ + tox_err_get_port = c_int() + LOG_DEBUG(f"tox.self_get_tcp_port") + result = Tox.libtoxcore.tox_self_get_tcp_port(self._tox_pointer, byref(tox_err_get_port)) + tox_err_get_port = tox_err_get_port.value + if tox_err_get_port == TOX_ERR_GET_PORT['OK']: + return int(result) + if tox_err_get_port == TOX_ERR_GET_PORT['NOT_BOUND']: + raise ToxError('The instance was not bound to any port.') + raise ToxError('The function did not return OK') + + # Group chat instance management + + def group_new(self, privacy_state: int, group_name: Union[bytes,str], nick: Union[bytes,str], status: str='') -> int: + """Creates a new group chat. + + This function creates a new group chat object and adds it to the chats array. + + The client should initiate its peer list with self info after + calling this function, as the peer_join callback will not be + triggered. + + :param privacy_state: The privacy state of the group. If this is set to TOX_GROUP_PRIVACY_STATE_PUBLIC, + the group will attempt to announce itself to the DHT and anyone with the Chat ID may join. + Otherwise a friend invite will be required to join the group. + :param group_name: The name of the group. The name must be non-NULL. + + :return group number on success, UINT32_MAX on failure. + + """ + + LOG_DEBUG(f"tox.group_new") + error = c_int() + if type(nick) == str: + nick = bytes(nick, 'utf-8') + if type(group_name) == str: + group_name = bytes(group_name, 'utf-8') + + result = Tox.libtoxcore.tox_group_new(self._tox_pointer, + privacy_state, + group_name, # sic + c_size_t(len(group_name)), + nick, + c_size_t(len(nick)), + byref(error)) + + if error.value: + s = sGetError(error.value, TOX_ERR_GROUP_NEW) + LOG_ERROR(f"group_new err={error.value} {s}") + raise ToxError(f"group_new {s} err={error.value}") + + # TypeError: '<' not supported between instances of 'c_uint' and 'int' + return int(result) + + def group_join(self, chat_id, password: Union[bytes,str], nick: Union[bytes,str], status='') -> int: + """Joins a group chat with specified Chat ID. + + This function creates a new group chat object, adds it to the + chats array, and sends a DHT announcement to find peers in the + group associated with chat_id. Once a peer has been found a + join attempt will be initiated. + + :param chat_id: The Chat ID of the group you wish to join. This must be TOX_GROUP_CHAT_ID_SIZE bytes. + :param password: The password required to join the group. Set to NULL if no password is required. + :param status: FixMe + + :return group_number on success, UINT32_MAX on failure. + """ + + LOG_DEBUG(f"tox.group_join") + assert chat_id, chat_id + assert nick, nick + error = c_int() + if type(nick) == str: + nick = bytes(nick, 'utf-8') + if True: # API change + if not password: + cpassword = None + else: + if password and type(password) == str: + nick = bytes(password, 'utf-8') +#?no cpassword = c_char_p(password) # it's const uint8_t *password + cpassword = password # sic + result = Tox.libtoxcore.tox_group_join(self._tox_pointer, + string_to_bin_charp(chat_id), + nick, # sic + c_size_t(len(nick)), + cpassword, + c_size_t(len(password)) if password else 0, + + byref(error)) + if error.value: + s = sGetError(error.value, TOX_ERR_GROUP_JOIN) + LOG_ERROR(f"group_new err={error.value} {s}") + raise ToxError(f"group_new {s} err={error.value}") + LOG_INFO(f"group_new result={result} chat_id={chat_id}") + + return int(result) + + def group_reconnect(self, group_number) -> bool: + """ + Reconnects to a group. + + This function disconnects from all peers in the group, then attempts to reconnect with the group. + The caller's state is not changed (i.e. name, status, role, chat public key etc.) + + :param group_number: The group number of the group we wish to reconnect to. + :return True on success. + """ + + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + error = c_int() + LOG_DEBUG(f"tox.group_reconnect") + result = Tox.libtoxcore.tox_group_reconnect(self._tox_pointer, + c_uint32(group_number), + byref(error)) + if error.value: + s = sGetError(error.value, TOX_ERR_GROUP_RECONNECT) + LOG_ERROR(f"group_new err={error.value} {s}") + raise ToxError(f"group_new {s} err={error.value}") + return bool(result) + + def group_is_connected(self, group_number) -> bool: + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_is_connected") + result = Tox.libtoxcore.tox_group_is_connected(self._tox_pointer, c_uint32(group_number), byref(error)) + if error.value: + # TOX_ERR_GROUP_IS_CONNECTED_GROUP_NOT_FOUND + s = sGetError(error.value, TOX_ERR_GROUP_IS_CONNECTED) + LOG_ERROR(f"group_new err={error.value} {s}") + raise ToxError("group_is_connected err={error.value} {s}") + return bool(result) + + def group_disconnect(self, group_number: int) -> bool: + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + error = c_int() + LOG_DEBUG(f"tox.group_disconnect") + result = Tox.libtoxcore.tox_group_disconnect(self._tox_pointer, c_uint32(group_number), byref(error)) + if error.value: + s = sGetError(error.value, TOX_ERR_GROUP_DISCONNECT) + LOG_ERROR(f"group_disconnect err={error.value} {s}") + raise ToxError(f"group_disconnect {s} err={error.value}") + return bool(result) + + def group_leave(self, group_number: int, message: Union[str,None]=None) -> bool: + """Leaves a group. + + This function sends a parting packet containing a custom + (non-obligatory) message to all peers in a group, and deletes + the group from the chat array. All group state information is + permanently lost, including keys and role credentials. + + :param group_number: The group number of the group we wish to leave. + :param message: The parting message to be sent to all the peers. Set to NULL if we do not wish to + send a parting message. + + :return True if the group chat instance was successfully deleted. + + """ + + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + LOG_DEBUG(f"tox.leave") + error = c_int() + f = Tox.libtoxcore.tox_group_leave + f.restype = c_bool + if message is not None and type(message) == str: + message = bytes(message, 'utf-8') + result = f(self._tox_pointer, c_uint32(group_number), message, + c_size_t(len(message)) if message else 0, byref(error)) + if error.value: + LOG_ERROR(f"group_leave err={error.value}") + raise ToxError("group_leave err={error.value}") + return bool(result) + + # Group user-visible client information (nickname/status/role/public key) + + def group_self_set_name(self, group_number: int, name) -> bool: + """Set the client's nickname for the group instance designated by the given group number. + + Nickname length cannot exceed TOX_MAX_NAME_LENGTH. If length + is equal to zero or name is a NULL pointer, the function call + will fail. + + :param name: A byte array containing the new nickname. + + :return True on success. + + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + if type(name) == str: + name = bytes(name, 'utf-8') # not c_char_p() + LOG_DEBUG(f"tox.group_self_set_name") + result = Tox.libtoxcore.tox_group_self_set_name(self._tox_pointer, + c_uint32(group_number), + name, c_size_t(len(name)), + byref(error)) + if error.value: + LOG_ERROR(f"group_self_set_name err={error.value}") + raise ToxError("group_self_set_name err={error.value}") + return bool(result) + + def group_self_get_name_size(self, group_number: int) -> int: + """ + Return the length of the client's current nickname for the group instance designated + by group_number as passed to tox_group_self_set_name. + + If no nickname was set before calling this function, the name is empty, + and this function returns 0. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_TRACE(f"tox_group_self_get_name_size") + result = Tox.libtoxcore.tox_group_self_get_name_size(self._tox_pointer, + c_uint32(group_number), + byref(error)) + if error.value: + LOG_ERROR(f"group_self_get_name_size err={error.value}") + raise ToxError("group_self_get_name_size err={error.value}") + return int(result) + + def group_self_get_name(self, group_number: int) -> str: + """Write the nickname set by tox_group_self_set_name to a byte array. + + If no nickname was set before calling this function, the name is empty, + and this function has no effect. + + Call tox_group_self_get_name_size to find out how much memory + to allocate for the result. + + :return nickname + + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + size = self.group_self_get_name_size(group_number) + name = create_string_buffer(size) + LOG_DEBUG(f"tox.group_self_get_name") + result = Tox.libtoxcore.tox_group_self_get_name(self._tox_pointer, + c_uint32(group_number), + name, + byref(error)) + if error.value: + LOG_ERROR(f"group_self_get_name err={error.value}") + raise ToxError("group_self_get_name err={error.value}") + return str(name[:size], 'utf-8', errors='ignore') + + def group_self_set_status(self, group_number: int, status: int) -> bool: + + """ + Set the client's status for the group instance. Status must be a TOX_USER_STATUS. + :return True on success. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_self_set_status") + result = Tox.libtoxcore.tox_group_self_set_status(self._tox_pointer, + c_uint32(group_number), + c_uint32(status), + byref(error)) + if error.value: + LOG_ERROR(f"group_self_set_status err={error.value}") + raise ToxError("group_self_set_status err={error.value}") + return bool(result) + + def group_self_get_status(self, group_number: int) -> int: + """ + returns the client's status for the group instance on success. + return value is unspecified on failure. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_self_get_status") + result = Tox.libtoxcore.tox_group_self_get_status(self._tox_pointer, c_uint32(group_number), byref(error)) + if error.value: + LOG_ERROR(f"group_self_get_status err={error.value}") + raise ToxError("group_self_get_status err={error.value}") + return int(result) + + def group_self_get_role(self, group_number: int) -> int: + """ + returns the client's role for the group instance on success. + return value is unspecified on failure. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_self_get_role") + result = Tox.libtoxcore.tox_group_self_get_role(self._tox_pointer, c_uint32(group_number), byref(error)) + if error.value: + LOG_ERROR(f"group_self_get_role err={error.value}") + raise ToxError(f"group_self_get_role err={error.value}") + return int(result) + + def group_self_get_peer_id(self, group_number: int) -> int: + """ + returns the client's peer id for the group instance on success. + return value is unspecified on failure. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_self_get_peer_id") + result = Tox.libtoxcore.tox_group_self_get_peer_id(self._tox_pointer, c_uint32(group_number), byref(error)) + if error.value: + LOG_ERROR(f"tox.group_self_get_peer_id err={error.value}") + raise ToxError("tox_group_self_get_peer_id err={error.value}") + return int(result) + + def group_self_get_public_key(self, group_number: int) -> str: + """Write the client's group public key designated by the given group number to a byte array. +s + This key will be permanently tied to the client's identity for + this particular group until the client explicitly leaves the + group or gets kicked/banned. This key is the only way for other + peers to reliably identify the client across client restarts. + + `public_key` should have room for at least TOX_GROUP_PEER_PUBLIC_KEY_SIZE bytes. + + :return public key + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + key = create_string_buffer(TOX_GROUP_PEER_PUBLIC_KEY_SIZE) + LOG_DEBUG(f"tox.group_self_get_public_key") + result = Tox.libtoxcore.tox_group_self_get_public_key(self._tox_pointer, + c_uint32(group_number), + key, byref(error)) + if error.value: + LOG_ERROR(f"tox.group_self_get_public_key {TOX_ERR_FRIEND_GET_PUBLIC_KEY[error.value]}") + raise ToxError(f"tox.group_self_get_public_key {TOX_ERR_FRIEND_GET_PUBLIC_KEY[error.value]}") + return bin_to_string(key, TOX_GROUP_PEER_PUBLIC_KEY_SIZE) + + # Peer-specific group state queries. + + def group_peer_get_name_size(self, group_number: int, peer_id: int) -> int: + """ + Return the length of the peer's name. If the group number or ID is invalid, the + return value is unspecified. + + The return value is equal to the `length` argument received by the last + `group_peer_name` callback. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + result = Tox.libtoxcore.tox_group_peer_get_name_size(self._tox_pointer, c_uint32(group_number), c_uint32(peer_id), byref(error)) + if error.value: + LOG_ERROR(f" err={error.value}") + raise ToxError(f" err={error.value}") + LOG_TRACE(f"tox_group_peer_get_name_size") + return int(result) + + def group_peer_get_name(self, group_number: int, peer_id: int) -> str: + """Write the name of the peer designated by the given ID to a byte + array. + + Call tox_group_peer_get_name_size to determine the allocation + size for the `name` parameter. + + The data written to `name` is equal to the data received by the last + `group_peer_name` callback. + + :param group_number: The group number of the group we wish to query. + :param peer_id: The ID of the peer whose name we want to retrieve. + + :return name. + + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + error = c_int() + size = self.group_peer_get_name_size(group_number, peer_id) + name = create_string_buffer(size) + LOG_DEBUG(f"tox.group_peer_get_name") + result = Tox.libtoxcore.tox_group_peer_get_name(self._tox_pointer, + c_uint32(group_number), + c_uint32(peer_id), + name, byref(error)) + if error.value: + LOG_ERROR(f"tox.group_peer_get_name err={error.value}") + raise ToxError(f"tox_group_peer_get_name err={error.value}") + sRet = str(name[:], 'utf-8', errors='ignore') + return sRet + + def group_peer_get_status(self, group_number: int, peer_id: int) -> int: + """ + Return the peer's user status (away/busy/...). If the ID or group number is + invalid, the return value is unspecified. + + The status returned is equal to the last status received through the + `group_peer_status` callback. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 group_number={group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_peer_get_status") + result = Tox.libtoxcore.tox_group_peer_get_status(self._tox_pointer, + c_uint32(group_number), + c_uint32(peer_id), + byref(error)) + if error.value: + # unwrapped + LOG_ERROR(f"tox.group_peer_get_status err={error.value}") + raise ToxError(f"tox.group_peer_get_status err={error.value}") + return int(result) + + def group_peer_get_role(self, group_number: int, peer_id: int) -> int: + """ + Return the peer's role (user/moderator/founder...). If the ID or group number is + invalid, the return value is unspecified. + + The role returned is equal to the last role received through the + `group_moderation` callback. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_peer_get_role") + result = Tox.libtoxcore.tox_group_peer_get_role(self._tox_pointer, + c_uint32(group_number), + c_uint32(peer_id), + byref(error)) + if error.value: + LOG_ERROR(f"tox.group_peer_get_role err={error.value}") + raise ToxError(f"tox.group_peer_get_role err={error.value}") + return int(result) + + def group_peer_get_public_key(self, group_number: int, peer_id: int) -> str: + """Write the group public key with the designated peer_id for the designated group number to public_key. + + This key will be permanently tied to a particular peer until + they explicitly leave the group or get kicked/banned, and is + the only way to reliably identify the same peer across client + restarts. + + `public_key` should have room for at least TOX_GROUP_PEER_PUBLIC_KEY_SIZE bytes. + + :return public key + + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + key = create_string_buffer(TOX_GROUP_PEER_PUBLIC_KEY_SIZE) + LOG_DEBUG(f"tox.group_peer_get_public_key") + result = Tox.libtoxcore.tox_group_peer_get_public_key(self._tox_pointer, + c_uint32(group_number), + c_uint32(peer_id), + key, byref(error)) + if error.value: + LOG_ERROR(f"tox.group_peer_get_public_key err={error.value}") + raise ToxError(f"tox.group_peer_get_public_key err={error.value}") + return bin_to_string(key, TOX_GROUP_PEER_PUBLIC_KEY_SIZE) + + def callback_group_peer_name(self, callback: Union[Callable,None], user_data: Union[bytes,None] = None) -> None: + """ + Set the callback for the `group_peer_name` event. Pass NULL to unset. + This event is triggered when a peer changes their nickname. + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + if callback is None: + Tox.libtoxcore.tox_callback_group_peer_name(self._tox_pointer, + POINTER(None)(), user_data) + self.group_peer_name_cb = None + return + + LOG_DEBUG(f"tox.callback_group_peer_name") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_char_p, c_size_t, c_void_p) + self.group_peer_name_cb = c_callback(callback) + try: + Tox.libtoxcore.tox_callback_group_peer_name(self._tox_pointer, self.group_peer_name_cb) + except Exception as e: # AttributeError + LOG_ERROR(f"tox.callback_conference_peer_name {e}") + + def callback_group_peer_status(self, callback: Union[Callable,None], user_data: Union[bytes,None] = None) -> None: + """ + Set the callback for the `group_peer_status` event. Pass NULL to unset. + This event is triggered when a peer changes their status. + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + + if callback is None: + Tox.libtoxcore.tox_callback_group_peer_status(self._tox_pointer, POINTER(None)()) + self.group_peer_status_cb = None + return + + LOG_DEBUG(f"tox.callback_group_peer_status") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_int, c_void_p) + #* @param group_number The group number of the group we wish to query. + #* @param peer_id The ID of the peer whose status we wish to query. + # *error + self.group_peer_status_cb = c_callback(callback) + try: + Tox.libtoxcore.tox_callback_group_peer_status(self._tox_pointer, self.group_peer_status_cb) + except Exception as e: + LOG_ERROR(f"callback_group_peer_status EXCEPTION {e}") + + + # Group chat state queries and events. + + def group_set_topic(self, group_number: int, topic: str) -> bool: + """Set the group topic and broadcast it to the rest of the group. + + topic length cannot be longer than TOX_GROUP_MAX_TOPIC_LENGTH. + If length is equal to zero or topic is set to NULL, the topic will be unset. + + :return True on success. + + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + if type(topic) == str: + topic = bytes(topic, 'utf-8') # not c_char_p() + try: + LOG_DEBUG(f"tox.group_set_topic") + result = Tox.libtoxcore.tox_group_set_topic(self._tox_pointer, + c_uint32(group_number), + topic, + c_size_t(len(topic)), + byref(error)) + except Exception as e: + LOG_WARN(f"group_set_topic EXCEPTION {e}") + raise + if error.value: + LOG_ERROR(f"group_set_topic err={error.value}") + raise ToxError("group_set_topic err={error.value}") + return bool(result) + + def group_get_topic_size(self, group_number: int) -> int: + """ + Return the length of the group topic. If the group number is invalid, the + return value is unspecified. + + The return value is equal to the `length` argument received by the last + `group_topic` callback. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_TRACE(f"tox_group_get_topic_size") + try: + result = Tox.libtoxcore.tox_group_get_topic_size(self._tox_pointer, + c_uint32(group_number), + byref(error)) + except Exception as e: + LOG_ERROR(f"group_get_topic_size EXCEPTION {e}") + raise + if error.value: + LOG_ERROR(f"tox_group_get_topic_size err={error.value}") + raise ToxError(f"tox_group_get_topic_size err={error.value}") + return int(result) + + def group_get_topic(self, group_number: int) -> str: + """ + Write the topic designated by the given group number to a byte array. + Call tox_group_get_topic_size to determine the allocation size for the `topic` parameter. + The data written to `topic` is equal to the data received by the last + `group_topic` callback. + + :return topic + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + size = self.group_get_topic_size(group_number) + topic = create_string_buffer(size) + LOG_DEBUG(f"tox.group_get_topic") + Tox.libtoxcore.tox_group_get_topic(self._tox_pointer, + c_uint32(group_number), + topic, byref(error)) + if error.value: + LOG_ERROR(f"group_get_topic err={error.value}") + raise ToxError(f"group_get_topic err={error.value}") + return str(topic[:size], 'utf-8', errors='ignore') + + def group_get_name_size(self, group_number: int) -> int: + """ + Return the length of the group name. If the group number is invalid, the + return value is unspecified. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + error = c_int() + result = Tox.libtoxcore.tox_group_get_name_size(self._tox_pointer, + c_uint32(group_number), + byref(error)) + if error.value: + LOG_ERROR(f"group_get_name_size err={error.value}") + raise ToxError(f"group_get_name_size err={error.value}") + return int(result) + + def group_get_name(self, group_number: int) -> str: + """ + Write the name of the group designated by the given group number to a byte array. + Call tox_group_get_name_size to determine the allocation size for the `name` parameter. + :return true on success. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + size = self.group_get_name_size(group_number) + name = create_string_buffer(size) + LOG_DEBUG(f"tox.group_get_name") + result = Tox.libtoxcore.tox_group_get_name(self._tox_pointer, + c_uint32(group_number), + name, byref(error)) + if error.value: + LOG_ERROR(f"group_get_name err={error.value}") + raise ToxError(f"group_get_name err={error.value}") + return str(name[:size], 'utf-8', errors='ignore') + + def group_get_chat_id(self, group_number: int) -> str: + """ + Write the Chat ID designated by the given group number to a byte array. + `chat_id` should have room for at least TOX_GROUP_CHAT_ID_SIZE bytes. + :return chat id. or None if not found. + """ + LOG_INFO(f"tox.group_get_chat_id group_number={group_number}") + if group_number < 0: + LOG_ERROR(f"group_get_chat_id group_number < 0 group_number={group_number}") + raise ToxError(f"group_get_chat_id group_number < 0 group_number={group_number}") + + error = c_int() + buff = create_string_buffer(TOX_GROUP_CHAT_ID_SIZE) + result = Tox.libtoxcore.tox_group_get_chat_id(self._tox_pointer, + c_uint32(group_number), + buff, byref(error)) + if error.value: + if error.value == 1: + LOG_ERROR(f"tox.group_get_chat_id ERROR GROUP_STATE_QUERIES_GROUP_NOT_FxOUND group_number={group_number}") + else: + LOG_ERROR(f"tox.group_get_chat_id group_number={group_number} err={error.value}") + raise ToxError(f"tox_group_get_chat_id err={error.value} group_number={group_number}") +# +# QObject::setParent: Cannot set parent, new parent is in a different thread +# QObject::installEventFilter(): Cannot filter events for objects in a different thread. +# QBasicTimer::start: Timers cannot be started from another thread + result = bin_to_string(buff, TOX_GROUP_CHAT_ID_SIZE) + LOG_DEBUG(f"tox.group_get_chat_id group_number={group_number} result={result}") + + return result + + def group_get_number_groups(self) -> int: + """ + Return the number of groups in the Tox chats array. + """ + LOG_DEBUG(f"tox.group_get_number_groups") + try: + result = Tox.libtoxcore.tox_group_get_number_groups(self._tox_pointer) + except Exception as e: + LOG_WARN(f"tox.group_get_number_groups EXCEPTION {e}") + result = 0 + LOG_INFO(f"tox.group_get_number_groups returning {result}") + return int(result) + + def groups_get_list(self): + raise NotImplementedError('tox_groups_get_list') +# groups_list_size = self.group_get_number_groups() +# groups_list = create_string_buffer(sizeof(c_uint32) * groups_list_size) +# groups_list = POINTER(c_uint32)(groups_list) +# LOG_DEBUG(f"tox.groups_get_list") +# Tox.libtoxcore.tox_groups_get_list(self._tox_pointer, groups_list) +# return groups_list[0:groups_list_size] + + def group_get_privacy_state(self, group_number: int) -> int: + """ + Return the privacy state of the group designated by the given group number. If group number + is invalid, the return value is unspecified. + + The value returned is equal to the data received by the last + `group_privacy_state` callback. + + see the `Group chat founder controls` section for the respective set function. + """ + if group_number < 0: + raise ToxError(f"group_get_privacy_state group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_get_privacy_state") + result = Tox.libtoxcore.tox_group_get_privacy_state(self._tox_pointer, + c_uint32(group_number), + byref(error)) + if error.value: + LOG_ERROR(f"tox.group_get_privacy_state err={error.value}") + raise ToxError(f"tox.group_get_privacy_state err={error.value}") + return int(result) + + def group_get_peer_limit(self, group_number: int) -> int: + """ + Return the maximum number of peers allowed for the group designated by the given group number. + If the group number is invalid, the return value is unspecified. + + The value returned is equal to the data received by the last + `group_peer_limit` callback. + + see the `Group chat founder controls` section for the respective set function. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_get_peer_limit") + result = Tox.libtoxcore.tox_group_get_peer_limit(self._tox_pointer, + c_uint(group_number), + byref(error)) + if error.value: + LOG_ERROR(f"tox.group_get_peer_limit err={error.value}") + raise ToxError(f"tox.group_get_peer_limit err={error.value}") + return int(result) + + def group_get_password_size(self, group_number: int) -> int: + """ + Return the length of the group password. If the group number is invalid, the + return value is unspecified. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_TRACE(f"tox_group_get_password_size") + result = Tox.libtoxcore.tox_group_get_password_size(self._tox_pointer, + c_uint(group_number), byref(error)) + if error.value: + LOG_ERROR(f"group_get_password_size err={error.value}") + raise ToxError(f"group_get_password_size err={error.value}") + return result + + def group_get_password(self, group_number: int) -> str: + """ + Write the password for the group designated by the given group number to a byte array. + + Call tox_group_get_password_size to determine the allocation size for the `password` parameter. + + The data received is equal to the data received by the last + `group_password` callback. + + see the `Group chat founder controls` section for the respective set function. + + :return password + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + size = self.group_get_password_size(group_number) + password = create_string_buffer(size) + LOG_DEBUG(f"tox.group_get_password") + result = Tox.libtoxcore.tox_group_get_password(self._tox_pointer, + c_uint(group_number), + password, byref(error)) + if error.value: + LOG_ERROR(f"group_get_password err={error.value}") + raise ToxError(f"group_get_password err={error.value}") + return str(password[:size], 'utf-8', errors='ignore') + + def callback_group_topic(self, callback: Union[Callable,None], user_data: Union[bytes,None] = None) -> None: + """ + Set the callback for the `group_topic` event. Pass NULL to unset. + This event is triggered when a peer changes the group topic. + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + + LOG_DEBUG(f"tox.callback_group_topic") + if callback is None: + Tox.libtoxcore.tox_callback_group_topic(self._tox_pointer, POINTER(None)()) + self.group_topic_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_char_p, c_size_t, c_void_p) + self.group_topic_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_topic") + Tox.libtoxcore.tox_callback_group_topic(self._tox_pointer, self.group_topic_cb) + except Exception as e: + LOG_WARN(f" Exception {e}") + + def callback_group_privacy_state(self, callback: Union[Callable,None], user_data: Union[bytes,None] = None) -> None: + """ + Set the callback for the `group_privacy_state` event. Pass NULL to unset. + This event is triggered when the group founder changes the privacy state. + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + + LOG_DEBUG(f"tox.callback_group_privacy_state") + if callback is None: + Tox.libtoxcore.tox_callback_group_privacy_state(self._tox_pointer, POINTER(None)()) + self.group_privacy_state_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_int, c_void_p) + self.group_privacy_state_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_privacy_state") + Tox.libtoxcore.tox_callback_group_privacy_state(self._tox_pointer, self.group_privacy_state_cb) + except Exception as e: + LOG_WARN(f" Exception {e}") + + def callback_group_peer_limit(self, callback: Union[Callable,None], user_data: Union[bytes,None] = None) -> None: + """ + Set the callback for the `group_peer_limit` event. Pass NULL to unset. + This event is triggered when the group founder changes the maximum peer limit. + """ + + LOG_DEBUG(f"tox.callback_group_peer_limit") + if user_data is not None: + isinstance(user_data, Array), type(user_data) + if callback is None: + Tox.libtoxcore.tox_callback_group_peer_limit(self._tox_pointer, POINTER(None)()) + self.group_peer_limit_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_void_p) + self.group_peer_limit_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_peer_limit") + Tox.libtoxcore.tox_callback_group_peer_limit(self._tox_pointer, self.group_peer_limit_cb) + except Exception as e: + LOG_WARN(f" Exception {e}") + + def callback_group_password(self, callback: Union[Callable,None], user_data: Union[bytes,None] = None) -> None: + """ + Set the callback for the `group_password` event. Pass NULL to unset. + This event is triggered when the group founder changes the group password. + """ + + LOG_DEBUG(f"tox.callback_group_password") + if user_data is not None: + isinstance(user_data, Array), type(user_data) + if callback is None: + Tox.libtoxcore.tox_callback_group_password(self._tox_pointer, POINTER(None)()) + self.group_password_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_char_p, c_size_t, c_void_p) + self.group_password_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_password") + Tox.libtoxcore.tox_callback_group_password(self._tox_pointer, self.group_password_cb) + except Exception as e: + LOG_WARN(f"tox.callback_group_password Exception {e}") + + # Group message sending + + def group_send_custom_packet(self, group_number: int, lossless: bool, data: bytes) -> bool: + """Send a custom packet to the group. + + If lossless is true the packet will be lossless. Lossless + packet behaviour is comparable to TCP (reliability, arrive in + order) but with packets instead of a stream. + + If lossless is false, the packet will be lossy. Lossy packets + behave like UDP packets, meaning they might never reach the + other side or might arrive more than once (if someone is + messing with the connection) or might arrive in the wrong + order. + + Unless latency is an issue or message reliability is not + important, it is recommended that you use lossless custom + packets. + + :param group_number: The group number of the group the message is intended for. + :param lossless: True if the packet should be lossless. + :param data A byte array containing the packet data. + :return True on success. + + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + isinstance(data, Array), type(data) + error = c_int() + LOG_DEBUG(f"tox.group_send_custom_packet") + result = Tox.libtoxcore.tox_group_send_custom_packet(self._tox_pointer, + c_uint(group_number), + lossless, + c_char_p(data), + c_size_t(len(data)), + byref(error)) + if error.value: + LOG_ERROR(f"group_send_custom_packet err={error.value}") + raise ToxError(f"group_send_custom_packet err={error.value}") + return bool(result) + + def group_send_private_message(self, group_number: int, peer_id: int, message_type: int, message: str) -> bool: + """ + Send a text chat message to the specified peer in the specified group. + + This function creates a group private message packet and pushes it into the send + queue. + + The message length may not exceed TOX_MAX_MESSAGE_LENGTH. Larger messages + must be split by the client and sent as separate messages. Other clients can + then reassemble the fragments. Messages may not be empty. + + :param group_number: The group number of the group the message is intended for. + :param peer_id: The ID of the peer the message is intended for. + :param message: A non-NULL pointer to the first element of a byte array containing the message text. + + :return True on success. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + if type(message) == str: + message = bytes(message, 'utf-8') # not c_char_p + error = c_int() + LOG_DEBUG(f"group_send_private_message") + result = Tox.libtoxcore.tox_group_send_private_message(self._tox_pointer, + c_uint(group_number), + c_uint32(peer_id), + c_uint32(message_type), + message, + c_size_t(len(message)), + byref(error)) + if error.value: + s = sGetError(error.value, TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE) + LOG_ERROR(f"group_send_private_message err={error.value} {s}") + raise ToxError(f"group_send_private_message err={error.value} {s}") + + return bool(result) + + def group_send_message(self, group_number: int, message_type: int, message: str) -> bool: + """ + Send a text chat message to the group. + + This function creates a group message packet and pushes it into the send + queue. + + The message length may not exceed TOX_MAX_MESSAGE_LENGTH. Larger messages + must be split by the client and sent as separate messages. Other clients can + then reassemble the fragments. Messages may not be empty. + + :param group_number: The group number of the group the message is intended for. + :param message_type: Message type (normal, action, ...). + :param message: A non-NULL pointer to the first element of a byte array containing the message text. + + :return True on success. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + # uint32_t message_id = 0; + message_id = c_int() # or POINTER(None)() + if type(message) != bytes: + message = bytes(message, 'utf-8') # not c_char_p() + LOG_DEBUG(f"tox.group_send_message") + # bool tox_group_send_message(const Tox *tox, uint32_t group_number, Tox_Message_Type type, const uint8_t *message, size_t length, uint32_t *message_id, Tox_Err_Group_Send_Message *error) + result = Tox.libtoxcore.tox_group_send_message(self._tox_pointer, + c_uint(group_number), + c_uint32(message_type), + message, + c_size_t(len(message)), + # dunno + byref(message_id), + byref(error)) + + if error.value: + s = sGetError(error.value, TOX_ERR_GROUP_SEND_MESSAGE) + LOG_ERROR(f"group_send_message err={error.value} {s}") + raise ToxError(f"group_send_message err={error.value} {s}") + + return bool(result) + + # Group message receiving + + def callback_group_message(self, callback: Union[Callable,None], user_data: Union[bytes,None] = None) -> None: + """ + Set the callback for the `group_message` event. Pass NULL to unset. + This event is triggered when the client receives a group message. + + Callback: python function with params: + tox Tox* instance + group_number The group number of the group the message is intended for. + peer_id The ID of the peer who sent the message. + type The type of message (normal, action, ...). + message The message data. + length The length of the message. + user_data - user data + """ + LOG_DEBUG(f"tox.callback_group_message") + if user_data is not None: + isinstance(user_data, Array), type(user_data) + if callback is None: + Tox.libtoxcore.tox_callback_group_message(self._tox_pointer, POINTER(None)()) + self.group_message_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_int, c_char_p, c_size_t, c_void_p) + self.group_message_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_message") + Tox.libtoxcore.tox_callback_group_message(self._tox_pointer, self.group_message_cb) + except Exception as e: + LOG_ERROR(f"tox.callback_group_message {e}") + + def callback_group_private_message(self, callback: Union[Callable,None], user_data: Union[bytes,None] = None) -> None: + """ + Set the callback for the `group_private_message` event. Pass NULL to unset. + This event is triggered when the client receives a private message. + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_uint8, c_char_p, c_size_t, c_void_p) + self.group_private_message_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_private_message") + Tox.libtoxcore.tox_callback_group_private_message(self._tox_pointer, self.group_private_message_cb) + except Exception as e: + LOG_ERROR(f"tox.callback_group_private_message {e}") # req + + def callback_group_custom_packet(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `group_custom_packet` event. Pass NULL to unset. + + This event is triggered when the client receives a custom packet. + """ + + LOG_DEBUG(f"tox.callback_group_custom_packet") + if user_data is not None: + isinstance(user_data, Array), type(user_data) + if callback is None: + Tox.libtoxcore.tox_callback_group_custom_packet(self._tox_pointer, POINTER(None)()) + self.group_custom_packet_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, POINTER(c_uint8), c_void_p) + self.group_custom_packet_cb = c_callback(callback) + LOG_DEBUG(f"tox.callback_group_custom_packet") + Tox.libtoxcore.tox_callback_group_custom_packet(self._tox_pointer, self.group_custom_packet_cb) + + # Group chat inviting and join/part events + + def group_invite_friend(self, group_number: int, friend_number: int) -> bool: + """ + Invite a friend to a group. + + This function creates an invite request packet and pushes it to the send queue. + + :param group_number: The group number of the group the message is intended for. + :param friend_number: The friend number of the friend the invite is intended for. + + :return True on success. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_invite_friend") + result = Tox.libtoxcore.tox_group_invite_friend(self._tox_pointer, c_uint(group_number), c_uint32(friend_number), byref(error)) + if error.value: + s = sGetError(error.value, TOX_ERR_GROUP_INVITE_FRIEND) + LOG_ERROR(f"group_invite_friend err={error.value} {s}") + raise ToxError(f"group_invite_friend err={error.value} {s}") + return bool(result) + + # API change - this no longer exists +# @staticmethod +# def group_self_peer_info_new(): +# error = c_int() +# f = Tox.libtoxcore.tox_group_self_peer_info_new +# f.restype = POINTER(GroupChatSelfPeerInfo) +# result = f(byref(error)) +# return result + + # status should be dropped + def group_invite_accept(self, invite_data, friend_number: int, nick: str, status: str='', password=None) -> int: + """ + Accept an invite to a group chat that the client previously received from a friend. The invite + is only valid while the inviter is present in the group. + + :param invite_data: The invite data received from the `group_invite` event. + :param password: The password required to join the group. Set to NULL if no password is required. + :return the group_number on success, UINT32_MAX on failure. + """ + + error = c_int() + f = Tox.libtoxcore.tox_group_invite_accept + f.restype = c_uint32 + if nick and type(nick) == str: + nick = bytes(nick, 'utf-8') + else: + nick = b'' + if password and type(password) == str: + password = bytes(password, 'utf-8') + else: + password = None + if invite_data and type(invite_data) == str: + invite_data = bytes(invite_data, 'utf-8') + else: + invite_data = b'' + + LOG_INFO(f"group_invite_accept friend_number={friend_number} nick={nick} {invite_data}") + try: + assert type(invite_data) == bytes + result = f(self._tox_pointer, + c_uint32(friend_number), + invite_data, + c_size_t(len(invite_data)), + c_char_p(nick), + c_size_t(len(nick)), + c_char_p(password), len(password) if password is not None else 0, + byref(error)) + except Exception as e: + LOG_ERROR(f"group_invite_accept ERROR {e}") + raise ToxError(f"group_invite_accept ERROR {e}") + if error.value: + s = sGetError(error.value, TOX_ERR_GROUP_INVITE_ACCEPT) + LOG_ERROR(f"group_invite_friend err={error.value} {s}") + raise ToxError(f"group_invite_accept {s} err={error.value}") + return result + + def callback_group_invite(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `group_invite` event. Pass NULL to unset. + + This event is triggered when the client receives a group invite from a friend. The client must store + invite_data which is used to join the group via tox_group_invite_accept. + + Callback: python function with params: + tox - Tox* + friend_number The friend number of the contact who sent the invite. + invite_data The invite data. + length The length of invite_data. + user_data - user data + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + if callback is None: + Tox.libtoxcore.tox_callback_group_invite(self._tox_pointer, POINTER(None)()) + self.group_invite_cb = None + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, POINTER(c_uint8), c_size_t, + POINTER(c_uint8), c_size_t, c_void_p) + self.group_invite_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_invite") + Tox.libtoxcore.tox_callback_group_invite(self._tox_pointer, self.group_invite_cb) + except Exception as e: + LOG_DEBUG(f"tox.callback_conference_invite") + + def callback_group_peer_join(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `group_peer_join` event. Pass NULL to unset. + + This event is triggered when a peer other than self joins the group. + Callback: python function with params: + tox - Tox* + group_number - group number + peer_id - peer id + user_data - user data + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + + if callback is None: + Tox.libtoxcore.tox_callback_group_peer_join(self._tox_pointer, POINTER(None)()) + self.group_peer_join_cb = None + return + + LOG_DEBUG(f"tox.callback_group_peer_join") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_void_p) + self.group_peer_join_cb = c_callback(callback) + try: + Tox.libtoxcore.tox_callback_group_peer_join(self._tox_pointer, self.group_peer_join_cb) + except Exception as e: + LOG_ERROR(f"callback_group_peer_join {e}") # req + + def callback_group_peer_exit(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `group_peer_exit` event. Pass NULL to unset. + + This event is triggered when a peer other than self exits the group. + """ + + if callback is None: + Tox.libtoxcore.tox_callback_group_peer_exit(self._tox_pointer, POINTER(None)()) + self.group_peer_exit_cb = None + return + + LOG_DEBUG(f"tox.callback_group_peer_exit") + c_callback = CFUNCTYPE(None, c_void_p, + c_uint32, # group_number, + c_uint32, # peer_id, + c_int, # exit_type + c_char_p, # name + c_size_t, # name length + c_char_p, # message + c_size_t, # message length + c_void_p) # user_data + self.group_peer_exit_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_peer_exit") + Tox.libtoxcore.tox_callback_group_peer_exit(self._tox_pointer, self.group_peer_exit_cb) + except Exception as e: + LOG_ERROR(f"tox.callback_group_peer_exit {e}") # req + else: + LOG_DEBUG(f"tox.callback_group_peer_exit") + + def callback_group_self_join(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `group_self_join` event. Pass NULL to unset. + + This event is triggered when the client has successfully joined a group. Use this to initialize + any group information the client may need. + Callback: python fucntion with params: + tox - *Tox + group_number - group number + user_data - user data + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + + if callback is None: + Tox.libtoxcore.tox_callback_group_self_join(self._tox_pointer, POINTER(None)()) + self.group_self_join_cb = None + return + + LOG_DEBUG(f"tox.callback_group_self_join") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_void_p) + self.group_self_join_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_self_join") + Tox.libtoxcore.tox_callback_group_self_join(self._tox_pointer, self.group_self_join_cb) + except Exception as e: + LOG_ERROR(f"tox.callback_group_self_join {e}") # req + else: + LOG_DEBUG(f"tox.callback_group_self_join") + + def callback_group_join_fail(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `group_join_fail` event. Pass NULL to unset. + + This event is triggered when the client fails to join a group. + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + + if callback is None: + Tox.libtoxcore.tox_callback_group_join_fail(self._tox_pointer, POINTER(None)()) + self.group_join_fail_cb = None + return + + LOG_DEBUG(f"tox.callback_group_join_fail") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_int, c_uint32, c_void_p) + self.group_join_fail_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_join_fail") + Tox.libtoxcore.tox_callback_group_join_fail(self._tox_pointer, self.group_join_fail_cb) + except Exception as e: + LOG_ERROR(f"tox.callback_group_join_fail {e}") # req + + # Group chat founder controls (these only work for the group founder) + + def group_founder_set_password(self, group_number: int, password: str) -> bool: + """ + Set or unset the group password. + + This function sets the groups password, creates a new group shared state including the change, + and distributes it to the rest of the group. + + :param group_number: The group number of the group for which we wish to set the password. + :param password: The password we want to set. Set password to NULL to unset the password. + + :return True on success. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_founder_set_password") + result = Tox.libtoxcore.tox_group_founder_set_password(self._tox_pointer, c_uint(group_number), password, + c_size_t(len(password)), + byref(error)) + if error.value: + s = sGetError(error.value, TOX_ERR_GROUP_FOUNDER_SET_PASSWORD) + LOG_ERROR(f"group_founder_set_password err={error.value} {s}") + raise ToxError(f"group_founder_set_password {s} err={error.value}") + return bool(result) + + def group_founder_set_privacy_state(self, group_number: int, privacy_state: int) -> bool: + """ + Set the group privacy state. + + This function sets the group's privacy state, creates a new group shared state + including the change, and distributes it to the rest of the group. + + If an attempt is made to set the privacy state to the same state that the group is already + in, the function call will be successful and no action will be taken. + + :param group_number: The group number of the group for which we wish to change the privacy state. + :param privacy_state: The privacy state we wish to set the group to. + + :return true on success. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_founder_set_privacy_state") + result = Tox.libtoxcore.tox_group_founder_set_privacy_state(self._tox_pointer, c_uint(group_number), privacy_state, + byref(error)) + if error.value: + LOG_ERROR(f"group_founder_set_privacy_state err={error.value}") + raise ToxError(f"group_founder_set_privacy_state err={error.value}") + return bool(result) + + def group_founder_set_peer_limit(self, group_number: int, max_peers: int) -> bool: + """ + Set the group peer limit. + + This function sets a limit for the number of peers who may be in the group, creates a new + group shared state including the change, and distributes it to the rest of the group. + + :param group_number: The group number of the group for which we wish to set the peer limit. + :param max_peers: The maximum number of peers to allow in the group. + + :return True on success. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_founder_set_peer_limit") + result = Tox.libtoxcore.tox_group_founder_set_peer_limit(self._tox_pointer, + c_uint(group_number), + max_peers, + byref(error)) + if error.value: + LOG_ERROR(f"group_founder_set_peer_limit err={error.value}") + raise ToxError(f"group_founder_set_peer_limit err={error.value}") + return bool(result) + + # Group chat moderation + + def group_mod_set_role(self, group_number: int, peer_id: int, role: int) -> bool: + """ + Set a peer's role. + + This function will first remove the peer's previous role and then assign them a new role. + It will also send a packet to the rest of the group, requesting that they perform + the role reassignment. Note: peers cannot be set to the founder role. + + :param group_number: The group number of the group the in which you wish set the peer's role. + :param peer_id: The ID of the peer whose role you wish to set. + :param role: The role you wish to set the peer to. + + :return True on success. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_mod_set_role") + result = Tox.libtoxcore.tox_group_mod_set_role(self._tox_pointer, + c_uint(group_number), + c_uint32(peer_id), + c_uint32(role), byref(error)) + if error.value: + LOG_ERROR(f"group_mod_set_role err={error.value}") + raise ToxError(f"group_mod_set_role err={error.value}") + return bool(result) + + def callback_group_moderation(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `group_moderation` event. Pass NULL to unset. + + This event is triggered when a moderator or founder executes a moderation event. + (tox_data->tox, group_number, source_peer_number, target_peer_number, + (Tox_Group_Mod_Event)mod_type, tox_data->user_data); + TOX_GROUP_MOD_EVENT = [0,1,2,3,4] TOX_GROUP_MOD_EVENT['MODERATOR'] + """ + if user_data is not None: + isinstance(user_data, Array), type(user_data) + +# LOG_DEBUG(f"callback_group_moderation") + if callback is None: + self.group_moderation_cb = None + LOG_DEBUG(f"tox.callback_group_moderation") + Tox.libtoxcore.tox_callback_group_moderation(self._tox_pointer, POINTER(None)()) + return + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_uint32, c_int, c_void_p) + self.group_moderation_cb = c_callback(callback) + try: + LOG_DEBUG(f"tox.callback_group_moderation") + Tox.libtoxcore.tox_callback_group_moderation(self._tox_pointer, self.group_moderation_cb) + except Exception as e: + LOG_ERROR(f"tox.callback_group_moderation {e}") # req + else: + LOG_DEBUG(f"tox.callback_group_moderation") + + def group_toggle_set_ignore(self, group_number: int, peer_id: int, ignore) -> bool: + return self.group_set_ignore(group_number, peer_id, ignore) + + def group_set_ignore(self, group_number: int, peer_id: int, ignore: bool) -> bool: + """ + Ignore or unignore a peer. + + :param group_number: The group number of the group the in which you wish to ignore a peer. + :param peer_id: The ID of the peer who shall be ignored or unignored. + :param ignore: True to ignore the peer, false to unignore the peer. + + :return True on success. + """ + if group_number < 0: + raise ToxError(f"tox_group_ group_number < 0 {group_number}") + + error = c_int() + LOG_DEBUG(f"tox.group_set_ignore") + result = Tox.libtoxcore.tox_group_set_ignore(self._tox_pointer, + c_uint32(group_number), + c_uint32(peer_id), + c_bool(ignore), + byref(error)) + if error.value: + LOG_ERROR(f"tox.group_set_ignore err={error.value}") + raise ToxError("tox_group_set_ignore err={error.value}") + return bool(result) diff --git a/src/toxygen_wrapper/toxav.py b/src/toxygen_wrapper/toxav.py new file mode 100644 index 0000000..dfe9ec8 --- /dev/null +++ b/src/toxygen_wrapper/toxav.py @@ -0,0 +1,409 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- + +from ctypes import (CFUNCTYPE, POINTER, ArgumentError, byref, c_bool, c_char_p, + c_int, c_int32, c_size_t, c_uint8, c_uint16, c_uint32, + c_void_p, cast) +from typing import Union, Callable + +try: + from tox_wrapper.libtox import LibToxAV + import tox_wrapper.toxav_enums as enum +except: + from libtox import LibToxAV + import toxav_enums as enum +class ToxError(RuntimeError): pass + +def LOG_ERROR(a: str) -> None: print('EROR> '+a) +def LOG_WARN(a: str) -> None: print('WARN> '+a) +def LOG_INFO(a: str) -> None: print('INFO> '+a) +def LOG_DEBUG(a: str) -> None: print('DBUG> '+a) +def LOG_TRACE(a: str) -> None: pass # print('DEBUGx: '+a) + +class ToxAV: + """ + The ToxAV instance type. Each ToxAV instance can be bound to only one Tox instance, and Tox instance can have only + one ToxAV instance. One must make sure to close ToxAV instance prior closing Tox instance otherwise undefined + behaviour occurs. Upon closing of ToxAV instance, all active calls will be forcibly terminated without notifying + peers. + """ + + # Creation and destruction + + def __init__(self, tox_pointer): + """ + Start new A/V session. There can only be only one session per Tox instance. + + :param tox_pointer: pointer to Tox instance + """ + self.libtoxav = LibToxAV() + toxav_err_new = c_int() + f = self.libtoxav.toxav_new + f.restype = POINTER(c_void_p) + self._toxav_pointer = f(tox_pointer, byref(toxav_err_new)) + toxav_err_new = toxav_err_new.value + if toxav_err_new == enum.TOXAV_ERR_NEW['NULL']: + raise ArgumentError('One of the arguments to the function was NULL when it was not expected.') + if toxav_err_new == enum.TOXAV_ERR_NEW['MALLOC']: + raise MemoryError('Memory allocation failure while trying to allocate structures required for the A/V ' + 'session.') + if toxav_err_new == enum.TOXAV_ERR_NEW['MULTIPLE']: + raise RuntimeError('Attempted to create a second session for the same Tox instance.') + + self.call_state_cb = None + self.audio_receive_frame_cb = None + self.video_receive_frame_cb = None + self.call_cb = None + + def kill(self) -> None: + """ + Releases all resources associated with the A/V session. + + If any calls were ongoing, these will be forcibly terminated without notifying peers. After calling this + function, no other functions may be called and the av pointer becomes invalid. + """ + self.libtoxav.toxav_kill(self._toxav_pointer) + + def get_tox_pointer(self): + """ + Returns the Tox instance the A/V object was created for. + + :return: pointer to the Tox instance + """ + self.libtoxav.toxav_get_tox.restype = POINTER(c_void_p) + return self.libtoxav.toxav_get_tox(self._toxav_pointer) + + # A/V event loop + + def iteration_interval(self) -> int: + """ + Returns the interval in milliseconds when the next toxav_iterate call should be. If no call is active at the + moment, this function returns 200. + + :return: interval in milliseconds + """ + return int(self.libtoxav.toxav_iteration_interval(self._toxav_pointer)) + + def iterate(self) -> None: + """ + Main loop for the session. This function needs to be called in intervals of toxav_iteration_interval() + milliseconds. It is best called in the separate thread from tox_iterate. + """ + self.libtoxav.toxav_iterate(self._toxav_pointer) + + # Call setup + + def call(self, friend_number: int, audio_bit_rate: int, video_bit_rate: int) -> bool: + """ + Call a friend. This will start ringing the friend. + + It is the client's responsibility to stop ringing after a certain timeout, if such behaviour is desired. If the + client does not stop ringing, the library will not stop until the friend is disconnected. Audio and video + receiving are both enabled by default. + + :param friend_number: The friend number of the friend that should be called. + :param audio_bit_rate: Audio bit rate in Kb/sec. Set this to 0 to disable audio sending. + :param video_bit_rate: Video bit rate in Kb/sec. Set this to 0 to disable video sending. + :return: True on success. + """ + toxav_err_call = c_int() + LOG_DEBUG(f"toxav_call") + result = self.libtoxav.toxav_call(self._toxav_pointer, c_uint32(friend_number), c_uint32(audio_bit_rate), + c_uint32(video_bit_rate), byref(toxav_err_call)) + toxav_err_call = toxav_err_call.value + if toxav_err_call == enum.TOXAV_ERR_CALL['OK']: + return bool(result) + if toxav_err_call == enum.TOXAV_ERR_CALL['MALLOC']: + raise MemoryError('A resource allocation error occurred while trying to create the structures required for ' + 'the call.') + if toxav_err_call == enum.TOXAV_ERR_CALL['SYNC']: + raise RuntimeError('Synchronization error occurred.') + if toxav_err_call == enum.TOXAV_ERR_CALL['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend number did not designate a valid friend.') + if toxav_err_call == enum.TOXAV_ERR_CALL['FRIEND_NOT_CONNECTED']: + raise ArgumentError('The friend was valid, but not currently connected.') + if toxav_err_call == enum.TOXAV_ERR_CALL['FRIEND_ALREADY_IN_CALL']: + raise ArgumentError('Attempted to call a friend while already in an audio or video call with them.') + if toxav_err_call == enum.TOXAV_ERR_CALL['INVALID_BIT_RATE']: + raise ArgumentError('Audio or video bit rate is invalid.') + raise ArgumentError('The function did not return OK') + + def callback_call(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `call` event. Pass None to unset. + + :param callback: The function for the call callback. + + Should take pointer (c_void_p) to ToxAV object, + The friend number (c_uint32) from which the call is incoming. + True (c_bool) if friend is sending audio. + True (c_bool) if friend is sending video. + pointer (c_void_p) to user_data + :param user_data: pointer (c_void_p) to user data + """ + if callback is None: + self.libtoxav.toxav_callback_call(self._toxav_pointer, POINTER(None)(), user_data) + self.call_cb = None + return + LOG_DEBUG(f"toxav_callback_call") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_bool, c_bool, c_void_p) + self.call_cb = c_callback(callback) + self.libtoxav.toxav_callback_call(self._toxav_pointer, self.call_cb, user_data) + + def answer(self, friend_number: int, audio_bit_rate: int, video_bit_rate: int) -> bool: + """ + Accept an incoming call. + + If answering fails for any reason, the call will still be pending and it is possible to try and answer it later. + Audio and video receiving are both enabled by default. + + :param friend_number: The friend number of the friend that is calling. + :param audio_bit_rate: Audio bit rate in Kb/sec. Set this to 0 to disable audio sending. + :param video_bit_rate: Video bit rate in Kb/sec. Set this to 0 to disable video sending. + :return: True on success. + """ + toxav_err_answer = c_int() + LOG_DEBUG(f"toxav_answer") + result = self.libtoxav.toxav_answer(self._toxav_pointer, + c_uint32(friend_number), + c_uint32(audio_bit_rate), + c_uint32(video_bit_rate), + byref(toxav_err_answer)) + toxav_err_answer = toxav_err_answer.value + if toxav_err_answer == enum.TOXAV_ERR_ANSWER['OK']: + return bool(result) + if toxav_err_answer == enum.TOXAV_ERR_ANSWER['SYNC']: + raise RuntimeError('Synchronization error occurred.') + if toxav_err_answer == enum.TOXAV_ERR_ANSWER['CODEC_INITIALIZATION']: + raise RuntimeError('Failed to initialize codecs for call session. Note that codec initiation will fail if ' + 'there is no receive callback registered for either audio or video.') + if toxav_err_answer == enum.TOXAV_ERR_ANSWER['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend number did not designate a valid friend.') + if toxav_err_answer == enum.TOXAV_ERR_ANSWER['FRIEND_NOT_CALLING']: + raise ArgumentError('The friend was valid, but they are not currently trying to initiate a call. This is ' + 'also returned if this client is already in a call with the friend.') + if toxav_err_answer == enum.TOXAV_ERR_ANSWER['INVALID_BIT_RATE']: + raise ArgumentError('Audio or video bit rate is invalid.') + raise ToxError('The function did not return OK') + + # Call state graph + + def callback_call_state(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `call_state` event. Pass None to unset. + + :param callback: Python function. + The function for the call_state callback. + + Should take pointer (c_void_p) to ToxAV object, + The friend number (c_uint32) for which the call state changed. + The bitmask of the new call state which is guaranteed to be different than the previous state. The state is set + to 0 when the call is paused. The bitmask represents all the activities currently performed by the friend. + pointer (c_void_p) to user_data + :param user_data: pointer (c_void_p) to user data + """ + if callback is None: + self.libtoxav.toxav_callback_call_state(self._toxav_pointer, POINTER(None)(), user_data) + self.call_state_cb = None + return + LOG_DEBUG(f"callback_call_state") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint32, c_void_p) + self.call_state_cb = c_callback(callback) + self.libtoxav.toxav_callback_call_state(self._toxav_pointer, self.call_state_cb, user_data) + + # Call control + + def call_control(self, friend_number: int, control: int) -> bool: + """ + Sends a call control command to a friend. + + :param friend_number: The friend number of the friend this client is in a call with. + :param control: The control command to send. + :return: True on success. + """ + toxav_err_call_control = c_int() + LOG_DEBUG(f"call_control") + result = self.libtoxav.toxav_call_control(self._toxav_pointer, + c_uint32(friend_number), + c_int(control), + byref(toxav_err_call_control)) + toxav_err_call_control = toxav_err_call_control.value + if toxav_err_call_control == enum.TOXAV_ERR_CALL_CONTROL['OK']: + return bool(result) + if toxav_err_call_control == enum.TOXAV_ERR_CALL_CONTROL['SYNC']: + raise RuntimeError('Synchronization error occurred.') + if toxav_err_call_control == enum.TOXAV_ERR_CALL_CONTROL['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number passed did not designate a valid friend.') + if toxav_err_call_control == enum.TOXAV_ERR_CALL_CONTROL['FRIEND_NOT_IN_CALL']: + raise RuntimeError('This client is currently not in a call with the friend. Before the call is answered, ' + 'only CANCEL is a valid control.') + if toxav_err_call_control == enum.TOXAV_ERR_CALL_CONTROL['INVALID_TRANSITION']: + raise RuntimeError('Happens if user tried to pause an already paused call or if trying to resume a call ' + 'that is not paused.') + raise ToxError('The function did not return OK.') + + # TODO Controlling bit rates + + # A/V sending + + def audio_send_frame(self, friend_number: int, pcm, sample_count: int, channels: int, sampling_rate: int) -> bool: + """ + Send an audio frame to a friend. + + The expected format of the PCM data is: [s1c1][s1c2][...][s2c1][s2c2][...]... + Meaning: sample 1 for channel 1, sample 1 for channel 2, ... + For mono audio, this has no meaning, every sample is subsequent. For stereo, this means the expected format is + LRLRLR... with samples for left and right alternating. + + :param friend_number: The friend number of the friend to which to send an audio frame. + :param pcm: An array of audio samples. The size of this array must be sample_count * channels. + :param sample_count: Number of samples in this frame. Valid numbers here are + ((sample rate) * (audio length) / 1000), where audio length can be 2.5, 5, 10, 20, 40 or 60 milliseconds. + :param channels: Number of audio channels. Sulpported values are 1 and 2. + :param sampling_rate: Audio sampling rate used in this frame. Valid sampling rates are 8000, 12000, 16000, + 24000, or 48000. + """ + toxav_err_send_frame = c_int() + LOG_TRACE(f"toxav_audio_send_frame") + assert sampling_rate in [8000, 12000, 16000, 24000, 48000] + result = self.libtoxav.toxav_audio_send_frame(self._toxav_pointer, + c_uint32(friend_number), + cast(pcm, c_void_p), + c_size_t(sample_count), c_uint8(channels), + c_uint32(sampling_rate), byref(toxav_err_send_frame)) + toxav_err_send_frame = toxav_err_send_frame.value + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['OK']: + return bool(result) + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['NULL']: + raise ArgumentError('The samples data pointer was NULL.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number passed did not designate a valid friend.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['FRIEND_NOT_IN_CALL']: + raise RuntimeError('This client is currently not in a call with the friend.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['SYNC']: + raise RuntimeError('Synchronization error occurred.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['INVALID']: + raise ArgumentError('One of the frame parameters was invalid. E.g. the resolution may be too small or too ' + 'large, or the audio sampling rate may be unsupported.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['PAYLOAD_TYPE_DISABLED']: + raise RuntimeError('Either friend turned off audio or video receiving or we turned off sending for the said' + 'payload.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['RTP_FAILED']: + RuntimeError('Failed to push frame through rtp interface.') + raise ToxError('The function did not return OK.') + + def video_send_frame(self, friend_number: int, width: int, height: int, y, u, v) -> bool: + """ + Send a video frame to a friend. + + Y - plane should be of size: height * width + U - plane should be of size: (height/2) * (width/2) + V - plane should be of size: (height/2) * (width/2) + + :param friend_number: The friend number of the friend to which to send a video frame. + :param width: Width of the frame in pixels. + :param height: Height of the frame in pixels. + :param y: Y (Luminance) plane data. + :param u: U (Chroma) plane data. + :param v: V (Chroma) plane data. + """ + toxav_err_send_frame = c_int() + LOG_TRACE(f"toxav_video_send_frame") + result = self.libtoxav.toxav_video_send_frame(self._toxav_pointer, + c_uint32(friend_number), + c_uint16(width), + c_uint16(height), + c_char_p(y), + c_char_p(u), + c_char_p(v), + byref(toxav_err_send_frame)) + toxav_err_send_frame = toxav_err_send_frame.value + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['OK']: + return bool(result) + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['NULL']: + raise ArgumentError('One of Y, U, or V was NULL.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['FRIEND_NOT_FOUND']: + raise ArgumentError('The friend_number passed did not designate a valid friend.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['FRIEND_NOT_IN_CALL']: + raise RuntimeError('This client is currently not in a call with the friend.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['SYNC']: + raise RuntimeError('Synchronization error occurred.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['INVALID']: + raise ArgumentError('One of the frame parameters was invalid. E.g. the resolution may be too small or too ' + 'large, or the audio sampling rate may be unsupported.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['PAYLOAD_TYPE_DISABLED']: + raise RuntimeError('Either friend turned off audio or video receiving or we turned off sending for the said' + 'payload.') + if toxav_err_send_frame == enum.TOXAV_ERR_SEND_FRAME['RTP_FAILED']: + RuntimeError('Failed to push frame through rtp interface.') + raise ToxError('The function did not return OK.') + + # A/V receiving + + def callback_audio_receive_frame(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `audio_receive_frame` event. Pass None to unset. + + :param callback: Python function. + Function for the audio_receive_frame callback. The callback can be called multiple times per single + iteration depending on the amount of queued frames in the buffer. The received format is the same as in send + function. + + Should take pointer (c_void_p) to ToxAV object, + The friend number (c_uint32) of the friend who sent an audio frame. + An array (c_uint8) of audio samples (sample_count * channels elements). + The number (c_size_t) of audio samples per channel in the PCM array. + Number (c_uint8) of audio channels. + Sampling rate (c_uint32) used in this frame. + pointer (c_void_p) to user_data + :param user_data: pointer (c_void_p) to user data + """ + if callback is None: + self.libtoxav.toxav_callback_audio_receive_frame(self._toxav_pointer, + POINTER(None)(), + user_data) + self.audio_receive_frame_cb = None + return + LOG_DEBUG(f"toxav_callback_audio_receive_frame") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, POINTER(c_uint8), c_size_t, c_uint8, c_uint32, c_void_p) + self.audio_receive_frame_cb = c_callback(callback) + self.libtoxav.toxav_callback_audio_receive_frame(self._toxav_pointer, self.audio_receive_frame_cb, user_data) + + def callback_video_receive_frame(self, callback: Union[Callable,None], user_data) -> None: + """ + Set the callback for the `video_receive_frame` event. Pass None to unset. + + :param callback: Python function. + The function type for the video_receive_frame callback. + + Should take + toxAV pointer (c_void_p) to ToxAV object, + friend_number The friend number (c_uint32) of the friend who sent a video frame. + width Width (c_uint16) of the frame in pixels. + height Height (c_uint16) of the frame in pixels. + y + u + v Plane data (POINTER(c_uint8)). + The size of plane data is derived from width and height where + Y = MAX(width, abs(ystride)) * height, + U = MAX(width/2, abs(ustride)) * (height/2) and + V = MAX(width/2, abs(vstride)) * (height/2). + ystride + ustride + vstride Strides data (c_int32). Strides represent padding for each plane that may or may not be present. You must + handle strides in your image processing code. Strides are negative if the image is bottom-up + hence why you MUST abs() it when calculating plane buffer size. + user_data pointer (c_void_p) to user_data + :param user_data: pointer (c_void_p) to user data + """ + if callback is None: + self.libtoxav.toxav_callback_video_receive_frame(self._toxav_pointer, POINTER(None)(), user_data) + self.video_receive_frame_cb = None + return + + LOG_DEBUG(f"toxav_callback_video_receive_frame") + c_callback = CFUNCTYPE(None, c_void_p, c_uint32, c_uint16, c_uint16, + POINTER(c_uint8), POINTER(c_uint8), POINTER(c_uint8), + c_int32, c_int32, c_int32, + c_void_p) + self.video_receive_frame_cb = c_callback(callback) + self.libtoxav.toxav_callback_video_receive_frame(self._toxav_pointer, self.video_receive_frame_cb, user_data) diff --git a/src/toxygen_wrapper/toxav_enums.py b/src/toxygen_wrapper/toxav_enums.py new file mode 100644 index 0000000..f8817e1 --- /dev/null +++ b/src/toxygen_wrapper/toxav_enums.py @@ -0,0 +1,133 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- + +TOXAV_ERR_NEW = { + # The function returned successfully. + 'OK': 0, + # One of the arguments to the function was NULL when it was not expected. + 'NULL': 1, + # Memory allocation failure while trying to allocate structures required for the A/V session. + 'MALLOC': 2, + # Attempted to create a second session for the same Tox instance. + 'MULTIPLE': 3, +} + +TOXAV_ERR_CALL = { + # The function returned successfully. + 'OK': 0, + # A resource allocation error occurred while trying to create the structures required for the call. + 'MALLOC': 1, + # Synchronization error occurred. + 'SYNC': 2, + # The friend number did not designate a valid friend. + 'FRIEND_NOT_FOUND': 3, + # The friend was valid, but not currently connected. + 'FRIEND_NOT_CONNECTED': 4, + # Attempted to call a friend while already in an audio or video call with them. + 'FRIEND_ALREADY_IN_CALL': 5, + # Audio or video bit rate is invalid. + 'INVALID_BIT_RATE': 6, +} + +TOXAV_ERR_ANSWER = { + # The function returned successfully. + 'OK': 0, + # Synchronization error occurred. + 'SYNC': 1, + # Failed to initialize codecs for call session. Note that codec initiation will fail if there is no receive callback + # registered for either audio or video. + 'CODEC_INITIALIZATION': 2, + # The friend number did not designate a valid friend. + 'FRIEND_NOT_FOUND': 3, + # The friend was valid, but they are not currently trying to initiate a call. This is also returned if this client + # is already in a call with the friend. + 'FRIEND_NOT_CALLING': 4, + # Audio or video bit rate is invalid. + 'INVALID_BIT_RATE': 5, +} + +TOXAV_FRIEND_CALL_STATE = { + # Set by the AV core if an error occurred on the remote end or if friend timed out. This is the final state after + # which no more state transitions can occur for the call. This call state will never be triggered in combination + # with other call states. + 'ERROR': 1, + # The call has finished. This is the final state after which no more state transitions can occur for the call. This + # call state will never be triggered in combination with other call states. + 'FINISHED': 2, + # The flag that marks that friend is sending audio. + 'SENDING_A': 4, + # The flag that marks that friend is sending video. + 'SENDING_V': 8, + # The flag that marks that friend is receiving audio. + 'ACCEPTING_A': 16, + # The flag that marks that friend is receiving video. + 'ACCEPTING_V': 32, +} + +TOXAV_CALL_CONTROL = { + # Resume a previously paused call. Only valid if the pause was caused by this client, if not, this control is + # ignored. Not valid before the call is accepted. + 'RESUME': 0, + # Put a call on hold. Not valid before the call is accepted. + 'PAUSE': 1, + # Reject a call if it was not answered, yet. Cancel a call after it was answered. + 'CANCEL': 2, + # Request that the friend stops sending audio. Regardless of the friend's compliance, this will cause the + # audio_receive_frame event to stop being triggered on receiving an audio frame from the friend. + 'MUTE_AUDIO': 3, + # Calling this control will notify client to start sending audio again. + 'UNMUTE_AUDIO': 4, + # Request that the friend stops sending video. Regardless of the friend's compliance, this will cause the + # video_receive_frame event to stop being triggered on receiving a video frame from the friend. + 'HIDE_VIDEO': 5, + # Calling this control will notify client to start sending video again. + 'SHOW_VIDEO': 6, +} + +TOXAV_ERR_CALL_CONTROL = { + # The function returned successfully. + 'OK': 0, + # Synchronization error occurred. + 'SYNC': 1, + # The friend_number passed did not designate a valid friend. + 'FRIEND_NOT_FOUND': 2, + # This client is currently not in a call with the friend. Before the call is answered, only CANCEL is a valid + # control. + 'FRIEND_NOT_IN_CALL': 3, + # Happens if user tried to pause an already paused call or if trying to resume a call that is not paused. + 'INVALID_TRANSITION': 4, +} + +TOXAV_ERR_BIT_RATE_SET = { + # The function returned successfully. + 'OK': 0, + # Synchronization error occurred. + 'SYNC': 1, + # The audio bit rate passed was not one of the supported values. + 'INVALID_AUDIO_BIT_RATE': 2, + # The video bit rate passed was not one of the supported values. + 'INVALID_VIDEO_BIT_RATE': 3, + # The friend_number passed did not designate a valid friend. + 'FRIEND_NOT_FOUND': 4, + # This client is currently not in a call with the friend. + 'FRIEND_NOT_IN_CALL': 5, +} + +TOXAV_ERR_SEND_FRAME = { + # The function returned successfully. + 'OK': 0, + # In case of video, one of Y, U, or V was NULL. In case of audio, the samples data pointer was NULL. + 'NULL': 1, + # The friend_number passed did not designate a valid friend. + 'FRIEND_NOT_FOUND': 2, + # This client is currently not in a call with the friend. + 'FRIEND_NOT_IN_CALL': 3, + # Synchronization error occurred. + 'SYNC': 4, + # One of the frame parameters was invalid. E.g. the resolution may be too small or too large, or the audio sampling + # rate may be unsupported. + 'INVALID': 5, + # Either friend turned off audio or video receiving or we turned off sending for the said payload. + 'PAYLOAD_TYPE_DISABLED': 6, + # Failed to push frame through rtp interface. + 'RTP_FAILED': 7, +} diff --git a/src/toxygen_wrapper/toxcore_enums_and_consts.py b/src/toxygen_wrapper/toxcore_enums_and_consts.py new file mode 100644 index 0000000..ed53861 --- /dev/null +++ b/src/toxygen_wrapper/toxcore_enums_and_consts.py @@ -0,0 +1,982 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- + +TOX_USER_STATUS = { + 'NONE': 0, + 'AWAY': 1, + 'BUSY': 2, +} + +TOX_MESSAGE_TYPE = { + 'NORMAL': 0, + 'ACTION': 1, +} + +TOX_PROXY_TYPE = { + 'NONE': 0, + 'HTTP': 1, + 'SOCKS5': 2, +} + +TOX_SAVEDATA_TYPE = { + 'NONE': 0, + 'TOX_SAVE': 1, + 'SECRET_KEY': 2, +} + +TOX_ERR_OPTIONS_NEW = { + 'OK': 0, + 'MALLOC': 1, +} + +TOX_ERR_NEW = { + 'OK': 0, + 'NULL': 1, + 'MALLOC': 2, + 'PORT_ALLOC': 3, + 'PROXY_BAD_TYPE': 4, + 'PROXY_BAD_HOST': 5, + 'PROXY_BAD_PORT': 6, + 'PROXY_NOT_FOUND': 7, + 'LOAD_ENCRYPTED': 8, + 'LOAD_BAD_FORMAT': 9, + 'TCP_SERVER_ALLOC': 10, +} + +TOX_ERR_BOOTSTRAP = { + 'OK': 0, + 'NULL': 1, + 'BAD_HOST': 2, + 'BAD_PORT': 3, +} + +TOX_CONNECTION = { + 'NONE': 0, + 'TCP': 1, + 'UDP': 2, +} + +TOX_ERR_SET_INFO = { + 'OK': 0, + 'NULL': 1, + 'TOO_LONG': 2, + # The function returned successfully. + 'TOX_ERR_SET_INFO_OK': 0, + # One of the arguments to the function was NULL when it was not expected. + 'TOX_ERR_SET_INFO_NULL': 1, + # Information length exceeded maximum permissible size. + 'TOX_ERR_SET_INFO_TOO_LONG': 2, +} + + +TOX_ERR_FRIEND_ADD = { + 'OK': 0, + 'NULL': 1, + 'TOO_LONG': 2, + 'NO_MESSAGE': 3, + 'OWN_KEY': 4, + 'ALREADY_SENT': 5, + 'BAD_CHECKSUM': 6, + 'SET_NEW_NOSPAM': 7, + 'MALLOC': 8, +} + +TOX_ERR_FRIEND_DELETE = { + 'OK': 0, + 'FRIEND_NOT_FOUND': 1, +} + +TOX_ERR_FRIEND_BY_PUBLIC_KEY = { + 'OK': 0, + 'NULL': 1, + 'NOT_FOUND': 2, +} + +TOX_ERR_FRIEND_GET_PUBLIC_KEY = { + 'OK': 0, + 'FRIEND_NOT_FOUND': 1, +} + +TOX_ERR_FRIEND_GET_LAST_ONLINE = { + 'OK': 0, + 'FRIEND_NOT_FOUND': 1, +} + +TOX_ERR_FRIEND_QUERY = { + 'OK': 0, + 'NULL': 1, + 'FRIEND_NOT_FOUND': 2, +} + +TOX_ERR_SET_TYPING = { + 'OK': 0, + 'FRIEND_NOT_FOUND': 1, +} + +TOX_ERR_FRIEND_SEND_MESSAGE = { + 'OK': 0, + 'NULL': 1, + 'FRIEND_NOT_FOUND': 2, + 'FRIEND_NOT_CONNECTED': 3, + 'SENDQ': 4, + 'TOO_LONG': 5, + 'EMPTY': 6, +} + +TOX_FILE_KIND = { + 'DATA': 0, + 'AVATAR': 1, +} + +TOX_FILE_CONTROL = { + 'RESUME': 0, + 'PAUSE': 1, + 'CANCEL': 2, +} + +TOX_ERR_FILE_CONTROL = { + 'OK': 0, + 'FRIEND_NOT_FOUND': 1, + 'FRIEND_NOT_CONNECTED': 2, + 'NOT_FOUND': 3, + 'NOT_PAUSED': 4, + 'DENIED': 5, + 'ALREADY_PAUSED': 6, + 'SENDQ': 7, +} + +TOX_ERR_FILE_SEEK = { + 'OK': 0, + 'FRIEND_NOT_FOUND': 1, + 'FRIEND_NOT_CONNECTED': 2, + 'NOT_FOUND': 3, + 'DENIED': 4, + 'INVALID_POSITION': 5, + 'SENDQ': 6, +} + +TOX_ERR_FILE_GET = { + 'OK': 0, + 'NULL': 1, + 'FRIEND_NOT_FOUND': 2, + 'NOT_FOUND': 3, +} + +TOX_ERR_FILE_SEND = { + 'OK': 0, + 'NULL': 1, + 'FRIEND_NOT_FOUND': 2, + 'FRIEND_NOT_CONNECTED': 3, + 'NAME_TOO_LONG': 4, + 'TOO_MANY': 5, +} + +TOX_ERR_FILE_SEND_CHUNK = { + 'OK': 0, + 'NULL': 1, + 'FRIEND_NOT_FOUND': 2, + 'FRIEND_NOT_CONNECTED': 3, + 'NOT_FOUND': 4, + 'NOT_TRANSFERRING': 5, + 'INVALID_LENGTH': 6, + 'SENDQ': 7, + 'WRONG_POSITION': 8, +} + +TOX_ERR_FRIEND_CUSTOM_PACKET = { + 'OK': 0, + 'NULL': 1, + 'FRIEND_NOT_FOUND': 2, + 'FRIEND_NOT_CONNECTED': 3, + 'INVALID': 4, + 'EMPTY': 5, + 'TOO_LONG': 6, + 'SENDQ': 7, +} + +TOX_ERR_GET_PORT = { + 'OK': 0, + 'NOT_BOUND': 1, +} + +TOX_GROUP_PRIVACY_STATE = { + + # + # The group is considered to be public. Anyone may join the group using the Chat ID. + # + # If the group is in this state, even if the Chat ID is never explicitly shared + # with someone outside of the group, information including the Chat ID, IP addresses, + # and peer ID's (but not Tox ID's) is visible to anyone with access to a node + # storing a DHT entry for the given group. + # + 'PUBLIC': 0, + + # + # The group is considered to be private. The only way to join the group is by having + # someone in your contact list send you an invite. + # + # If the group is in this state, no group information (mentioned above) is present in the DHT; + # the DHT is not used for any purpose at all. If a public group is set to private, + # all DHT information related to the group will expire shortly. + # + 'PRIVATE': 1 +} + +TOX_GROUP_ROLE = { + + # + # May kick and ban all other peers as well as set their role to anything (except founder). + # Founders may also set the group password, toggle the privacy state, and set the peer limit. + # + 'FOUNDER': 0, + + # + # May kick, ban and set the user and observer roles for peers below this role. + # May also set the group topic. + # + 'MODERATOR': 1, + + # + # May communicate with other peers normally. + # + 'USER': 2, + + # + # May observe the group and ignore peers; may not communicate with other peers or with the group. + # + 'OBSERVER': 3 +} + +TOX_ERR_GROUP_NEW = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_NEW_OK': 0, + + # + # The group name exceeded TOX_GROUP_MAX_GROUP_NAME_LENGTH. + # + 'TOX_ERR_GROUP_NEW_TOO_LONG': 1, + + # + # group_name is NULL or length is zero. + # + 'TOX_ERR_GROUP_NEW_EMPTY': 2, + + # + # TOX_GROUP_PRIVACY_STATE is an invalid type. + # + 'TOX_ERR_GROUP_NEW_PRIVACY': 3, + + # + # The group instance failed to initialize. + # + 'TOX_ERR_GROUP_NEW_INIT': 4, + + # + # The group state failed to initialize. This usually indicates that something went wrong + # related to cryptographic signing. + # + 'TOX_ERR_GROUP_NEW_STATE': 5, + + # + # The group failed to announce to the DHT. This indicates a network related error. + # + 'TOX_ERR_GROUP_NEW_ANNOUNCE': 6, +} + +TOX_ERR_GROUP_JOIN = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_JOIN_OK': 0, + + # + # The group instance failed to initialize. + # + 'TOX_ERR_GROUP_JOIN_INIT': 1, + + # + # The chat_id pointer is set to NULL or a group with chat_id already exists. This usually + # happens if the client attempts to create multiple sessions for the same group. + # + 'TOX_ERR_GROUP_JOIN_BAD_CHAT_ID': 2, + + # + # Password length exceeded TOX_GROUP_MAX_PASSWORD_SIZE. + # + 'TOX_ERR_GROUP_JOIN_TOO_LONG': 3, +} + +TOX_ERR_GROUP_IS_CONNECTED = { + 'TOX_ERR_GROUP_IS_CONNECTED_OK': 0, + 'TOX_ERR_GROUP_IS_CONNECTED_GROUP_NOT_FOUND': 1 +} + +TOX_ERR_GROUP_RECONNECT = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_RECONNECT_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_RECONNECT_GROUP_NOT_FOUND': 1, +} + +TOX_ERR_GROUP_DISCONNECT = { + + # The function returned successfully. + 'TOX_ERR_GROUP_DISCONNECT_OK': 0, + + # The group number passed did not designate a valid group. + 'TOX_ERR_GROUP_DISCONNECT_GROUP_NOT_FOUND': 1, + + # The group is already disconnected. + 'TOX_ERR_GROUP_DISCONNECT_ALREADY_DISCONNECTED': 2, +} + + +TOX_ERR_GROUP_LEAVE = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_LEAVE_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_LEAVE_GROUP_NOT_FOUND': 1, + + # + # Message length exceeded 'TOX_GROUP_MAX_PART_LENGTH. + # + 'TOX_ERR_GROUP_LEAVE_TOO_LONG': 2, + + # + # The parting packet failed to send. + # + 'TOX_ERR_GROUP_LEAVE_FAIL_SEND': 3, + + # + # The group chat instance failed to be deleted. This may occur due to memory related errors. + # + 'TOX_ERR_GROUP_LEAVE_DELETE_FAIL': 4, +} + +TOX_ERR_GROUP_SELF_QUERY = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_SELF_QUERY_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_SELF_QUERY_GROUP_NOT_FOUND': 1, +} + + +TOX_ERR_GROUP_SELF_NAME_SET = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_SELF_NAME_SET_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_SELF_NAME_SET_GROUP_NOT_FOUND': 1, + + # + # Name length exceeded 'TOX_MAX_NAME_LENGTH. + # + 'TOX_ERR_GROUP_SELF_NAME_SET_TOO_LONG': 2, + + # + # The length given to the set function is zero or name is a NULL pointer. + # + 'TOX_ERR_GROUP_SELF_NAME_SET_INVALID': 3, + + # + # The name is already taken by another peer in the group. + # + 'TOX_ERR_GROUP_SELF_NAME_SET_TAKEN': 4, + + # + # The packet failed to send. + # + 'TOX_ERR_GROUP_SELF_NAME_SET_FAIL_SEND': 5 +} + +TOX_ERR_GROUP_SELF_STATUS_SET = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_SELF_STATUS_SET_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_SELF_STATUS_SET_GROUP_NOT_FOUND': 1, + + # + # An invalid type was passed to the set function. + # + 'TOX_ERR_GROUP_SELF_STATUS_SET_INVALID': 2, + + # + # The packet failed to send. + # + 'TOX_ERR_GROUP_SELF_STATUS_SET_FAIL_SEND': 3 +} + +TOX_ERR_GROUP_PEER_QUERY = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_PEER_QUERY_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_PEER_QUERY_GROUP_NOT_FOUND': 1, + + # + # The ID passed did not designate a valid peer. + # + 'TOX_ERR_GROUP_PEER_QUERY_PEER_NOT_FOUND': 2 +} + +TOX_ERR_GROUP_STATE_QUERIES = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_STATE_QUERIES_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_STATE_QUERIES_GROUP_NOT_FOUND': 1 +} + + +TOX_ERR_GROUP_TOPIC_SET = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_TOPIC_SET_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_TOPIC_SET_GROUP_NOT_FOUND': 1, + + # + # Topic length exceeded 'TOX_GROUP_MAX_TOPIC_LENGTH. + # + 'TOX_ERR_GROUP_TOPIC_SET_TOO_LONG': 2, + + # + # The caller does not have the required permissions to set the topic. + # + 'TOX_ERR_GROUP_TOPIC_SET_PERMISSIONS': 3, + + # + # The packet could not be created. This error is usually related to cryptographic signing. + # + 'TOX_ERR_GROUP_TOPIC_SET_FAIL_CREATE': 4, + + # + # The packet failed to send. + # + 'TOX_ERR_GROUP_TOPIC_SET_FAIL_SEND': 5 +} + +TOX_ERR_GROUP_SEND_MESSAGE = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_SEND_MESSAGE_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_SEND_MESSAGE_GROUP_NOT_FOUND': 1, + + # + # Message length exceeded 'TOX_MAX_MESSAGE_LENGTH. + # + 'TOX_ERR_GROUP_SEND_MESSAGE_TOO_LONG': 2, + + # + # The message pointer is null or length is zero. + # + 'TOX_ERR_GROUP_SEND_MESSAGE_EMPTY': 3, + + # + # The message type is invalid. + # + 'TOX_ERR_GROUP_SEND_MESSAGE_BAD_TYPE': 4, + + # + # The caller does not have the required permissions to send group messages. + # + 'TOX_ERR_GROUP_SEND_MESSAGE_PERMISSIONS': 5, + + # + # Packet failed to send. + # + 'TOX_ERR_GROUP_SEND_MESSAGE_FAIL_SEND': 6 +} + +TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_GROUP_NOT_FOUND': 1, + + # + # The ID passed did not designate a valid peer. + # + 'TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_PEER_NOT_FOUND': 2, + + # + # Message length exceeded 'TOX_MAX_MESSAGE_LENGTH. + # + 'TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_TOO_LONG': 3, + + # + # The message pointer is null or length is zero. + # + 'TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_EMPTY': 4, + + # + # The caller does not have the required permissions to send group messages. + # + 'TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_PERMISSIONS': 5, + + # + # Packet failed to send. + # + 'TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_FAIL_SEND': 6 +} + +TOX_ERR_GROUP_SEND_CUSTOM_PACKET = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_SEND_CUSTOM_PACKET_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_SEND_CUSTOM_PACKET_GROUP_NOT_FOUND': 1, + + # + # Message length exceeded 'TOX_MAX_MESSAGE_LENGTH. + # + 'TOX_ERR_GROUP_SEND_CUSTOM_PACKET_TOO_LONG': 2, + + # + # The message pointer is null or length is zero. + # + 'TOX_ERR_GROUP_SEND_CUSTOM_PACKET_EMPTY': 3, + + # + # The caller does not have the required permissions to send group messages. + # + 'TOX_ERR_GROUP_SEND_CUSTOM_PACKET_PERMISSIONS': 4 +} + +TOX_ERR_GROUP_INVITE_FRIEND = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_INVITE_FRIEND_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_INVITE_FRIEND_GROUP_NOT_FOUND': 1, + + # + # The friend number passed did not designate a valid friend. + # + 'TOX_ERR_GROUP_INVITE_FRIEND_FRIEND_NOT_FOUND': 2, + + # + # Creation of the invite packet failed. This indicates a network related error. + # + 'TOX_ERR_GROUP_INVITE_FRIEND_INVITE_FAIL': 3, + + # + # Packet failed to send. + # + 'TOX_ERR_GROUP_INVITE_FRIEND_FAIL_SEND': 4 +} + +TOX_ERR_GROUP_INVITE_ACCEPT = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_INVITE_ACCEPT_OK': 0, + + # + # The invite data is not in the expected format. + # + 'TOX_ERR_GROUP_INVITE_ACCEPT_BAD_INVITE': 1, + + # + # The group instance failed to initialize. + # + 'TOX_ERR_GROUP_INVITE_ACCEPT_INIT_FAILED': 2, + + # + # Password length exceeded 'TOX_GROUP_MAX_PASSWORD_SIZE. + # + 'TOX_ERR_GROUP_INVITE_ACCEPT_TOO_LONG': 3 +} + +TOX_GROUP_JOIN_FAIL = { + + # + # You are using the same nickname as someone who is already in the group. + # + 'TOX_GROUP_JOIN_FAIL_NAME_TAKEN': 0, + + # + # The group peer limit has been reached. + # + 'TOX_GROUP_JOIN_FAIL_PEER_LIMIT': 1, + + # + # You have supplied an invalid password. + # + 'TOX_GROUP_JOIN_FAIL_INVALID_PASSWORD': 2, + + # + # The join attempt failed due to an unspecified error. This often occurs when the group is + # not found in the DHT. + # + 'TOX_GROUP_JOIN_FAIL_UNKNOWN': 3 +} + +TOX_ERR_GROUP_FOUNDER_SET_PASSWORD = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PASSWORD_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PASSWORD_GROUP_NOT_FOUND': 1, + + # + # The caller does not have the required permissions to set the password. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PASSWORD_PERMISSIONS': 2, + + # + # Password length exceeded 'TOX_GROUP_MAX_PASSWORD_SIZE. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PASSWORD_TOO_LONG': 3, + + # + # The packet failed to send. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PASSWORD_FAIL_SEND': 4 +} + +TOX_ERR_GROUP_FOUNDER_SET_PRIVACY_STATE = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PRIVACY_STATE_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PRIVACY_STATE_GROUP_NOT_FOUND': 1, + + # + # 'TOX_GROUP_PRIVACY_STATE is an invalid type. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PRIVACY_STATE_INVALID': 2, + + # + # The caller does not have the required permissions to set the privacy state. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PRIVACY_STATE_PERMISSIONS': 3, + + # + # The privacy state could not be set. This may occur due to an error related to + # cryptographic signing of the new shared state. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PRIVACY_STATE_FAIL_SET': 4, + + # + # The packet failed to send. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PRIVACY_STATE_FAIL_SEND': 5 +} + +TOX_ERR_GROUP_FOUNDER_SET_PEER_LIMIT = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PEER_LIMIT_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PEER_LIMIT_GROUP_NOT_FOUND': 1, + + # + # The caller does not have the required permissions to set the peer limit. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PEER_LIMIT_PERMISSIONS': 2, + + # + # The peer limit could not be set. This may occur due to an error related to + # cryptographic signing of the new shared state. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PEER_LIMIT_FAIL_SET': 3, + + # + # The packet failed to send. + # + 'TOX_ERR_GROUP_FOUNDER_SET_PEER_LIMIT_FAIL_SEND': 4 +} + +TOX_ERR_GROUP_TOGGLE_IGNORE = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_TOGGLE_IGNORE_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_TOGGLE_IGNORE_GROUP_NOT_FOUND': 1, + + # + # The ID passed did not designate a valid peer. + # + 'TOX_ERR_GROUP_TOGGLE_IGNORE_PEER_NOT_FOUND': 2 +} + +TOX_ERR_GROUP_MOD_SET_ROLE = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_MOD_SET_ROLE_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_MOD_SET_ROLE_GROUP_NOT_FOUND': 1, + + # + # The ID passed did not designate a valid peer. Note: you cannot set your own role. + # + 'TOX_ERR_GROUP_MOD_SET_ROLE_PEER_NOT_FOUND': 2, + + # + # The caller does not have the required permissions for this action. + # + 'TOX_ERR_GROUP_MOD_SET_ROLE_PERMISSIONS': 3, + + # + # The role assignment is invalid. This will occur if you try to set a peer's role to + # the role they already have. + # + 'TOX_ERR_GROUP_MOD_SET_ROLE_ASSIGNMENT': 4, + + # + # The role was not successfully set. This may occur if something goes wrong with role setting': , + # or if the packet fails to send. + # + 'TOX_ERR_GROUP_MOD_SET_ROLE_FAIL_ACTION': 5 +} + +TOX_ERR_GROUP_MOD_REMOVE_PEER = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_MOD_REMOVE_PEER_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_MOD_REMOVE_PEER_GROUP_NOT_FOUND': 1, + + # + # The ID passed did not designate a valid peer. + # + 'TOX_ERR_GROUP_MOD_REMOVE_PEER_PEER_NOT_FOUND': 2, + + # + # The caller does not have the required permissions for this action. + # + 'TOX_ERR_GROUP_MOD_REMOVE_PEER_PERMISSIONS': 3, + + # + # The peer could not be removed from the group. + # + # If a ban was set': , this error indicates that the ban entry could not be created. + # This is usually due to the peer's IP address already occurring in the ban list. It may also + # be due to the entry containing invalid peer information': , or a failure to cryptographically + # authenticate the entry. + # + 'TOX_ERR_GROUP_MOD_REMOVE_PEER_FAIL_ACTION': 4, + + # + # The packet failed to send. + # + 'TOX_ERR_GROUP_MOD_REMOVE_PEER_FAIL_SEND': 5 +} + +TOX_ERR_GROUP_MOD_REMOVE_BAN = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_MOD_REMOVE_BAN_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_MOD_REMOVE_BAN_GROUP_NOT_FOUND': 1, + + # + # The caller does not have the required permissions for this action. + # + 'TOX_ERR_GROUP_MOD_REMOVE_BAN_PERMISSIONS': 2, + + # + # The ban entry could not be removed. This may occur if ban_id does not designate + # a valid ban entry. + # + 'TOX_ERR_GROUP_MOD_REMOVE_BAN_FAIL_ACTION': 3, + + # + # The packet failed to send. + # + 'TOX_ERR_GROUP_MOD_REMOVE_BAN_FAIL_SEND': 4 +} + +TOX_GROUP_MOD_EVENT = { + + # + # A peer has been kicked from the group. + # + 'KICK': 0, + + # + # A peer has been banned from the group. + # + 'BAN': 1, + + # + # A peer as been given the observer role. + # + 'OBSERVER': 2, + + # + # A peer has been given the user role. + # + 'USER': 3, + + # + # A peer has been given the moderator role. + # + 'MODERATOR': 4, +} + +TOX_ERR_GROUP_BAN_QUERY = { + + # + # The function returned successfully. + # + 'TOX_ERR_GROUP_BAN_QUERY_OK': 0, + + # + # The group number passed did not designate a valid group. + # + 'TOX_ERR_GROUP_BAN_QUERY_GROUP_NOT_FOUND': 1, + + # + # The ban_id does not designate a valid ban list entry. + # + 'TOX_ERR_GROUP_BAN_QUERY_BAD_ID': 2, +} + + +TOX_GROUP_BAN_TYPE = { + + 'IP_PORT': 0, + + 'PUBLIC_KEY': 1, + + 'NICK': 2 +} + +TOX_PUBLIC_KEY_SIZE = 32 + +TOX_ADDRESS_SIZE = TOX_PUBLIC_KEY_SIZE + 6 + +TOX_MAX_FRIEND_REQUEST_LENGTH = 1016 + +TOX_MAX_MESSAGE_LENGTH = 1372 + +TOX_GROUP_MAX_TOPIC_LENGTH = 512 + +TOX_GROUP_MAX_PART_LENGTH = 128 + +TOX_GROUP_MAX_GROUP_NAME_LENGTH = 48 + +TOX_GROUP_MAX_PASSWORD_SIZE = 32 + +TOX_GROUP_CHAT_ID_SIZE = 32 + +TOX_GROUP_PEER_PUBLIC_KEY_SIZE = 32 + +TOX_MAX_NAME_LENGTH = 128 + +TOX_MAX_STATUS_MESSAGE_LENGTH = 1007 + +TOX_SECRET_KEY_SIZE = 32 + +TOX_FILE_ID_LENGTH = 32 + +TOX_HASH_LENGTH = 32 + +TOX_MAX_CUSTOM_PACKET_SIZE = 1373 diff --git a/src/toxygen_wrapper/toxencryptsave.py b/src/toxygen_wrapper/toxencryptsave.py new file mode 100644 index 0000000..646d7bb --- /dev/null +++ b/src/toxygen_wrapper/toxencryptsave.py @@ -0,0 +1,91 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- + +try: + from tox_wrapper import libtox + import tox_wrapper.toxencryptsave_enums_and_consts as enum +except: + import libtox + import toxencryptsave_enums_and_consts as enum + +from typing import Union, Callable +from ctypes import (ArgumentError, byref, c_bool, c_char_p, c_int, c_size_t, + create_string_buffer, Array) +def ToxError(ArgumentError): pass + +class ToxEncryptSave: + + def __init__(self): + self.libtoxencryptsave = libtox.LibToxEncryptSave() + + def is_data_encrypted(self, data: bytes) -> bool: + """ + Checks if given data is encrypted + """ + func = self.libtoxencryptsave.tox_is_data_encrypted + func.restype = c_bool + result = func(c_char_p(bytes(data))) + return bool(result) + + def pass_encrypt(self, data: bytes, password: Union[str,bytes]) -> bytes: + """ + Encrypts the given data with the given password. + + :return: output array + """ + out = create_string_buffer(len(data) + enum.TOX_PASS_ENCRYPTION_EXTRA_LENGTH) + tox_err_encryption = c_int() + assert password + if type(password) != bytes: + password = bytes(password, 'utf-8') + self.libtoxencryptsave.tox_pass_encrypt(c_char_p(data), + c_size_t(len(data)), + c_char_p(password), + c_size_t(len(password)), + out, + byref(tox_err_encryption)) + tox_err_encryption = tox_err_encryption.value + if tox_err_encryption == enum.TOX_ERR_ENCRYPTION['OK']: + return bytes(out[:]) + if tox_err_encryption == enum.TOX_ERR_ENCRYPTION['NULL']: + raise ArgumentError('Some input data, or maybe the output pointer, was null.') + if tox_err_encryption == enum.TOX_ERR_ENCRYPTION['KEY_DERIVATION_FAILED']: + raise RuntimeError('The crypto lib was unable to derive a key from the given passphrase, which is usually a' + ' lack of memory issue. The functions accepting keys do not produce this error.') + if tox_err_encryption == enum.TOX_ERR_ENCRYPTION['FAILED']: + raise RuntimeError('The encryption itself failed.') + raise ToxError('The function did not return OK.') + + def pass_decrypt(self, data: bytes, password: Union[str,bytes]) -> bytes: + """ + Decrypts the given data with the given password. + + :return: output array + """ + out = create_string_buffer(len(data) - enum.TOX_PASS_ENCRYPTION_EXTRA_LENGTH) + tox_err_decryption = c_int() + assert password + if type(password) != bytes: + password = bytes(password, 'utf-8') + self.libtoxencryptsave.tox_pass_decrypt(c_char_p(bytes(data)), + c_size_t(len(data)), + c_char_p(password), + c_size_t(len(password)), + out, + byref(tox_err_decryption)) + tox_err_decryption = tox_err_decryption.value + if tox_err_decryption == enum.TOX_ERR_DECRYPTION['OK']: + return bytes(out[:]) + if tox_err_decryption == enum.TOX_ERR_DECRYPTION['NULL']: + raise ArgumentError('Some input data, or maybe the output pointer, was null.') + if tox_err_decryption == enum.TOX_ERR_DECRYPTION['INVALID_LENGTH']: + raise ArgumentError('The input data was shorter than TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes') + if tox_err_decryption == enum.TOX_ERR_DECRYPTION['BAD_FORMAT']: + raise ArgumentError('The input data is missing the magic number (i.e. wasn\'t created by this module, or is' + ' corrupted)') + if tox_err_decryption == enum.TOX_ERR_DECRYPTION['KEY_DERIVATION_FAILED']: + raise RuntimeError('The crypto lib was unable to derive a key from the given passphrase, which is usually a' + ' lack of memory issue. The functions accepting keys do not produce this error.') + if tox_err_decryption == enum.TOX_ERR_DECRYPTION['FAILED']: + raise RuntimeError('The encrypted byte array could not be decrypted. Either the data was corrupt or the ' + 'password/key was incorrect.') + raise ToxError('The function did not return OK.') diff --git a/src/toxygen_wrapper/toxencryptsave_enums_and_consts.py b/src/toxygen_wrapper/toxencryptsave_enums_and_consts.py new file mode 100644 index 0000000..cf795f8 --- /dev/null +++ b/src/toxygen_wrapper/toxencryptsave_enums_and_consts.py @@ -0,0 +1,29 @@ +TOX_ERR_ENCRYPTION = { + # The function returned successfully. + 'OK': 0, + # Some input data, or maybe the output pointer, was null. + 'NULL': 1, + # The crypto lib was unable to derive a key from the given passphrase, which is usually a lack of memory issue. The + # functions accepting keys do not produce this error. + 'KEY_DERIVATION_FAILED': 2, + # The encryption itself failed. + 'FAILED': 3 +} + +TOX_ERR_DECRYPTION = { + # The function returned successfully. + 'OK': 0, + # Some input data, or maybe the output pointer, was null. + 'NULL': 1, + # The input data was shorter than TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes + 'INVALID_LENGTH': 2, + # The input data is missing the magic number (i.e. wasn't created by this module, or is corrupted) + 'BAD_FORMAT': 3, + # The crypto lib was unable to derive a key from the given passphrase, which is usually a lack of memory issue. The + # functions accepting keys do not produce this error. + 'KEY_DERIVATION_FAILED': 4, + # The encrypted byte array could not be decrypted. Either the data was corrupt or the password/key was incorrect. + 'FAILED': 5, +} + +TOX_PASS_ENCRYPTION_EXTRA_LENGTH = 80 diff --git a/src/toxygen_wrapper/toxygen_echo.py b/src/toxygen_wrapper/toxygen_echo.py new file mode 100644 index 0000000..d43fc6e --- /dev/null +++ b/src/toxygen_wrapper/toxygen_echo.py @@ -0,0 +1,458 @@ +#!/var/local/bin/python3.bash +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- + +# A work in progress - chat works, but I don't think AV does. + +""" echo.py a basic Tox echo service. Features: + - accept friend request + - echo back friend message +# - accept and answer friend call request +# - send back friend audio/video data +# - send back files friend sent +""" + +import sys +import os +import traceback +import threading +import random +from ctypes import * +import time +from typing import Union, Callable + +# LOG=util.log +global LOG +import logging +# log = lambda x: LOG.info(x) +LOG = logging.getLogger('app') +def LOG_error(a): print('EROR_ '+a) +def LOG_warn(a): print('WARN_ '+a) +def LOG_info(a): print('INFO_ '+a) +def LOG_debug(a): print('DBUG_ '+a) +def LOG_trace(a): pass # print('TRAC_ '+a) + +from tox_wrapper import tox +import tox_wrapper.toxcore_enums_and_consts as enums +from tox_wrapper.tox import Tox, UINT32_MAX +from tox_wrapper.toxcore_enums_and_consts import TOX_CONNECTION, TOX_USER_STATUS, \ + TOX_MESSAGE_TYPE, TOX_PUBLIC_KEY_SIZE, TOX_FILE_CONTROL, TOX_FILE_KIND + +import tox_wrapper.tests.support_testing as ts +from tox_wrapper.tests.support_testing import oMainArgparser + +def sleep(fSec) -> None: + if 'QtCore' in globals(): + if fSec > .000001: QtCore.QThread.msleep(fSec) + QtCore.QCoreApplication.processEvents() + else: + time.sleep(fSec) + +try: + import coloredlogs + if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ: + os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red' +except ImportError as e: + # logging.log(logging.DEBUG, f"coloredlogs not available: {e}") + coloredlogs = None + +if 'USER' in os.environ: + sDATA_FILE = '/tmp/logging_toxygen_' +os.environ['USER'] +'.tox' +elif 'USERNAME' in os.environ: + sDATA_FILE = '/tmp/logging_toxygen_' +os.environ['USERNAME'] +'.tox' +else: + sDATA_FILE = '/tmp/logging_toxygen_' +'data' +'.tox' + +bHAVE_AV = False +iDHT_TRIES = 100 +iDHT_TRY = 0 + +#?SERVER = lLOCAL[-1] + +if not bHAVE_AV: + class AV(): pass +else: + class AV(tox.ToxAV): + def __init__(self, core): + super(AV, self).__init__(core) + self.core = self.get_tox() + + def on_call(self, fid:int, audio_enabled:bool, video_enabled:bool) -> None: + LOG.info("Incoming %s call from %d:%s ..." % ( + "video" if video_enabled else "audio", + fid, + self.core.friend_get_name(fid))) + bret = self.answer(fid, 48, 64) + LOG.info(f"Answered, in call... {bret}") + + def on_call_state(self, fid:int, state:int) -> None: + LOG.info('call state:fn=%d, state=%d' % (fid, state)) + + def on_audio_bit_rate(self, fid:int, audio_bit_rate:int) -> None: + LOG.info('audio bit rate status: fn=%d, abr=%d' % + (fid, audio_bit_rate)) + + def on_video_bit_rate(self, fid:int, video_bit_rate:int) -> None: + LOG.info('video bit rate status: fn=%d, vbr=%d' % + (fid, video_bit_rate)) + + def on_audio_receive_frame(self, fid:int, + pcm:int, + sample_count:int, + channels:int, + sampling_rate:int) -> None: + # LOG.info('audio frame: %d, %d, %d, %d' % + # (fid, sample_count, channels, sampling_rate)) + # LOG.info('pcm len:%d, %s' % (len(pcm), str(type(pcm)))) + sys.stdout.write('.') + sys.stdout.flush() + bret = self.audio_send_frame(fid, pcm, sample_count, + channels, sampling_rate) + if bret is False: + LOG.error('on_audio_receive_frame error.') + + def on_video_receive_frame(self, fid:int, width:int, height:int, frame, u, v) -> None: + LOG.info('video frame: %d, %d, %d, ' % (fid, width, height)) + sys.stdout.write('*') + sys.stdout.flush() + bret = self.video_send_frame(fid, width, height, frame, u, v) + if bret is False: + LOG.error('on_video_receive_frame error.') + + def witerate(self) -> None: + self.iterate() + + +def save_to_file(tox, fname: str) -> None: + data = tox.get_savedata() + with open(fname, 'wb') as f: + f.write(data) + +def load_from_file(fname: str) -> bytes: + assert os.path.exists(fname) + return open(fname, 'rb').read() + +class EchoBot(): + def __init__(self, oTox): + self._tox = oTox + self._tox.self_set_name("PyEchoBot") + LOG.info(f'ID: {self._tox.self_get_address()}') + + self.files = {} + self.av = None + self.on_connection_status = None + + def start(self) -> None: + self.connect() + if bHAVE_AV: + # RuntimeError: Attempted to create a second session for the same Tox instance. + + self.av = True # AV(self._tox_pointer) + def bobs_on_friend_request(iTox, + public_key, + message_data, + message_data_size, + *largs) -> None: + key = ''.join(chr(x) for x in public_key[:TOX_PUBLIC_KEY_SIZE]) + sPk = tox.bin_to_string(key, TOX_PUBLIC_KEY_SIZE) + sMd = str(message_data, 'UTF-8') + LOG.debug('on_friend_request ' +sPk +' ' +sMd) + self.on_friend_request(sPk, sMd) + LOG.info('setting bobs_on_friend_request') + self._tox.callback_friend_request(bobs_on_friend_request) + + def bobs_on_friend_message(iTox, + iFriendNum, + iMessageType, + message_data, + message_data_size, + *largs) -> None: + sMd = str(message_data, 'UTF-8') + LOG_debug(f"on_friend_message {iFriendNum}" +' ' +sMd) + self.on_friend_message(iFriendNum, iMessageType, sMd) + LOG.info('setting bobs_on_friend_message') + self._tox.callback_friend_message(bobs_on_friend_message) + + def bobs_on_file_chunk_request(iTox, fid, filenumber, position, length, *largs) -> None: + if length == 0: + return + + data = self.files[(fid, filenumber)]['f'][position:(position + length)] + self._tox.file_send_chunk(fid, filenumber, position, data) + self._tox.callback_file_chunk_request(bobs_on_file_chunk_request) + + def bobs_on_file_recv(iTox, fid, filenumber, kind, size, filename, *largs): + LOG_info(f"on_file_recv {fid} {filenumber} {kind} {size} {filename}") + if size == 0: + return + self.files[(fid, filenumber)] = { + 'f': bytes(), + 'filename': filename, + 'size': size + } + self._tox.file_control(fid, filenumber, TOX_FILE_CONTROL['RESUME']) + + + def connect(self) -> None: + if not self.on_connection_status: + def on_connection_status(iTox, iCon, *largs) -> None: + LOG_info('ON_CONNECTION_STATUS - CONNECTED ' + repr(iCon)) + self._tox.callback_self_connection_status(on_connection_status) + LOG.info('setting on_connection_status callback ') + self.on_connection_status = on_connection_status + if self._oargs.network in ['newlocal', 'local']: + LOG.info('connecting on the new network ') + sNet = 'newlocal' + elif self._oargs.network == 'new': + LOG.info('connecting on the new network ') + sNet = 'new' + else: # main old + LOG.info('connecting on the old network ') + sNet = 'old' + sFile = self._oargs.nodes_json + lNodes = ts.generate_nodes_from_file(sFile) + lElts = lNodes + random.shuffle(lElts) + for lElt in lElts[:10]: + status = self._tox.self_get_connection_status() + try: + if self._tox.bootstrap(*lElt): + LOG.info('connected to ' + lElt[0]+' '+repr(status)) + else: + LOG.warn('failed connecting to ' + lElt[0]) + except Exception as e: + LOG.warn('error connecting to ' + lElt[0]) + + if self._oargs.proxy_type > 0: + random.shuffle(lElts) + for lElt in lElts[:10]: + status = self._tox.self_get_connection_status() + try: + if self._tox.add_tcp_relay(*lElt): + LOG.info('relayed to ' + lElt[0] +' '+repr(status)) + else: + LOG.warn('failed relay to ' + lElt[0]) + except Exception as e: + LOG.warn('error relay to ' + lElt[0]) + + def loop(self) -> None: + if not self.av: + self.start() + checked = False + save_to_file(self._tox, sDATA_FILE) + + LOG.info('Starting loop.') + while True: + + status = self._tox.self_get_connection_status() + if not checked and status: + LOG.info('Connected to DHT.') + checked = True + if not checked and not status: + global iDHT_TRY + iDHT_TRY += 10 + self.connect() + self.iterate(100) + if iDHT_TRY >= iDHT_TRIES: + raise RuntimeError("Failed to connect to the DHT.") + LOG.warn(f"NOT Connected to DHT. {iDHT_TRY}") + checked = True + if checked and not status: + LOG.info('Disconnected from DHT.') + self.connect() + checked = False + + if bHAVE_AV: + True # self.av.witerate() + self.iterate(100) + + LOG.info('Ending loop.') + + def iterate(self, n:int = 100) -> None: + interval = self._tox.iteration_interval() + for i in range(n): + self._tox.iterate() + sleep(interval / 1000.0) + self._tox.iterate() + + def on_friend_request(self, pk: Union[bytes,str], message: Union[bytes,str]) -> None: + LOG.debug('Friend request from %s: %s' % (pk, message)) + self._tox.friend_add_norequest(pk) + LOG.info('on_friend_request Accepted.') + save_to_file(self._tox, sDATA_FILE) + + def on_friend_message(self, friendId: int, message_type: int, message: Union[bytes,str]) -> None: + name = self._tox.friend_get_name(friendId) + LOG.debug(f"{name}, {message}, {message_type}") + yMessage = bytes(message, 'UTF-8') + self._tox.friend_send_message(friendId, TOX_MESSAGE_TYPE['NORMAL'], yMessage) + LOG.info('EchoBot sent: %s' % message) + + def on_file_recv_chunk(self, fid: int, filenumber, position, data) -> None: + filename = self.files[(fid, filenumber)]['filename'] + size = self.files[(fid, filenumber)]['size'] + LOG.debug(f"on_file_recv_chunk {fid} {filenumber} {filename} {position/float(size)*100}") + + if data is None: + msg = "I got '{}', sending it back right away!".format(filename) + self._tox.friend_send_message(fid, TOX_MESSAGE_TYPE['NORMAL'], msg) + + self.files[(fid, 0)] = self.files[(fid, filenumber)] + + length = self.files[(fid, filenumber)]['size'] + self._tox.file_send(fid, TOX_FILE_KIND['DATA'], length, filename) + + del self.files[(fid, filenumber)] + return + + self.files[(fid, filenumber)]['f'] += data + +class App(): + def __init__(self): + self.mode = 0 +oAPP = App() + +class EchobotTox(Tox): + + def __init__(self, opts, app=None): + + super().__init__(opts, app=app) + self._address = self.self_get_address() + self.name = 'pyechobot' + self._opts = opts + self._app = app + +class BaseThread(threading.Thread): + + def __init__(self, name=None, target=None): + if name: + super().__init__(name=name, target=target) + else: + super().__init__(target=target) + self._stop_thread = False + self.name = name + + def stop_thread(self, timeout=-1) -> None: + self._stop_thread = True + if timeout < 0: + timeout = ts.iTHREAD_TIMEOUT + i = 0 + while i < ts.iTHREAD_JOINS: + self.join(timeout) + if not self.is_alive(): break + i = i + 1 + else: + LOG.warning(f"{self.name} BLOCKED") + +class ToxIterateThread(BaseThread): + + def __init__(self, tox): + super().__init__(name='ToxIterateThread') + self._tox = tox + + def run(self) -> None: + while not self._stop_thread: + self._tox.iterate() + sleep(self._tox.iteration_interval() / 1000) + +def oArgparse(lArgv): + parser = ts.oMainArgparser() + parser.add_argument('--norequest', type=str, default='False', + choices=['True','False'], + help='Use _norequest') + parser.add_argument('profile', type=str, nargs='?', default=None, + help='Path to Tox profile') + oArgs = parser.parse_args(lArgv) + ts.clean_booleans(oArgs) + + if hasattr(oArgs, 'sleep'): + if oArgs.sleep == 'qt': + pass # broken or gevent.sleep(idle_period) + elif oArgs.sleep == 'gevent': + pass # broken or gevent.sleep(idle_period) + else: + oArgs.sleep = 'time' + + return oArgs + +def iMain(oArgs) -> int: + global sDATA_FILE + # oTOX_OPTIONS = ToxOptions() + global oTOX_OPTIONS + oMainArgparser + oTOX_OPTIONS = ts.oToxygenToxOptions(oArgs) + opts = oTOX_OPTIONS + if coloredlogs: + coloredlogs.install( + level=oArgs.loglevel, + logger=LOG, + # %(asctime)s,%(msecs)03d %(hostname)s [%(process)d] + fmt='%(name)s %(levelname)s %(message)s' + ) + else: + if 'logfile' in oArgs: + logging.basicConfig(filename=oArgs.logfile, + level=oArgs.loglevel, + format='%(levelname)-8s %(message)s') + else: + logging.basicConfig(level=oArgs.loglevel, + format='%(levelname)-8s %(message)s') + + iRet = 0 + if hasattr(oArgs,'profile') and oArgs.profile and os.path.isfile(oArgs.profile): + sDATA_FILE = oArgs.profile + LOG.info(f"loading from {sDATA_FILE}") + opts.savedata_data = load_from_file(sDATA_FILE) + opts.savedata_length = len(opts.savedata_data) + opts.savedata_type = enums.TOX_SAVEDATA_TYPE['TOX_SAVE'] + else: + opts.savedata_data = None + + try: + oTox = EchobotTox(opts, app=oAPP) + t = EchoBot(oTox) + t._oargs = oArgs + t.start() + t.loop() + save_to_file(t._tox, sDATA_FILE) + except KeyboardInterrupt: + save_to_file(t._tox, sDATA_FILE) + except RuntimeError as e: + LOG.error(f"ERROR {e}") + iRet = 1 + except Exception as e: + LOG.error(f"EXCEPTION {e}") + LOG.warn(' iMain(): ' \ + +'\n' + traceback.format_exc()) + iRet = 1 + return iRet + +def main(lArgs=None) -> int: + global oTOX_OARGS + global oTOX_OPTIONS + global bIS_LOCAL + if lArgs is None: lArgs = [] + oArgs = oArgparse(lArgs) + bIS_LOCAL = oArgs.network in ['newlocal', 'localnew', 'local'] + oTOX_OARGS = oArgs + setattr(oTOX_OARGS, 'bIS_LOCAL', bIS_LOCAL) + oTOX_OPTIONS = ts.oToxygenToxOptions(oArgs) + if coloredlogs: + # https://pypi.org/project/coloredlogs/ + coloredlogs.install(level=oArgs.loglevel, + logger=LOG, + # %(asctime)s,%(msecs)03d %(hostname)s [%(process)d] + fmt='%(name)s %(levelname)s %(message)s' + ) + else: + logging.basicConfig(level=oArgs.loglevel) # logging.INFO + + return iMain(oArgs) + +if __name__ == '__main__': + try: + i = main(sys.argv[1:]) + except KeyboardInterrupt as e: + i = 0 + except Exception as e: + i = 1 + sys.exit(i) diff --git a/tox_profile.py b/tox_profile.py new file mode 100644 index 0000000..56a924c --- /dev/null +++ b/tox_profile.py @@ -0,0 +1,1471 @@ +# -*- mode: python; indent-tabs-mode: nil; py-indent-offset: 4; coding: utf-8 -*- + +""" +Reads a tox profile and prints out information on what's in there to stderr. + +Call it with one argument, the filename of the profile for the decrypt or info +commands, or the filename of the nodes file for the nodes command. + +4 commands are supported: +--command info - default + prints info about what's in the Tox profile to stderr + +--command nodes + assumes you are reading a json nodes file instead of a profile + +--command decrypt + decrypts the profile and writes to the result to stdout + +--command edits + edits fields in a Tox profile with --output to a file + +--command onions + cleans or checks a /etc/tor/torrc file with --output to a file + +""" + +""" + --output Destination for info/decrypt/edit/nodes + --info default='info', + choices=[info, save, repr, yaml,json, pprint] + with --info=info prints info about the profile to stderr + yaml,json, pprint, repr - output format + nmap_dht - test DHT nodes with nmap + nmap_relay - test TCP_RELAY nodes with nmap + nmap_path - test PATH_NODE nodes with nmap + --indent for pprint/yaml/json default=2 + + --nodes + choices=[select_tcp, select_udp, nmap_tcp, select_version, nmap_udp, check, download] + select_udp - select udp nodes + select_tcp - select tcp nodes + nmap_udp - test UDP nodes with nmap + nmap_tcp - test TCP nodes with nmap + select_version - select nodes that are the latest version + download - download nodes from --download_nodes_url + check - check nodes from --download_nodes_url + clean - check nodes and save them as --output + --download_nodes_url https://nodes.tox.chat/json + + --edit + help - print a summary of what fields can be edited + section,num,key,val - edit the field section,num,key with val + + --onions experimental + config - check your /etc/tor/torrc configuration + test - test your /etc/tor/torrc configuration + +""" + +# originally from: +# https://stackoverflow.com/questions/30901873/what-format-are-tox-files-stored-in + +import argparse +import json +import logging +import os +import shutil +import struct +import sys +import time +import warnings +from pprint import pprint +from socket import AF_INET, AF_INET6, inet_ntop + +warnings.filterwarnings('ignore') + +from tox_wrapper.tests import support_testing as ts + +try: + # https://pypi.org/project/msgpack/ + import msgpack +except ImportError as e: # noqa + msgpack = None +try: + import yaml +except ImportError as e: # noqa + yaml = None +try: + import stem +except ImportError as e: # noqa + stem = None +try: + import nmap +except ImportError as e: # noqa + nmap = None +try: + # https://pypi.org/project/coloredlogs/ + import coloredlogs + if 'COLOREDLOGS_LEVEL_STYLES' not in os.environ: + os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red' +except ImportError as e: # noqa + coloredlogs = False + +try: + # https://git.plastiras.org/emdee/toxygen_wrapper + from tox_wrapper.toxencryptsave import ToxEncryptSave + from tox_wrapper.tests import support_testing as ts + from tox_wrapper.tests.support_http import bAreWeConnected, download_url + from tox_wrapper.tests.support_testing import sTorResolve +except ImportError as e: + print(f"Import Warning {e}") + print("Download toxygen_wrapper to deal with encrypted tox files, from:") + print("https://git.plastiras.org/emdee/toxygen_wrapper") + print("Just put the parent of the tox_wrapper directory on your PYTHONPATH") + print("You also need to link your libtoxcore.so and libtoxav.so") + print("and libtoxencryptsave.so into tox_wrapper/../libs/") + print("Link all 3 from libtoxcore.so if you have only libtoxcore.so") + ToxEncryptSave = None + download_url = None + bAreWeConnected = None + sTorResolve = None + ts = None + +LOG = logging.getLogger('TSF') + +def LOG_error(a): print('EROR> '+a) +def LOG_warn(a): print('WARN> '+a) +def LOG_info(a): + bVERBOSE = hasattr(__builtins__, 'oArgs') and oArgs.log_level <= 20 + if bVERBOSE: print('INFO> '+a) +def LOG_debug(a): + bVERBOSE = hasattr(__builtins__, 'oArgs') and oArgs.log_level <= 10-1 + if bVERBOSE: print('DBUG> '+a) +def LOG_trace(a): + bVERBOSE = hasattr(__builtins__, 'oArgs') and oArgs.log_level < 10 + if bVERBOSE: print('TRAC> '+a) + +__version__ = "0.1.0" +# FixMe for Windows +sDIR = os.environ.get('TMPDIR', '/tmp') +sTOX_VERSION = "1000002018" +sVER_MIN = "1000002013" +# 3 months +iOLD_SECS = 60*60*24*30*3 + +bHAVE_NMAP = shutil.which('nmap') +bHAVE_TOR = shutil.which('tor') +bHAVE_JQ = shutil.which('jq') +bHAVE_BASH = shutil.which('bash') +bMARK = b'\x00\x00\x00\x00\x1f\x1b\xed\x15' +bDEBUG = 'DEBUG' in os.environ and os.environ['DEBUG'] != 0 +def trace(s): LOG.log(LOG.level, '+ ' +s) +LOG.trace = trace + +global bOUT, aOUT, sENC +aOUT = {} +bOUT = b'' +lLABELS = [] +sENC = sys.getdefaultencoding() # 'utf-8' +lNULLS = ['', '[]', 'null'] +lNONES = ['', '-', 'NONE'] +# grep '#''#' logging_tox_savefile.py|sed -e 's/.* //' +sEDIT_HELP = """ +NAME,.,Nick_name,str +STATUSMESSAGE,.,Status_message,str +STATUS,.,Online_status,int +NOSPAMKEYS,.,Nospam,hexstr +NOSPAMKEYS,.,Public_key,hexstr +NOSPAMKEYS,.,Private_key,hexstr +DHT,.,DHTnode, +TCP_RELAY,.,TCPnode, +PATH_NODE,.,PATHnode, +""" +# a dictionary of sets of lines +lONION_CONFIG = {"hsconfig": [ + '# Tox hidden service configuration.', + 'HiddenServiceDir /var/lib/tor/tox-hsv3', + 'HiddenServicePort 33446 127.0.0.1:33446', + ], + "vadr": [ + 'VirtualAddrNetworkIPv4 172.16.0.0/12', + 'AutomapHostsSuffixes .exit,.onion', + ], + "mapaddress": [] + } + + +lONION_NODES = [ + dict(maintainer="Tha_14", + public_key="8E8B63299B3D520FB377FE5100E65E3322F7AE5B20A0ACED2981769FC5B43725", + motd="Add me on Tox: F0AA7C8C55552E8593B2B77AC6FCA598A40D1F5F52A26C2322690A4BF1DFCB0DD8AEDD2822FF", + onions=[ + "h5g52d26mmi67pzzln2uya5msfzjdewengefaj75diipeskoo252lnqd.onion:33446"], + ), + dict(motd="Emdee", + public_key= "EC8F7405F79F281569B6C66D9F03490973AB99BC9175C44FBEF4C3428A63B80D", + onions=[ + "l2ct3xnuaiwwtoybtn46qp2av4ndxcguwupzyv6xrsmnwi647vvmwtqd.onion:33446", + ] + ), +] + +#messenger.c +MESSENGER_STATE_TYPE_NOSPAMKEYS = 1 +MESSENGER_STATE_TYPE_DHT = 2 +MESSENGER_STATE_TYPE_FRIENDS = 3 +MESSENGER_STATE_TYPE_NAME = 4 +MESSENGER_STATE_TYPE_STATUSMESSAGE = 5 +MESSENGER_STATE_TYPE_STATUS = 6 +MESSENGER_STATE_TYPE_GROUPS = 7 +MESSENGER_STATE_TYPE_TCP_RELAY = 10 +MESSENGER_STATE_TYPE_PATH_NODE = 11 +MESSENGER_STATE_TYPE_CONFERENCES = 20 +MESSENGER_STATE_TYPE_END = 255 +dSTATE_TYPE = { + MESSENGER_STATE_TYPE_NOSPAMKEYS: "NOSPAMKEYS", + MESSENGER_STATE_TYPE_DHT: "DHT", + MESSENGER_STATE_TYPE_FRIENDS: "FRIENDS", + MESSENGER_STATE_TYPE_NAME: "NAME", + MESSENGER_STATE_TYPE_STATUSMESSAGE: "STATUSMESSAGE", + MESSENGER_STATE_TYPE_STATUS: "STATUS", + MESSENGER_STATE_TYPE_GROUPS: "GROUPS", + MESSENGER_STATE_TYPE_TCP_RELAY: "TCP_RELAY", + MESSENGER_STATE_TYPE_PATH_NODE: "PATH_NODE", + MESSENGER_STATE_TYPE_CONFERENCES: "CONFERENCES", + MESSENGER_STATE_TYPE_END: "END", +} + +def decrypt_data(data): + from getpass import getpass + + if not ToxEncryptSave: return data + + oToxES = ToxEncryptSave() + if not oToxES.is_data_encrypted(data): + LOG.debug('Not encrypted') + return data + assert data[:8] == b'toxEsave', data[:8] + + sys.stdout.flush() + password = getpass('Password: ') + assert password + newData = oToxES.pass_decrypt(data, password) + LOG.debug('Decrypted: ' +str(len(newData)) +' bytes') + return newData + +def str_to_hex(raw_id, length=None): + if length is None: length = len(raw_id) + res = ''.join('{:02x}'.format(ord(raw_id[i])) for i in range(length)) + return res.upper() + +def bin_to_hex(raw_id, length=None): + if length is None: length = len(raw_id) + res = ''.join('{:02x}'.format(raw_id[i]) for i in range(length)) + return res.upper() + +def lProcessFriends(state, index, length, result): + """Friend: + +The integers in this structure are stored in Big Endian format. + +Length Contents +1 uint8_t Status +32 Long term public key +1024 Friend request message as a byte string +1 PADDING +2 uint16_t Size of the friend request message +128 Name as a byte string +2 uint16_t Size of the name +1007 Status message as a byte string +1 PADDING +2 uint16_t Size of the status message +1 uint8_t User status (see also: USERSTATUS) +3 PADDING +4 uint32_t Nospam (only used for sending a friend request) +8 uint64_t Last seen time + +""" + dStatus = { # Status Meaning + 0: 'Not a friend', + 1: 'Friend added', + 2: 'Friend request sent', + 3: 'Confirmed friend', + 4: 'Friend online' + } + slen = 1+32+1024+1+2+128+2+1007+1+2+1+3+4+8 # 2216 + assert length % slen == 0, length + lIN = [] + for i in range(length // slen): + delta = i*slen + status = struct.unpack_from(">b", result, delta)[0] + o = delta+1; l = 32 + pk = bin_to_hex(result[o:o+l], l) + + o = delta+1+32+1024+1+2+128; l = 2 + nsize = struct.unpack_from(">H", result, o)[0] + o = delta+1+32+1024+1+2; l = 128 + name = str(result[o:o+nsize], sENC) + + o = delta+1+32+1024+1+2+128+2+1007; l = 2 + msize = struct.unpack_from(">H", result, o)[0] + o = delta+1+32+1024+1+2+128+2; l = 1007 + mame = str(result[o:o+msize], sENC) + LOG.info(f"Friend #{i} {dStatus[status]} {name} {pk}") + lIN += [{"Status": dStatus[status], + "Name": name, + "Pk": pk}] + return lIN + +def lProcessGroups(state, index, length, result, label="GROUPS"): + """ + No GROUPS description in spec.html + """ + global sENC + lIN = [] + if not msgpack: + LOG.warn(f"process_chunk Groups = NO msgpack bytes={length}") + return [] + try: + groups = msgpack.loads(result, raw=True) + LOG.info(f"{label} {len(groups)} groups") + i = 0 + for group in groups: + assert len(group) == 7, group + + state_values, \ + state_bin, \ + topic_info, \ + mod_list, \ + keys, \ + self_info, \ + saved_peers, = group + + if state_values is None: + LOG.warn(f"lProcessGroups #{i} state_values is None") + else: + assert len(state_values) == 8, state_values + manually_disconnected, \ + group_name_len, \ + privacy_state, \ + maxpeers, \ + password_length, \ + version, \ + topic_lock, \ + voice_state = state_values + LOG.info(f"lProcessGroups #{i} version={version}") + dBINS = {"Version": version, + "Privacy_state": privacy_state} + lIN += [{"State_values": dBINS}] + + if state_bin is None: + LOG.warn(f"lProcessGroups #{i} state_bin is None") + else: + assert len(state_bin) == 5, state_bin + shared_state_sig, \ + founder_public_key, \ + group_name_len, \ + password_length, \ + mod_list_hash = state_bin + LOG.info(f"lProcessGroups #{i} founder_public_key={bin_to_hex(founder_public_key)}") + dBINS = {"Founder_public_key": bin_to_hex(founder_public_key)} + lIN += [{"State_bin": dBINS}] + + if topic_info is None: + LOG.warn(f"lProcessGroups #{i} topic_info is None") + else: + assert len(topic_info) == 6, topic_info + version, \ + length, \ + checksum, \ + topic, \ + public_sig_key, \ + topic_sig = topic_info + + topic_info_topic = str(topic, sENC) + LOG.info(f"lProcessGroups #{i} topic_info_topic={topic_info_topic}") + dBINS = {"Topic_info_topic": topic_info_topic + } + lIN += [{"Topic_info": dBINS}] + + if mod_list is None: + LOG.warn(f"lProcessGroups #{i} mod_list is None") + else: + assert len(mod_list) == 2, mod_list + num_moderators = mod_list[0] + LOG.info(f"lProcessGroups #{i} num moderators={mod_list[0]}") + #define CRYPTO_SIGN_PUBLIC_KEY_SIZE 32 + lMODS = [] + if not num_moderators: + LOG.warn(f"lProcessGroups #{i} num_moderators is 0") + else: + mods = mod_list[1] + assert len(mods) % 32 == 0, len(mods) + assert len(mods) == num_moderators * 32, len(mods) + for j in range(num_moderators): + mod = mods[j*32:j*32 + 32] + LOG.info(f"lProcessGroups group#{i} mod#{j} sig_pk={bin_to_hex(mod)}") + lMODS += [{"Sig_pk": bin_to_hex(mod)}] + lIN += [{"Moderators": lMODS}] + + if keys is None: + LOG.warn(f"lProcessGroups #{i} keys is None") + else: + assert len(keys) == 4, keys + LOG.debug(f"lProcessGroups #{i} {repr(list(map(len, keys)))}") + chat_public_key, \ + chat_secret_key, \ + self_public_key, \ + self_secret_key = keys + LOG.info(f"lProcessGroups #{i} chat_public_key={bin_to_hex(chat_public_key)}") + lIN[0].update({"Chat_public_key": bin_to_hex(chat_public_key)}) + if int(bin_to_hex(chat_secret_key), 16) != 0: + # 192 * b'0' + LOG.info(f"lProcessGroups #{i} chat_secret_key={bin_to_hex(chat_secret_key)}") + lIN[0].update({"Chat_secret_key": bin_to_hex(chat_secret_key)}) + + LOG.info(f"lProcessGroups #{i} self_public_key={bin_to_hex(self_public_key)}") + lIN[0].update({"Self_public_key": bin_to_hex(self_public_key)}) + LOG.info(f"lProcessGroups #{i} self_secret_key={bin_to_hex(self_secret_key)}") + lIN[0].update({"Self_secret_key": bin_to_hex(self_secret_key)}) + + if self_info is None: + LOG.warn(f"lProcessGroups #{i} self_info is None") + else: + assert len(self_info) == 4, self_info + self_nick_len, self_role, self_status, self_nick = self_info + self_nick = str(self_nick, sENC) + dBINS = {"Self_nick": self_nick, + "Self_role": self_role, + "Self_status": self_status, + "Self_info": self_info, + } + LOG.info(f"lProcessGroups #{i} {repr(dBINS)}") + lIN += [dBINS] + + if saved_peers is None: + LOG.warn(f"lProcessGroups #{i} saved_peers is None") + else: + assert len(saved_peers) == 2, saved_peers + i += 1 + + except Exception as e: + LOG.warn(f"process_chunk Groups #{i} error={e}") + return lIN + +def lProcessNodeInfo(state, index, length, result, label="DHTnode"): + """Node Info (packed node format) + +The Node Info data structure contains a Transport Protocol, a Socket + Address, and a Public Key. This is sufficient information to start + communicating with that node. The binary representation of a Node Info is + called the “packed node format”. + + Length Type Contents + 1 bit Transport Protocol UDP = 0, TCP = 1 + 7 bit Address Family 2 = IPv4, 10 = IPv6 + 4 | 16 IP address 4 bytes for IPv4, 16 bytes for IPv6 + 2 Port Number Port number + 32 Public Key Node ID + +""" + delta = 0 + relay = 0 + lIN = [] + while length > 0: + status = struct.unpack_from(">B", result, delta)[0] + if status >= 128: + prot = 'TCP' + af = status - 128 + else: + prot = 'UDP' + af = status + if af == 2: + af = 'IPv4' + alen = 4 + ipaddr = inet_ntop(AF_INET, result[delta+1:delta+1+alen]) + else: + af = 'IPv6' + alen = 16 + ipaddr = inet_ntop(AF_INET6, result[delta+1:delta+1+alen]) + total = 1 + alen + 2 + 32 + port = int(struct.unpack_from(">H", result, delta+1+alen)[0]) + pk = bin_to_hex(result[delta+1+alen+2:delta+1+alen+2+32], 32) + LOG.info(f"{label} #{relay} bytes={length} status={status} prot={prot} af={af} ip={ipaddr} port={port} pk={pk}") + lIN += [{"Bytes": length, + "Status": status, + "Prot": prot, + "Af": af, + "Ip": ipaddr, + "Port": port, + "Pk": pk}] + relay += 1 + delta += total + length -= total + return lIN + +def lProcessDHTnodes(state, index, length, result, label="DHTnode"): + relay = 0 + status = struct.unpack_from(" 0: + slen = struct.unpack_from("B", result, offset+8)[0] + assert status < 12 + prot = 'UDP' + if status == 2: + af = 'IPv4' + alen = 4 + ipaddr = inet_ntop(AF_INET, result[offset+8+1:offset+8+1+alen]) + else: + af = 'IPv6' + alen = 16 + ipaddr = inet_ntop(AF_INET6, result[offset+8+1:offset+8+1+alen]) + subtotal = 1 + alen + 2 + 32 + port = int(struct.unpack_from(">H", result, offset+8+1+alen)[0]) + pk = bin_to_hex(result[offset+8+1+alen+2:offset+8+1+alen+2+32], 32) + + LOG.info(f"{label} #{relay} status={status} ipaddr={ipaddr} port={port} {pk}") + lIN += [{ + "Status": status, + "Prot": prot, + "Af": af, + "Ip": ipaddr, + "Port": port, + "Pk": pk}] + offset += subtotal + relay += 1 + delta += total + length -= total + return lIN + +def process_chunk(index, state, oArgs=None): + global bOUT, aOUT, lLABELS + global sENC + + length = struct.unpack_from(" 0: + LOG.warn(f"PROCESS_CHUNK {label} index={index} bOUT={len(bOUT)} delta={diff} length={length}") + elif bDEBUG: + LOG.trace(f"PROCESS_CHUNK {label} index={index} bOUT={len(bOUT)} delta={diff} length={length}") + + if data_type == MESSENGER_STATE_TYPE_NOSPAMKEYS: + lLABELS += [label] + nospam = bin_to_hex(result[0:4]) + public_key = bin_to_hex(result[4:36]) + private_key = bin_to_hex(result[36:68]) + LOG.info(f"{label} Nospam = {nospam}") + LOG.info(f"{label} Public_key = {public_key}") + LOG.info(f"{label} Private_key = {private_key}") + aIN = {"Nospam": f"{nospam}", + "Public_key": f"{public_key}", + "Private_key": f"{private_key}"} + aOUT.update({label: aIN}) + if oArgs.command == 'edit' and section == label: + ## NOSPAMKEYS,.,Nospam,hexstr + if key == "Nospam": + assert len(val) == 4*2, val + result = bytes.fromhex (val) +result[4:] + LOG.info(f"{label} {key} EDITED to {val}") + ## NOSPAMKEYS,.,Public_key,hexstr + elif key == "Public_key": + assert len(val) == 32 * 2, val + result = +result[0:4] +bytes.fromhex(val) +result[36:] + LOG.info(f"{label} {key} EDITED to {val}") + ## NOSPAMKEYS,.,Private_key,hexstr + elif key == "Private_key": + assert len(val) == 32 * 2, val + result = +result[0:36] +bytes.fromhex(val) + LOG.info(f"{label} {key} EDITED to {val}") + + elif data_type == MESSENGER_STATE_TYPE_DHT: + lLABELS += [label] + LOG.debug(f"process_chunk {label} length={length}") + if length > 4: + lIN = lProcessDHTnodes(state, index, length, result, "DHTnode") + else: + lIN = [] + LOG.info(f"NO {label}") + aOUT.update({label: lIN}) + if oArgs.command == 'edit' and section == label: + ## DHT,.,DHTnode, + if num == '.' and key == "DHTnode" and val in lNULLS: + # 4 uint32_t (0x159000D) + status = 0x159000D + # FixMe - dunno + result = struct.pack(" 0: + lIN = lProcessFriends(state, index, length, result) + else: + lIN = [] + LOG.info(f"NO {label}") + aOUT.update({label: lIN}) + + elif data_type == MESSENGER_STATE_TYPE_NAME: + lLABELS += [label] + name = str(result, sENC) + LOG.info(f"{label} Nick_name = " +name) + aIN = {"Nick_name": name} + aOUT.update({label: aIN}) + if oArgs.command == 'edit' and section == label: + ## NAME,.,Nick_name,str + if key == "Nick_name": + result = bytes(val, sENC) + length = len(result) + LOG.info(f"{label} {key} EDITED to {val}") + + elif data_type == MESSENGER_STATE_TYPE_STATUSMESSAGE: + lLABELS += [label] + mess = str(result, sENC) + LOG.info(f"{label} StatusMessage = " +mess) + aIN = {"Status_message": mess} + aOUT.update({label: aIN}) + if oArgs.command == 'edit' and section == label: + ## STATUSMESSAGE,.,Status_message,str + if key == "Status_message": + result = bytes(val, sENC) + length = len(result) + LOG.info(f"{label} {key} EDITED to {val}") + + elif data_type == MESSENGER_STATE_TYPE_STATUS: + lLABELS += [label] + # 1 uint8_t status (0 = online, 1 = away, 2 = busy) + dStatus = {0: 'online', 1: 'away', 2: 'busy'} + status = struct.unpack_from(">b", state, index)[0] + status = dStatus[status] + LOG.info(f"{label} = " +status) + aIN = {f"Online_status": status} + aOUT.update({label: aIN}) + if oArgs.command == 'edit' and section == label: + ## STATUS,.,Online_status,int + if key == "Online_status": + result = struct.pack(">b", int(val)) + length = len(result) + LOG.info(f"{label} {key} EDITED to {val}") + + elif data_type == MESSENGER_STATE_TYPE_GROUPS: + lLABELS += [label] + if length > 0: + lIN = lProcessGroups(state, index, length, result, label) + else: + lIN = [] + LOG.info(f"NO {label}") + aOUT.update({label: lIN}) + + elif data_type == MESSENGER_STATE_TYPE_TCP_RELAY: + lLABELS += [label] + if length > 0: + lIN = lProcessNodeInfo(state, index, length, result, "TCPnode") + LOG.info(f"TYPE_TCP_RELAY {len(lIN)} nodes {length} length") + else: + lIN = [] + LOG.warn(f"NO {label} {length} length") + aOUT.update({label: lIN}) + if oArgs.command == 'edit' and section == label: + ## TCP_RELAY,.,TCPnode, + if num == '.' and key == "TCPnode" and val in lNULLS: + result = b'' + length = 0 + LOG.info(f"{label} {key} EDITED to {val}") + + elif data_type == MESSENGER_STATE_TYPE_PATH_NODE: + lLABELS += [label] + #define NUM_SAVED_PATH_NODES 8 + if not length % 8 == 0: + # this should be an assert? + LOG.warn(f"process_chunk {label} mod={length % 8}") + else: + LOG.debug(f"process_chunk {label} bytes={length}") + lIN = lProcessNodeInfo(state, index, length, result, "PATHnode") + aOUT.update({label: lIN}) + if oArgs.command == 'edit' and section == label: + ## PATH_NODE,.,PATHnode, + if num == '.' and key == "PATHnode" and val in lNULLS: + result = b'' + length = 0 + LOG.info(f"{label} {key} EDITED to {val}") + + elif data_type == MESSENGER_STATE_TYPE_CONFERENCES: + lLABELS += [label] + lIN = [] + if length > 0: + LOG.debug(f"TODO process_chunk {label} bytes={length}") + else: + LOG.info(f"NO {label}") + aOUT.update({label: []}) + + elif data_type != MESSENGER_STATE_TYPE_END: + LOG.error("UNRECOGNIZED datatype={datatype}") + sys.exit(1) + else: + LOG.info("END") # That's all folks... + # drop through + if len(lLABELS) == len(dSTATE_TYPE.keys()) - 1: + LOG.info(f"{len(lLABELS)} sections") # That's all folks... + else: + LOG.warn(f"{10 - len(lLABELS)} sections missing {lLABELS}") # That's all folks... + + # We repack as we read: or edit as we parse; simply edit result and length. + # We'll add the results back to bOUT to see if we get what we started with. + # Then will will be able to selectively null sections or selectively edit. + assert length == len(result), length + bOUT += struct.pack("= len(state): + diff = len(bSAVE) - len(bOUT) + if oArgs.command != 'edit' and diff > 0: + # if short repacking as we read - tox_profile is padded with nulls + LOG.warn(f"PROCESS_CHUNK bSAVE={len(bSAVE)} bOUT={len(bOUT)} delta={diff}") + return + + process_chunk(new_index, state, oArgs) + +sNMAP_TCP = """#!/bin/bash +ip="" +declare -a ports +jq '.|with_entries(select(.key|match("nodes"))).nodes[]|select(.status_tcp)|select(.ipv4|match("."))|.ipv4,.tcp_ports' | while read line ; do + if [ -z "$ip" ] ; then + ip=`echo $line|sed -e 's/"//g'` + ports=() + continue + elif [ "$line" = '[' ] ; then + continue + elif [ "$line" = ']' ] ; then + if ! route | grep -q ^def ; then + echo ERROR no route + exit 3 + fi + if [ "$ip" = '"NONE"' -o "$ip" = 'NONE' ] ; then + : + elif ping -c 1 $ip | grep '100% packet loss' ; then + echo WARN failed ping $ip + else + echo INFO $ip "${ports[*]}" + cmd="nmap -Pn -n -sT -p T:"`echo "${ports[*]}" |sed -e 's/ /,/g'` + echo DBUG $cmd $ip + $cmd $ip | grep /tcp + fi + ip="" + continue + else + port=`echo $line|sed -e 's/,//'` + ports+=($port) + fi +done""" + +def sBashFileNmapTcp(): + assert bHAVE_JQ, "jq is required for this command" + assert bHAVE_NMAP, "nmap is required for this command" + assert bHAVE_BASH, "bash is required for this command" + f = "NmapTcp.bash" + sFile = os.path.join(sDIR, f) + if not os.path.exists(sFile): + with open(sFile, 'wt') as iFd: + iFd.write(sNMAP_TCP) + os.chmod(sFile, 0o0775) + assert os.path.exists(sFile) + return sFile + +def vBashFileNmapUdp(): + assert bHAVE_JQ, "jq is required for this command" + assert bHAVE_NMAP, "nmap is required for this command" + assert bHAVE_BASH, "bash is required for this command" + f = "NmapUdp.bash" + sFile = os.path.join(sDIR, f) + if not os.path.exists(sFile): + with open(sFile, 'wt') as iFd: + iFd.write(sNMAP_TCP. + replace('nmap -Pn -n -sT -p T', + 'nmap -Pn -n -sU -p U'). + replace('tcp_ports','udp_ports'). + replace('status_tcp','status_udp')) + os.chmod(sFile, 0o0775) + assert os.path.exists(sFile) + return sFile + +def lParseNapOutput(sFile): + lRet = [] + for sLine in open(sFile, 'rt').readlines(): + if sLine.startswith('Failed to resolve ') or \ + 'Temporary failure in name resolution' in sLine or \ + '/udp closed' in sLine or \ + '/tcp closed' in sLine: + lRet += [sLine] + return lRet + +sBLURB = """ +I see you have a torrc. You can help the network by running a bootstrap daemon +as a hidden service, or even using the --tcp_server option of your client. +""" + +def lNodesCheckNodes(json_nodes, oArgs, bClean=False): + """ + Checking NODES.json + """ + lErrs = [] + ierrs = 0 + nth = 0 + if bClean: lNew=[] + # assert type(json_nodes) == dict + bRUNNING_TOR = False + if bHAVE_TOR: + iret = os.system("netstat -nle4|grep -q :9050") + if iret == 0: + bRUNNING_TOR = True + + lOnions = [] + for node in json_nodes: + # new fields: + if bClean: + new_node = {} + for key,val in node.items(): + if type(val) == bytes: + new_node[key] = str(val, 'UTF-8') + else: + new_node[key] = val + if 'onions' not in new_node: + new_node['onions'] = [] + for elt in lONION_NODES: + if node['public_key'] == elt['public_key']: + new_node['onions'].extend(elt['onions']) + break + else: + # add to nodes + pass + else: + for keypair in node['onions']: + s = keypair.split(':')[0] + lOnions.append(s) + + for ipv in ['ipv4','ipv6']: + for fam in ["status_tcp", "status_udp"]: + if node[ipv] in lNONES \ + and node[fam] in [True, "true"]: + LOG.debug(f"{ipv} {node[ipv]} but node[{fam}] is true") + bLinux = os.path.exists('/proc') + if bLinux and not os.path.exists(f"/proc/sys/net/{ipv}/"): + continue + elif True: + if not node[ipv] in lNONES and ipv == 'ipv4': + # just ping for now + iret = os.system(f"ping -c 1 {node[ipv]} > /dev/null") + if iret == 0: + LOG.info(f"Pinged {node[ipv]}") + else: + LOG.warn(f"Failed ping {node[ipv]}") + continue + elif not node[ipv] in lNONES \ + and bHAVE_NMAP and bAreWeConnected and ts \ + and not bRUNNING_TOR \ + and not node[ipv] in lNONES: + # nmap test the ipv4/ipv6 + lElts = [[node[ipv], node['port'], node['public_key']]] + ts.bootstrap_iNmapInfo(lElts, oArgs, bIS_LOCAL=False, + iNODES=2, nmap=oArgs.nmap_cmd) + + if node['ipv4'] in lNONES and node['ipv6'] in lNONES and \ + not node['tcp_ports'] and not '.onion' in node['location']: + LOG.warn("No ports to contact the daemon on") + + if node["version"] and node["version"] < "1000002013": + lErrs += [nth] + LOG.error(f"{node['ipv4']}: vulnerable version {node['version']} < 1000002013") + elif node["version"] and node["version"] < sVER_MIN: + LOG.warn(f"{node['ipv4']}: outdated version {node['version']} < {sVER_MIN}") + + # Put the onion address in the location after the country code + if len(node["location"]) not in [2, 65]: + LOG.warn(f"{node['ipv4']}: location {node['location']} should be a 2 digit country code, or 'code onion'") + elif len(node["location"]) == 65 and \ + not node["location"].endswith('.onion'): + LOG.warn(f"{node['ipv4']}: location {node['location']} should be a 2 digit country code 'code onion'") + elif len(node["location"]) == 65 and \ + node["location"].endswith('.onion') and bHAVE_TOR: + onion = node["location"][3:] + if bHAVE_TOR and bAreWeConnected and bAreWeConnected() \ + and (not node[ipv] in lNONES and + not node[ipv] in lNONES): + # torresolve the onion + # Fixme - see if tor is running + try: + s = sTorResolve(onion, + verbose=False, + sHost='127.0.0.1', + iPort=9050) + except: + # assume tor isnt running + pass + else: + if s: + LOG.info(f"{node['ipv4']}: Found an onion that resolves to {s}") + else: + LOG.warn(f"{node['ipv4']}: Found an onion that resolves to {s}") + + if node['last_ping'] and time.time() - node['last_ping'] > iOLD_SECS: + LOG.debug(f"{node['ipv4']}: node has not pinged in more than 3 months") + + # suggestions YMMV + + if len(node['maintainer']) > 75 and len(node['motd']) < 75: + pass + # look for onion + if not node['motd']: + # LOG.info(f"Maybe put a ToxID: in motd so people can contact you.") + pass + if bClean and nth not in lErrs: + lNew += [new_node] + nth += 1 + + # fixme look for /etc/tor/torrc but it may not be readable + if bHAVE_TOR and os.path.exists('/etc/tor/torrc'): + # print(sBLURB) + LOG.info("Add this section to your /etc/tor/torrc") + for line in lONION_CONFIG['vadr']: + print(line) + if lOnions: + LOG.info("Add this section to your /etc/tor/torrc") + i = 1 + for line in lOnions: + hosts = line.split(':') + print(f"MapAddress {hosts[0]} 172.16.1.{i}") + i += 1 + + if bClean: + return lNew + else: + return lErrs + +def iNodesFileCheck(sProOrNodes, oArgs, bClean=False): + try: + if not os.path.exists(sProOrNodes): + raise RuntimeError("iNodesFileCheck file not found " +sProOrNodes) + with open(sProOrNodes, 'rt') as fl: + json_all = json.loads(fl.read()) + json_nodes = json_all['nodes'] + except Exception as e: # noqa + LOG.exception(f"{oArgs.command} error reading {sProOrNodes}") + return 1 + + LOG.info(f"iNodesFileCheck checking JSON") + i = 0 + try: + al = lNodesCheckNodes(json_nodes, oArgs, bClean=bClean) + if bClean == False: + i = len(al) + else: + now = time.time() + aOut = dict(last_scan=json_all['last_scan'], + last_refresh=now, + nodes=al) + sout = oArgs.output + try: + LOG.debug(f"iNodesFileClean saving to {sout}") + oStream = open(sout, 'wt', encoding=sENC) + json.dump(aOut, oStream, indent=oArgs.indent) + if oStream.write('\n') > 0: i = 0 + except Exception as e: # noqa + LOG.exception(f"iNodesFileClean error dumping JSON to {sout}") + return 3 + + except Exception as e: # noqa + LOG.exception(f"iNodesFileCheck error checking JSON") + i = -2 + else: + if i: + LOG.error(f"iNodesFileCheck {i} errors in {sProOrNodes}") + else: + LOG.info(f"iNodesFileCheck NO errors in {sProOrNodes}") + return i + +def iNodesFileClean(sProOrNodes): + # unused + return 0 + f = "DHTNodes.clean" + if not oArgs.output: + sout = os.path.join(sDIR, f) + else: + sout = oArgs.output + + try: + LOG.debug(f"iNodesFileClean saving to {sout}") + oStream = open(sout, 'wt', encoding=sENC) + json.dump(aOUT, oStream, indent=oArgs.indent) + if oStream.write('\n') > 0: iret = 0 + except Exception as e: # noqa + LOG.exception(f"iNodesFileClean error dumping JSON to {sout}") + return 3 + + LOG.info(f"{oArgs.info}ing iret={iret} to {oArgs.output}") + return 0 + +def iOsSystemNmapUdp(l, oArgs): + ierrs = 0 + for elt in l: + cmd = f"sudo nmap -Pn -n -sU -p U:{elt['Port']} {elt['Ip']}" + LOG.debug(f"{oArgs.info} {cmd} to {oArgs.output}") + ierrs += os.system(cmd +f" >> {oArgs.output} 2>&1") + if ierrs: + LOG.warn(f"{oArgs.info} {ierrs} ERRORs to {oArgs.output}") + else: + LOG.info(f"{oArgs.info} NO errors to {oArgs.output}") + lRet = lParseNapOutput(oArgs.output) + if lRet: + for sLine in lRet: + LOG.warn(f"{oArgs.nodes} {sLine}") + ierr = len(lRet) + ierrs += ierr + return ierrs + +def iOsSystemNmapTcp(l, oArgs): + ierrs = 0 + LOG.debug(f"{len(l)} nodes to {oArgs.output}") + for elt in l: + cmd = f"sudo nmap -Pn -n -sT -p T:{elt['Port']} {elt['Ip']}" + LOG.debug(f"iOsSystemNmapTcp {cmd} to {oArgs.output}") + ierr = os.system(cmd +f" >> {oArgs.output} 2>&1") + if ierr: + LOG.warn(f"iOsSystemNmapTcp {ierrs} ERRORs to {oArgs.output}") + else: + lRet = lParseNapOutput(oArgs.output) + if lRet: + for sLine in lRet: + LOG.warn(f"{oArgs.nodes} {sLine}") + ierr = len(lRet) + ierrs += ierr + return ierrs + +def vSetupLogging(log_level=logging.DEBUG): + global LOG + if coloredlogs: + aKw = dict(level=log_level, + logger=LOG, + fmt='%(name)s %(levelname)s %(message)s') + coloredlogs.install(**aKw) + else: + aKw = dict(level=log_level, + format='%(name)s %(levelname)-4s %(message)s') + logging.basicConfig(**aKw) + + logging._defaultFormatter = logging.Formatter(datefmt='%m-%d %H:%M:%S') + logging._defaultFormatter.default_time_format = '%m-%d %H:%M:%S' + logging._defaultFormatter.default_msec_format = '' + +def iTestTorConfig(sProOrNodes, oArgs, bClean=False): + # add_onion + LOG.info(f"iTestTorConfig {sProOrNodes}") + lEtcTorrc = open(sProOrNodes, 'rt').readlines() + if bClean == False: + LOG.info(f"Add these lines to {sProOrNodes}") + for key,val in lONION_CONFIG.items(): + for line in val: + if line.startswith('#'): continue + if line not in lEtcTorrc: + print(line) + # add_mapaddress + if bClean == False: + LOG.info(f"Add these lines to {sProOrNodes}") + i=1 + for elt in lONION_NODES: + for line in elt['onions']: + host,port = line.split(':') + print(f"MapAddress {host} 172.16.1.{i}") + i += 1 + + # add_bootstrap + return 0 + +def iTestTorExits(sProOrNodes, oArgs, bClean=False): + LOG.info(f"iTestTorExits") +# import pdb; pdb.set_trace() + # sProOrNodes + try: + if hasattr(ts, 'oSTEM_CONTROLER') and ts.oSTEM_CONTROLER \ + and ts.oSTEM_CONTROLER.is_set('ExcludeExitNodes'): + LOG_info(f"ExcludeExitNodes is set so we cant continue") + return 0 + LOG_info(f"ExcludeExitNodes is not set so we can continue") + l = ts.lExitExcluder(iPort=9051) + except Exception as e: + LOG.error(f"ExcludeExitNodes errored {e}") + return 1 + + return 0 + +def iTestTorTest(sProOrNodes, oArgs, bClean=False): + # test_onion + # check_mapaddress + # check_bootstrap + LOG.info(f"iTestTorTest {sProOrNodes}") + for elt in lONION_NODES: + for line in elt['onions']: + (host, port,) = line.split(':') + LOG.debug(f"iTestTorTest resolving {host}") + ip = ts.sTorResolve(host) + if ip: LOG.info(f"{host} resolved to {ip}") + + # test debian + # http://5ekxbftvqg26oir5wle3p27ax3wksbxcecnm6oemju7bjra2pn26s3qd.onion/ + return 0 + +def iTestOnionNodes(): + return 0 + +def iMainFun(sProOrNodes, oArgs): + global bOUT, aOUT, sENC + global bSAVE + + assert os.path.isfile(sProOrNodes), sProOrNodes + + sENC = oArgs.encoding + + bSAVE = open(sProOrNodes, 'rb').read() + if ToxEncryptSave and bSAVE[:8] == b'toxEsave': + try: + bSAVE = decrypt_data(bSAVE) + except Exception as e: + LOG.error(f"decrypting {sProOrNodes} - {e}") + sys.exit(1) + assert bSAVE + LOG.debug(f"{oArgs.command} {len(bSAVE)} bytes") + + oStream = None + LOG.info(f"Running {oArgs.command}") + if oArgs.command == 'decrypt': + assert oArgs.output, "--output required for this command" + oStream = open(oArgs.output, 'wb') + iret = oStream.write(bSAVE) + LOG.info(f"Wrote {iret} to {oArgs.output}") + iret = 0 + + elif oArgs.command == 'nodes': + iret = -1 + ep_sec = str(int(time.time())) + json_head = '{"last_scan":' +ep_sec \ + +',"last_refresh":' +ep_sec \ + +',"nodes":[' + if oArgs.nodes == 'select_tcp': + assert oArgs.output, "--output required for this command" + assert bHAVE_JQ, "jq is required for this command" + with open(oArgs.output, 'wt') as oFd: + oFd.write(json_head) + cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_tcp)|select(.ipv4|match(\".\"))' " + iret = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}") + with open(oArgs.output, 'at') as oFd: oFd.write(']}\n') + + elif oArgs.nodes == 'select_udp': + assert oArgs.output, "--output required for this command" + assert bHAVE_JQ, "jq is required for this command" + with open(oArgs.output, 'wt') as oFd: + oFd.write(json_head) + cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.ipv4|match(\".\"))' " + iret = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}") + with open(oArgs.output, 'at') as oFd: oFd.write(']}\n') + + elif oArgs.nodes == 'select_version': + assert bHAVE_JQ, "jq is required for this command" + assert oArgs.output, "--output required for this command" + with open(oArgs.output, 'wt') as oFd: + oFd.write(json_head) + cmd = f"cat '{sProOrNodes}' | jq '.|with_entries(select(.key|match(\"nodes\"))).nodes[]|select(.status_udp)|select(.version|match(\"{sTOX_VERSION}\"))'" + + iret = os.system(cmd +"| sed -e '2,$s/^{/,{/'" +f" >>{oArgs.output}") + with open(oArgs.output, 'at') as oFd: + oFd.write(']}\n') + + elif oArgs.nodes == 'nmap_tcp': + assert oArgs.output, "--output required for this command" + if not bAreWeConnected(): + LOG.warn(f"{oArgs.nodes} we are not connected") + else: + cmd = sBashFileNmapTcp() + cmd = f"sudo bash {cmd} < '{sProOrNodes}' >'{oArgs.output}' 2>&1" + LOG.debug(cmd) + iret = os.system(cmd) + if iret == 0: + lRet = lParseNapOutput(oArgs.output) + if lRet: + for sLine in lRet: + LOG.warn(f"{oArgs.nodes} {sLine}") + iret = len(lRet) + + elif oArgs.nodes == 'nmap_udp': + assert oArgs.output, "--output required for this command" + if not bAreWeConnected(): + LOG.warn(f"{oArgs.nodes} we are not connected") + elif bHAVE_TOR: + LOG.warn(f"{oArgs.nodes} this wont work behind tor") + cmd = vBashFileNmapUdp() + cmd = f"sudo bash {cmd} < '{sProOrNodes}'" +f" >'{oArgs.output}' 2>&1" + LOG.debug(cmd) + iret = os.system(cmd) + if iret == 0: + lRet = lParseNapOutput(oArgs.output) + if lRet: + for sLine in lRet: + LOG.warn(f"{oArgs.nodes} {sLine}") + iret = len(lRet) + + elif oArgs.nodes == 'download' and download_url: + if not bAreWeConnected(): + LOG.warn(f"{oArgs.nodes} we are not connected") + url = oArgs.download_nodes_url + b = download_url(url) + if not b: + LOG.warn("failed downloading list of nodes") + iret = -1 + else: + if oArgs.output: + oStream = open(oArgs.output, 'wb') + oStream.write(b) + else: + oStream = sys.stdout + oStream.write(str(b, sENC)) + iret = 0 + LOG.info(f"downloaded list of nodes to {oStream}") + + elif oArgs.nodes == 'check': + i = iNodesFileCheck(sProOrNodes, oArgs, bClean=False) + iret = i + + elif oArgs.nodes == 'clean': + assert oArgs.output, "--output required for this command" + i = iNodesFileCheck(sProOrNodes, oArgs, bClean=True) + iret = i + + if iret > 0: + LOG.warn(f"{oArgs.nodes} iret={iret} to {oArgs.output}") + + elif iret == 0: + LOG.info(f"{oArgs.nodes} iret={iret} to {oArgs.output}") + + elif oArgs.command == 'onions': + + LOG.info(f"{oArgs.command} {oArgs.onions} {oArgs.output}") + + if oArgs.onions == 'config': + i = iTestTorConfig(sProOrNodes, oArgs) + iret = i + + elif oArgs.onions == 'test': + i = iTestTorTest(sProOrNodes, oArgs) + iret = i + + elif oArgs.onions == 'exits': + i = iTestTorExits(sProOrNodes, oArgs) + iret = i + else: + RuntimeError(oArgs.onions) + + elif oArgs.command in ['info', 'edit']: + + if oArgs.command in ['edit']: + assert oArgs.output, "--output required for this command" + assert oArgs.edit != '', "--edit required for this command" + + elif oArgs.command == 'info': + # assert oArgs.info != '', "--info required for this command" + if oArgs.info in ['save', 'yaml', 'json', 'repr', 'pprint']: + assert oArgs.output, "--output required for this command" + + # toxEsave + assert bSAVE[:8] == bMARK, "Not a Tox profile" + bOUT = bMARK + + iret = 0 + process_chunk(len(bOUT), bSAVE, oArgs) + if not bOUT: + LOG.error(f"{oArgs.command} NO bOUT results") + iret = 1 + else: + oStream = None + LOG.debug(f"command={oArgs.command} len bOUT={len(bOUT)} results") + + if oArgs.command in ['edit'] or oArgs.info in ['save']: + LOG.debug(f"{oArgs.command} saving to {oArgs.output}") + oStream = open(oArgs.output, 'wb', encoding=None) + if oStream.write(bOUT) > 0: iret = 0 + LOG.info(f"{oArgs.info}ed iret={iret} to {oArgs.output}") + elif oArgs.info == 'info': + pass + iret = 0 + elif oArgs.info == 'yaml': + if not yaml: + LOG.warn(f"{oArgs.command} no yaml support") + iret = -1 + else: + LOG.debug(f"{oArgs.command} saving to {oArgs.output}") + oStream = open(oArgs.output, 'wt', encoding=sENC) + try: + assert aOUT + yaml.dump(aOUT, stream=oStream, indent=oArgs.indent) + except Exception as e: + LOG.warn(f'WARN: {e}') + else: + oStream.write('\n') + iret = 0 + LOG.info(f"{oArgs.info}ing iret={iret} to {oArgs.output}") + + elif oArgs.info == 'json': + if not yaml: + LOG.warn(f"{oArgs.command} no json support") + iret = -1 + else: + LOG.debug(f"{oArgs.command} saving to {oArgs.output}") + oStream = open(oArgs.output, 'wt', encoding=sENC) + try: + json.dump(aOUT, oStream, indent=oArgs.indent, skipkeys=True) + except: + LOG.warn("There are somtimes problems with the json info dump of bytes keys: ```TypeError: Object of type bytes is not JSON serializable```") + oStream.write('\n') > 0 + iret = 0 + LOG.info(f"{oArgs.info}ing iret={iret} to {oArgs.output}") + + elif oArgs.info == 'repr': + LOG.debug(f"{oArgs.command} saving to {oArgs.output}") + oStream = open(oArgs.output, 'wt', encoding=sENC) + if oStream.write(repr(bOUT)) > 0: iret = 0 + if oStream.write('\n') > 0: iret = 0 + LOG.info(f"{oArgs.info}ing iret={iret} to {oArgs.output}") + + elif oArgs.info == 'pprint': + LOG.debug(f"{oArgs.command} saving to {oArgs.output}") + oStream = open(oArgs.output, 'wt', encoding=sENC) + pprint(aOUT, stream=oStream, indent=oArgs.indent, width=80) + iret = 0 + LOG.info(f"{oArgs.info}ing iret={iret} to {oArgs.output}") + + elif oArgs.info == 'nmap_relay': + assert bHAVE_NMAP, "nmap is required for this command" + assert oArgs.output, "--output required for this command" + if aOUT["TCP_RELAY"]: + iret = iOsSystemNmapTcp(aOUT["TCP_RELAY"], oArgs) + else: + LOG.warn(f"{oArgs.info} no TCP_RELAY") + iret = 0 + + elif oArgs.info == 'nmap_dht': + assert bHAVE_NMAP, "nmap is required for this command" + assert oArgs.output, "--output required for this command" + if aOUT["DHT"]: + iret = iOsSystemNmapUdp(aOUT["DHT"], oArgs) + else: + LOG.warn(f"{oArgs.info} no DHT") + iret = 0 + + elif oArgs.info == 'nmap_path': + assert bHAVE_NMAP, "nmap is required for this command" + assert oArgs.output, "--output required for this command" + if aOUT["PATH_NODE"]: + iret = iOsSystemNmapUdp(aOUT["PATH_NODE"], oArgs) + else: + LOG.warn(f"{oArgs.info} no PATH_NODE") + iret = 0 + + else: + LOG.warn(f"{oArgs.command} UNREGOGNIZED") + + if oStream and oStream != sys.stdout and oStream != sys.stderr: + oStream.close() + return iret + +def oMainArgparser(_=None): + if not os.path.exists('/proc/sys/net/ipv6'): + bIpV6 = 'False' + else: + bIpV6 = 'True' + lIpV6Choices=[bIpV6, 'False'] + + parser = argparse.ArgumentParser(epilog=__doc__) + # list(dSTATE_TYPE.values()) + # ['nospamkeys', 'dht', 'friends', 'name', 'statusmessage', 'status', 'groups', 'tcp_relay', 'path_node', 'conferences'] + + parser.add_argument('--output', type=str, default='', + help='Destination for info/decrypt - defaults to stderr') + parser.add_argument('--command', type=str, default='info', + choices=['info', 'decrypt', 'nodes', 'edit', 'onions'], + help='Action command - default: info') + # nargs='+', + parser.add_argument('--edit', type=str, default='', + help='comma seperated SECTION,num,key,value - or help for ') + parser.add_argument('--indent', type=int, default=2, + help='Indent for yaml/json/pprint') + choices = ['info', 'save', 'repr', 'yaml','json', 'pprint'] + if bHAVE_NMAP: + choices += ['nmap_relay', 'nmap_dht', 'nmap_path'] + parser.add_argument('--info', type=str, default='info', + choices=choices, + help='Format for info command') + choices = ['check', 'clean'] + if bHAVE_JQ: + choices += ['select_tcp', 'select_udp', 'select_version'] + if bHAVE_NMAP: choices += ['nmap_tcp', 'nmap_udp'] + if download_url: + choices += ['download'] + # behind tor you may need 'sudo -u debian-tor nmap' + parser.add_argument('--nmap_cmd', type=str, default='nmap', + help="the command to run nmap") + parser.add_argument('--nodes', type=str, default='', + choices=choices, + help='Action for nodes command (requires jq)') + parser.add_argument('--download_nodes_url', type=str, + default='https://nodes.tox.chat/json') + parser.add_argument('--onions', type=str, default='', + choices=['config', 'test'] if bHAVE_TOR else [], + help='Action for onion command (requires tor)') + + parser.add_argument('--encoding', type=str, default=sENC) + parser.add_argument('lprofile', type=str, nargs='+', default=None, + help='tox profile files - may be encrypted') + parser.add_argument('--log_level', type=int, default=10) + + parser.add_argument('--proxy_host', '--proxy-host', type=str, + default='', + help='proxy host') + parser.add_argument('--proxy_port', '--proxy-port', default=0, type=int, + help='proxy port') + parser.add_argument('--proxy_type', '--proxy-type', default=0, type=int, + choices=[0,1,2], + help='proxy type 1=http, 2=socks') + + return parser + +def iMain(lArgv=None): + if lArgv is None: lArgv = sys.argv[1:] + parser = oMainArgparser() + oArgs = parser.parse_args(lArgv) + if oArgs.command in ['edit'] and oArgs.edit == 'help': + l = list(dSTATE_TYPE.values()) + l.remove('END') + print('Available Sections: ' +repr(l)) + print('Supported Quads: section,num,key,type ' +sEDIT_HELP) + sys.exit(0) + + __builtins__.oArgs = oArgs + vSetupLogging(oArgs.log_level) + i = 0 + for sProOrNodes in oArgs.lprofile: + i = iMainFun(sProOrNodes, oArgs) + +if __name__ == '__main__': + sys.exit(iMain(sys.argv[1:])) diff --git a/tox_profile_examples.bash b/tox_profile_examples.bash new file mode 100644 index 0000000..cc2d2b3 --- /dev/null +++ b/tox_profile_examples.bash @@ -0,0 +1,24 @@ +#!/bin/sh -e +# -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*- + +# some examples of tox-profile usage + +export PYTHONPATH=/mnt/o/var/local/src/toxygen_wrapper.git +TOX_HOME=$HOME/.config/tox +NMAP_CMD='sudo -u debian-tor nmap' + +echo INFO: check the download json file +python3 tox_profile.py --command nodes --nodes check \ + $TOX_HOME/DHTnodes.json.new \ + 2>&1 | tee /tmp/DHTnodes.json.log + +echo INFO: get the tcp nodes/ports from the downloaded json file +python3 tox_profile.py --command nodes --nodes select_tcp \ + --output /tmp/DHTnodes.json.tcp \ + $TOX_HOME/DHTnodes.json.new + +echo INFO: run ping/nmap on the tcp nodes/ports from the downloaded json file +python3 tox_profile.py --command nodes --nodes nmap_tcp \ + --nmap_cmd $NMAP_CMD \ + --output /tmp/DHTnodes.json.tcp.out \ + /tmp/DHTnodes.json.tcp diff --git a/tox_profile_test.bash b/tox_profile_test.bash new file mode 100755 index 0000000..8160fb3 --- /dev/null +++ b/tox_profile_test.bash @@ -0,0 +1,337 @@ +#!/bin/sh +# -*- mode: sh; fill-column: 75; tab-width: 8; coding: utf-8-unix -*- + +# tox_profile.py has a lot of features so it needs test coverage + +PREFIX=/mnt/o/var/local +ROLE=text +DEBUG=1 +EXE=/var/local/bin/python3.bash +WRAPPER=$PREFIX/src/toxygen_wrapper.git +tox=$HOME/.config/tox/toxic_profile.tox +[ -s $tox ] || exit 2 +target=$PREFIX/src/tox_profile/tox_profile.py + +OUT=/tmp/toxic_profile + +ps ax | grep -q tor && netstat -n4le | grep -q :9050 +[ $? -eq 0 ] && HAVE_TOR=1 || HAVE_TOR=0 + +[ -f /usr/local/bin/usr_local_tput.bash ] && \ + . /usr/local/bin/usr_local_tput.bash || { + DBUG() { echo DEBUG $* ; } + INFO() { echo INFO $* ; } + WARN() { echo WARN $* ; } + ERROR() { echo ERROR $* ; } + } + +if [ -z "$TOXCORE_LIBS" ] && [ ! -d libs ] ; then + mkdir libs + cd libs + # /lib/x86_64-linux-gnu/libtoxcore.so.2 + for pro in qtox toxic ; do + if which $pro 2> /dev/null ; then + DBUG linking to $pro libtoxcore + lib=$( ldd `which $pro` | grep libtoxcore|sed -e 's/.* => //' -e 's/ .*//') + [ -n "$lib" -a -f "$lib" ] || { WARN $Lib ; continue ; } + INFO linking to $lib + for elt in libtoxcore.so libtoxav.so libtoxencryptsave.so ; do + ln -s "$lib" "$elt" + done + export TOXCORE_LIBS=$PWD + break + fi + done + cd .. +elif [ -z "$TOXCORE_LIBS" ] && [ -d libs ] ; then + export TOXCORE_LIBS=$PWD/libs +fi + + +# set -- -e +[ -s $target ] || exit 1 + +[ -d $WRAPPER ] || { + ERROR wrapper is required https://git.plastiras.org/emdee/toxygen_wrapper + exit 3 +} +export PYTHONPATH=$WRAPPER + +json=$HOME/.config/tox/DHTnodes.json +[ -s $json ] || exit 4 + +which jq > /dev/null && HAVE_JQ=1 || HAVE_JQ=0 +which nmap > /dev/null && HAVE_NMAP=1 || HAVE_NMAP=0 + +sudo rm -f $OUT.* /tmp/toxic_nodes.* + +test_jq () { + [ $# -eq 3 ] || { + ERROR test_jq '#' "$@" + return 3 + } + in=$1 + out=$2 + err=$3 + [ -s $in ] || { + ERROR $i test_jq null $in + return 4 + } + jq . < $in >$out 2>$err || { + ERROR $i test_jq $json + return 5 + } + grep error: $err && { + ERROR $i test_jq $json + return 6 + } + [ -s $out ] || { + ERROR $i null $out + return 7 + } + [ -s $err ] || rm -f $err + return 0 +} + +i=0 +[ "$HAVE_JQ" = 0 ] || \ + test_jq $json /tmp/toxic_nodes.json /tmp/toxic_nodes.err || { + ERROR test_jq failed on $json + exit ${i}$? + } +[ -f /tmp/toxic_nodes.json ] || cp -p $json /tmp/toxic_nodes.json +json=/tmp/toxic_nodes.json + +i=1 +# required password +INFO $i decrypt $OUT.bin +$EXE $target --command decrypt --output $OUT.bin $tox || exit ${i}1 +[ -s $OUT.bin ] || exit ${i}2 + +tox=$OUT.bin +INFO $i info $tox +$EXE $target --command info --info info $tox 2>$OUT.info || { + ERROR $i $EXE $target --command info --info info $tox + exit ${i}3 +} +[ -s $OUT.info ] || exit ${i}4 + +INFO $i $EXE $target --command info --info save --output $OUT.save $tox +$EXE $target --command info --info save --output $OUT.save $tox 2>/dev/null || { + ERROR $? + exit ${i}5 +} + +[ -s $OUT.save ] || exit ${i}6 + +i=2 +[ $# -ne 0 -a $1 -ne $i ] || \ +! INFO $i Info and editing || \ +for the_tox in $tox $OUT.save ; do + DBUG $i $the_tox + the_base=`echo $the_tox | sed -e 's/.save$//' -e 's/.tox$//'` + for elt in json yaml pprint repr ; do + if [ $elt = yaml -o $elt = json ] ; then + # ModuleNotFoundError + python3 -c "import $elt" 2>/dev/null || continue + fi + INFO $i $the_base.$elt + DBUG $EXE $target \ + --command info --info $elt \ + --output $the_base.$elt $the_tox '2>'$the_base.$elt.err + $EXE $target --command info --info $elt \ + --output $the_base.$elt $the_tox 2>$the_base.$elt.err || { + tail $the_base.$elt.err + if [ $elt != yaml -a $elt != json ] ; then + exit ${i}0 + else + WARN $elt + fi + } + [ -s $the_base.$elt ] || { + WARN no output $the_base.$elt +# exit ${i}1 + } + done + + DBUG $EXE $target --command edit --edit help $the_tox + $EXE $target --command edit --edit help $the_tox 2>/dev/null || exit ${i}2 + + # edit the status message + INFO $i $the_base.Status_message 'STATUSMESSAGE,.,Status_message,Toxxed on Toxic' + $EXE $target --command edit --edit 'STATUSMESSAGE,.,Status_message,Toxxed on Toxic' \ + --output $the_base.Status_message.tox $the_tox 2>&1|grep EDIT || exit ${i}3 + [ -s $the_base.Status_message.tox ] || exit ${i}3 + $EXE $target --command info $the_base.Status_message.tox 2>&1|grep Toxxed || exit ${i}4 + + # edit the nick_name + INFO $i $the_base.Nick_name 'NAME,.,Nick_name,FooBar' + $EXE $target --command edit --edit 'NAME,.,Nick_name,FooBar' \ + --output $the_base.Nick_name.tox $the_tox 2>&1|grep EDIT || exit ${i}5 + [ -s $the_base.Nick_name.tox ] || exit ${i}5 + $EXE $target --command info $the_base.Nick_name.tox 2>&1|grep FooBar || exit ${i}6 + + # set the DHTnodes to empty + INFO $i $the_base.noDHT 'DHT,.,DHTnode,' + $EXE $target --command edit --edit 'DHT,.,DHTnode,' \ + --output $the_base.noDHT.tox $the_tox 2>&1|grep EDIT || exit ${i}7 + [ -s $the_base.noDHT.tox ] || exit ${i}7 + $EXE $target --command info $the_base.noDHT.tox 2>&1 | grep 'NO DHT' || exit ${i}8 + +done + +i=3 +[ "$#" -ne 0 -a "$1" != "$i" ] || \ +[ "$HAVE_JQ" = 0 ] || \ +! INFO $i Nodes || \ +for the_json in $json ; do + DBUG $i $the_json + the_base=`echo $the_json | sed -e 's/.json$//' -e 's/.tox$//'` + for nmap in clean check select_tcp select_udp select_version; do + $EXE $target --command nodes --nodes $nmap \ + --output $the_base.$nmap.json $the_json || { + WARN $i $the_json $nmap ${i}1 + continue + } + [ -s $the_base.$nmap.json ] || { + WARN $i $the_json $nmap ${i}2 + continue + } + [ $nmap = select_tcp ] && \ + grep '"status_tcp": false' $the_base.$nmap.json && { + WARN $i $the_json $nmap ${i}3 + continue + } + [ $nmap = select_udp ] && \ + grep '"status_udp": false' $the_base.$nmap.json && { + WARN $i $the_json $nmap ${i}4 + continue + } + test_jq $the_base.$nmap.json $the_base.$nmap.json.out /tmp/toxic_nodes.err || { + retval=$? + WARN $i $the_base.$nmap.json 3$? + } + INFO $i $the_base.$nmap + done +done + +i=4 +[ $# -ne 0 -a "$1" -ne $i ] || \ +[ "$HAVE_TOR" = 0 ] || \ +[ ! -f /etc/tor/torrc ] || \ +! INFO $i Onions || \ +for the_tox in /etc/tor/torrc ; do + DBUG $i $the_tox + the_base=`echo $OUT.save | sed -e 's/.save$//' -e 's/.tox$//'` + # exits + for slot in config test; do + if [ $slot = exits ] && ! netstat -nle4 | grep -q :9050 ; then + WARN Tor not running + continue + fi + INFO $target --command onions --onions $slot \ + --output $the_base.$slot.out $the_tox + DBUG=1 $EXE $target --command onions --onions $slot \ + --log_level 10 \ + --output $the_base.$slot.out $the_tox|| { + WARN $i $? + continue + } + [ true -o -s $the_base.$slot.out ] || { + WARN $i empty $the_base.$slot.out + continue + } + done + done + +# ls -l $OUT.* /tmp/toxic_nodes.* + +# DEBUG=0 /usr/local/bin/proxy_ping_test.bash tor || exit 0 +ip route | grep ^def || exit 0 + +i=5 +the_tox=$tox +[ $# -ne 0 -a "$1" != "$i" ] || \ +[ "$HAVE_JQ" = 0 ] || \ +[ "$HAVE_NMAP" = 0 ] || \ +! INFO $i Making dogfood || \ +for the_tox in $tox $OUT.save ; do + DBUG $i $the_tox + the_base=`echo $the_tox | sed -e 's/.save$//' -e 's/.tox$//'` + for nmap in nmap_relay nmap_dht nmap_path ; do +# [ $nmap = select_tcp ] && continue + if [ $nmap = nmap_dht ] && [ $HAVE_TOR = 1 ] ; then + INFO skipping $nmap because HAVE_TOR + continue + fi + INFO $i $the_base.$nmap + DBUG $target --command info --info $nmap \ + --output $the_base.$nmap.out $the_tox + $EXE $target --command info --info $nmap \ + --output $the_base.$nmap.out $the_tox 2>$the_base.$nmap.err || { + # select_tcp may be empty and jq errors + # exit ${i}1 + WARN $i $? $the_base.$nmap.err + tail $the_base.$nmap.err + continue + } + [ -s $the_base.$nmap.out ] || { + WARN $i empty $the_base.$nmap.out + continue + } + done +done + +i=6 +[ $# -ne 0 -a "$1" != "$i" ] || \ +[ "$HAVE_JQ" = 0 ] || \ +! INFO $i Eating dogfood || \ +for the_json in $json ; do + DBUG $i $the_json + the_base=`echo $the_json | sed -e 's/.save$//' -e 's/.json$//'` + for nmap in nmap_tcp nmap_udp ; do + if [ $nmap = nmap_udp ] && [ $HAVE_TOR = 1 ] ; then + INFO skipping $nmap because HAVE_TOR + continue + fi + INFO $i $target --command nodes --nodes $nmap --output $the_base.$nmap + $EXE $target --command nodes --nodes $nmap \ + --output $the_base.$nmap $the_json 2>$the_base.$nmap.err || { + WARN $i $the_json $nmap ${i}1 + continue + } + [ -s $the_base.$nmap ] || { + ERROR $i $the_json $nmap ${i}2 + exit ${i}2 + } + done +done + +i=7 +DBUG $i +$EXE $target --command nodes --nodes download \ + --output /tmp/toxic_nodes.new $json || { + ERROR $i $EXE $target --command nodes --nodes download $json + exit ${i}1 +} +[ -s /tmp/toxic_nodes.new ] || exit ${i}4 +INFO $i downloaded /tmp/toxic_nodes.new +json=/tmp/toxic_nodes.new +[ $# -ne 0 -a "$1" != "$i" ] || \ + [ "$HAVE_JQ" = 0 ] || \ + jq . < $json >/tmp/toxic_nodes.new.json 2>>/tmp/toxic_nodes.new.json.err || { + ERROR $i jq $json + exit ${i}2 + } +INFO $i jq from /tmp/toxic_nodes.new.json + +[ $# -ne 0 -a "$1" != "$i" ] || \ + [ "$HAVE_JQ" = 0 ] || \ + grep error: /tmp/toxic_nodes.new.json.err && { + ERROR $i jq $json + exit ${i}3 + } +INFO $i no errors in /tmp/toxic_nodes.new.err + + +exit 0