From 8d441fb84a7bc7e2b5515801f125610304a1caad Mon Sep 17 00:00:00 2001 From: localhost_frssoft Date: Mon, 26 Jun 2023 14:26:20 +0300 Subject: [PATCH] Breaking changes in instances selector Structure has changes Add fetch nodeinfo and avalaibility for all instances in list Can remove failed instances --- funkwhale_cli.py | 22 +------ src/fw_instances.py | 141 ++++++++++++++++++++++++++++++++++++++++++++ src/settings.py | 121 ++++++++++--------------------------- 3 files changed, 174 insertions(+), 110 deletions(-) create mode 100644 src/fw_instances.py diff --git a/funkwhale_cli.py b/funkwhale_cli.py index f793ed9..d4235b8 100755 --- a/funkwhale_cli.py +++ b/funkwhale_cli.py @@ -8,7 +8,7 @@ from src.fw_tracks import list_tracks from src.fw_channels import list_channels from src.fw_playlists import list_playlists from src.fw_recents import list_fav_or_history -import src.settings as settings +from src.fw_instances import instances_menu import src.mpv_control import json import os @@ -89,25 +89,7 @@ def main(): if selected == 'Recently listened': list_fav_or_history(is_history_view=True) if selected == 'Switch instance': - with open('config.json', 'rt') as f: - conf = json.loads(f.read()) - if conf.get('automatic_fetch_new_instances'): - public_server_list_instances = settings.get_new_funkwhale_servers() - new_ins_count = len(settings.get_new_funkwhale_servers()) - else: - public_server_list_instances = [] - new_ins_count = 'Disabled' - list_instances = conf.get( - 'public_list_instances') + public_server_list_instances - settings.set_config('public_list_instances', list_instances) - instance = fzf.prompt(list_instances, - '--header='+quote(f'Select instance\nNew instances: {new_ins_count}')) - if instance == []: - continue - else: - instance = instance[0] - - current_instance.select_instance(instance) + instances_menu() if selected == 'Sign in': print(f''' If You want sign in, please visit: diff --git a/src/fw_instances.py b/src/fw_instances.py new file mode 100644 index 0000000..8c7521f --- /dev/null +++ b/src/fw_instances.py @@ -0,0 +1,141 @@ +from src.fw_api import current_instance +import src.settings as settings +from pyfzf.pyfzf import FzfPrompt +from shlex import quote +from loguru import logger +import json +import time +import concurrent +import requests + +fzf = FzfPrompt() + + +@logger.catch +def get_new_funkwhale_servers(): + # Uses official API network.funkwhale.audio for getting new instances + public_server_api = 'https://network.funkwhale.audio/dashboards/api/tsdb/query' + now = int(time.time()) + timeback = now - 86400 + + request_public_servers = { + 'from': f"{timeback}", + 'to': f"{now}", + 'queries': [ + { + 'refId': "A", + 'intervalMs': 60000, + 'maxDataPoints': 1174, + 'datasourceId': 1, + 'rawSql': "SELECT * FROM (\n SELECT\n DISTINCT on (c.domain) c.domain as \"Name\",\n c.up as \"Is up\",\n coalesce(c.open_registrations, false) as \"Open registrations\",\n coalesce(anonymous_can_listen, false) as \"Anonymous can listen\",\n coalesce(c.usage_users_total, 0) as \"Total users\",\n coalesce(c.usage_users_active_month, 0) as \"Active users (this month)\",\n coalesce(c.software_version_major, 0)::text || '.' || coalesce(c.software_version_minor, 0)::text || '.' || coalesce(c.software_version_patch, 0)::text as \"Version\",\n c.time as \"Last checked\",\n d.first_seen as \"First seen\"\n FROM checks as c\n INNER JOIN domains AS d ON d.name = c.domain\n WHERE d.blocked = false AND c.up = true AND c.time > now() - INTERVAL '7 days'\n AND c.anonymous_can_listen IN ('true')\n AND c.open_registrations IN ('true','false')\n\n ORDER BY c.domain, c.time DESC\n) as t ORDER BY \"Active users (this month)\" DESC", + 'format': "table" + } + ] + } + try: + r = requests.post(public_server_api, json=request_public_servers) + results = r.json() + new_instances = {} + if results: + new_instances_list = results['results']['A']['tables'][0]['rows'] + for i in new_instances_list: + anonymousCanListen = i[1] + if anonymousCanListen: + new_instances[i[0]] = f'{anonymousCanListen} | ?' + + for i in get_new_funkwhale_servers_fediverse_observer(): + new_instances[i] = "?" + return new_instances + except: # If any errors then return empty list + return {} + + +def get_new_funkwhale_servers_fediverse_observer(): + try: + graphQL_request = { + 'query': + '{\n nodes(softwarename: \"funkwhale\") {\n domain\n metanodeinfo\n }\n}' + } + r = requests.post('https://api.fediverse.observer/', + headers={'Accept-Encoding': 'gzip, deflate'}, + json=graphQL_request) + new_instances = [] + for i in r.json()['data']['nodes']: + if i.get('metanodeinfo'): + auth_no_required = json.loads(i['metanodeinfo'])['library']['anonymousCanListen'] + if auth_no_required and i['domain']: + new_instances.append(i['domain']) + return new_instances + except: + return [] + + +def fetch_instances_nodeinfo_and_avalaibility(instances): + extended_instances_info = {} + + def request_nodeinfo(instance): + return requests.get('https://' + instance + '/api/v1/instance/nodeinfo/2.0/', + headers={ + 'Accept-Encoding': 'gzip, brotli, deflate', + 'User-Agent': 'funkwhale-cli/latest-commit; +https://git.phreedom.club/localhost_frssoft/funkwhale-cli'}, + timeout=5).json() + + with concurrent.futures.ThreadPoolExecutor() as executor: # optimally defined number of threads + res = [executor.submit(request_nodeinfo, instance) for instance in instances] + concurrent.futures.wait(res) + for idx, v in enumerate(instances): + try: + data_for_instance = res[idx].result() + anon = data_for_instance['metadata']['library']['anonymousCanListen'] + tracks = data_for_instance['metadata']['library']['tracks']['total'] + extended_instances_info[v] = f'{anon} | {tracks}' + except: + extended_instances_info[v] = 'fail' + return extended_instances_info + + +def instances_menu(fetch_manually=False, fetch_node_info=False): + with open('config.json', 'rt') as f: + conf = json.loads(f.read()) + if conf.get('automatic_fetch_new_instances') or fetch_manually: + public_server_list_instances = get_new_funkwhale_servers() + new_ins_count = len(public_server_list_instances) + else: + public_server_list_instances = {} + new_ins_count = 'Disabled' + + list_instances = conf.get('public_list_instances_extended') + if public_server_list_instances != {}: + list_instances_merge = {**list_instances, **public_server_list_instances} + settings.set_config('public_list_instances_extended', list_instances_merge) + list_instances = list_instances_merge + + map_in_extend_mode = '' + if fetch_node_info: + list_instances = fetch_instances_nodeinfo_and_avalaibility([instance.split('|')[0].strip() for instance in list_instances.keys()]) + settings.set_config('public_list_instances_extended', list_instances) + map_in_extend_mode = '\nmap: instance | anonymousCanListen | tracks' + instance_menu_selector = ['Fetch new instances', + 'Fetch nodeinfo and avalaibility', + 'Remove unreachible instances'] + + instance = fzf.prompt( + instance_menu_selector + + [f'{instance} | {info}' for instance, info in list_instances.items()], + '--header='+quote(f'Select instance\nNew instances: {new_ins_count}{map_in_extend_mode}')) + if instance == []: + return + else: + instance = instance[0].split('|')[0].strip() + if instance == 'Fetch new instances': + return instances_menu(fetch_manually=True) + if instance == 'Fetch nodeinfo and avalaibility': + return instances_menu(fetch_node_info=True) + if instance == 'Remove unreachible instances': + clean_unreach = {} + for ins, info in list_instances.items(): + if 'fail' not in info.split(): + clean_unreach[ins] = info + settings.set_config('public_list_instances_extended', clean_unreach) + return instances_menu() + current_instance.select_instance(instance) diff --git a/src/settings.py b/src/settings.py index f24e426..e88d133 100644 --- a/src/settings.py +++ b/src/settings.py @@ -1,6 +1,4 @@ import json -import requests -import time from os.path import exists from loguru import logger from pyfzf.pyfzf import FzfPrompt @@ -11,36 +9,37 @@ conf_file = 'config.json' default_conf = { 'instance': 'fw.ponychord.rocks', - 'public_list_instances': [ - "open.audio", - "funkwhale.co.uk", - "am.pirateradio.social", - "audio.liberta.vip", - "audio.gafamfree.party", - "tanukitunes.com", - "funkwhale.juniorjpdj.pl", - "audio.securetown.in.ua", - "tavia.mle.party", - "funkwhale.thurk.org", - "buzzworkers.com", - "soundship.de", - "funkwhale.kameha.click", - "music.chosto.me", - "zik.goe.land", - "music.humanoids.be", - "music.hempton.us", - "mizik.o-k-i.net", - "klh.radiolivre.org", - "hudba.feildel.fr", - "funkwhale.mita.me", - "funk.deko.cloud", - "audio.graz.social", - "funkwhale.desmu.fr", - "listen.knsm.cc", - "funkwhale.gegeweb.eu", - "shitnoise.monster" - ], - 'automatic_fetch_new_instances': True, + 'public_list_instances_extended': + { + "open.audio": None, + "funkwhale.co.uk": None, + "am.pirateradio.social": None, + "audio.liberta.vip": None, + "audio.gafamfree.party": None, + "tanukitunes.com": None, + "funkwhale.juniorjpdj.pl": None, + "audio.securetown.in.ua": None, + "tavia.mle.party": None, + "funkwhale.thurk.org": None, + "buzzworkers.com": None, + "soundship.de": None, + "funkwhale.kameha.click": None, + "music.chosto.me": None, + "zik.goe.land": None, + "music.humanoids.be": None, + "music.hempton.us": None, + "mizik.o-k-i.net": None, + "klh.radiolivre.org": None, + "hudba.feildel.fr": None, + "funkwhale.mita.me": None, + "funk.deko.cloud": None, + "audio.graz.social": None, + "funkwhale.desmu.fr": None, + "listen.knsm.cc": None, + "funkwhale.gegeweb.eu": None, + "shitnoise.monster": None, + }, + 'automatic_fetch_new_instances': False, 'enable_server_transcoding': False, 'external_transcoder_http_proxy_path': "", 'track_activity_history': False, @@ -97,61 +96,3 @@ def set_config(key, value): with open(conf_file, 'wt') as f: read_conf[key] = value f.write(json.dumps(read_conf, indent=4)) - -@logger.catch -def get_new_funkwhale_servers(): - # Uses official API network.funkwhale.audio for getting new instances - public_server_api = 'https://network.funkwhale.audio/dashboards/api/tsdb/query' - now = int(time.time()) - timeback = now - 86400 - - request_public_servers = { - 'from': f"{timeback}", - 'to': f"{now}", - 'queries': [ - { - 'refId': "A", - 'intervalMs': 60000, - 'maxDataPoints': 1174, - 'datasourceId': 1, - 'rawSql': "SELECT * FROM (\n SELECT\n DISTINCT on (c.domain) c.domain as \"Name\",\n c.up as \"Is up\",\n coalesce(c.open_registrations, false) as \"Open registrations\",\n coalesce(anonymous_can_listen, false) as \"Anonymous can listen\",\n coalesce(c.usage_users_total, 0) as \"Total users\",\n coalesce(c.usage_users_active_month, 0) as \"Active users (this month)\",\n coalesce(c.software_version_major, 0)::text || '.' || coalesce(c.software_version_minor, 0)::text || '.' || coalesce(c.software_version_patch, 0)::text as \"Version\",\n c.time as \"Last checked\",\n d.first_seen as \"First seen\"\n FROM checks as c\n INNER JOIN domains AS d ON d.name = c.domain\n WHERE d.blocked = false AND c.up = true AND c.time > now() - INTERVAL '7 days'\n AND c.anonymous_can_listen IN ('true')\n AND c.open_registrations IN ('true','false')\n\n ORDER BY c.domain, c.time DESC\n) as t ORDER BY \"Active users (this month)\" DESC", - 'format': "table" - } - ] - } - try: - r = requests.post(public_server_api, json=request_public_servers) - results = r.json() - new_instances = [] - if results: - new_instances_list = results['results']['A']['tables'][0]['rows'] - exists_instances = get_config('public_list_instances') - for i in new_instances_list: - if i[0] not in exists_instances and i[1]: - new_instances.append(i[0]) - new_instances.extend(get_new_funkwhale_servers_fediverse_observer()) - new_instances = list(dict.fromkeys(new_instances)) - return new_instances - except: # If any errors then return empty list - return [] - - -def get_new_funkwhale_servers_fediverse_observer(): - try: - graphQL_request = { - 'query': - '{\n nodes(softwarename: \"funkwhale\") {\n domain\n metanodeinfo\n }\n}' - } - r = requests.post('https://api.fediverse.observer/', - headers={'Accept-Encoding': 'gzip, deflate'}, - json=graphQL_request) - new_instances = [] - exists_instances = get_config('public_list_instances') - for i in r.json()['data']['nodes']: - if i.get('metanodeinfo'): - auth_no_required = json.loads(i['metanodeinfo'])['library']['anonymousCanListen'] - if auth_no_required and i['domain'] not in exists_instances: - new_instances.append(i['domain']) - return new_instances - except: - return []