emdee/libvirt_cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

roles/ansible-gentoo_install/

Browse Source
This commit is contained in:
emdee 2023-12-31 03:19:26 +00:00
parent 94c76b2e4b
commit f2575772ec
11 changed files with 363 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ansible.cfg
View File

@ -1,5 +1,5 @@
[defaults]
log_path = var/tmp/2023/12/30/pentoo/base_proxy_toxcore.log
log_path = var/tmp/2023/12/31/pentoo/base_proxy_toxcore.log
callback_plugins = ./lib/plugins/
# /i/data/DevOps/net/Http/docs.ansible.com/ansible/intro_configuration.html
# http://docs.ansible.com/ansible/intro_configuration.html#command-warnings

30
hosts.yml
View File

@ -219,12 +219,6 @@ all:
BOX_ALSO_USERS:
- gentoo
BOX_PROXY_MODE: "{{lookup('env', 'MODE'|default('tor'}}"
BOX_NBD_DEV: nbd1
BOX_NBD_MP: /mnt/gentoo
BOX_NBD_FILES: "/i/data/Agile/tmp/Topics/GentooImgr"
BOX_NBD_BASE_QCOW: "/g/Agile/tmp/Topics/GentooImgr/gentoo.qcow2"
BOX_NBD_OVERLAY_NAME: "gentoo1"
BOX_NBD_OVERLAY_QCOW: "/o/var/lib/libvirt/images/gentoo1.qcow2"
BOX_GENTOO_FROM_MP: "/mnt/linuxPen19"
linuxPen19:
@ -251,12 +245,6 @@ all:
BOX_BASE_FEATURES: []
BOX_TOXCORE_FEATURES: ['nbd', 'libvirt', 'docker']
BOX_PROXY_MODE: "{{lookup('env', 'MODE'|default('tor'}}"
BOX_NBD_DEV: nbd1
BOX_NBD_MP: /mnt/gentoo
BOX_NBD_OVERLAY_NAME: "gentoo1"
BOX_NBD_BASE_QCOW: "/g/Agile/tmp/Topics/GentooImgr/gentoo.qcow2"
BOX_NBD_OVERLAY_QCOW: "/o/var/lib/libvirt/images/gentoo1.qcow2"
BOX_NBD_FILES: "/i/data/Agile/tmp/Topics/GentooImgr"
# linux_chroot_group vars
vars:
@ -271,7 +259,7 @@ all:
# for a non-root login: ansible_ssh_extra_args: "--userspec=foo:adm"
vars: # linux_unix_group
# toxcore
BOX_NBD_DEV: nbd1
BOX_NBD_DEV: nbd3
BOX_NBD_MP: /mnt/gentoo
BOX_NBD_OVERLAY_NAME: "gentoo1"
BOX_NBD_FILES: "/i/data/Agile/tmp/Topics/GentooImgr"
@ -281,18 +269,21 @@ all:
BOX_NBD_BASE_PROFILE: openrc
BOX_NBD_BASE_DIR: "/a/tmp/GentooImgr"
BOX_NBD_BASE_QCOW: "{{BOX_NBD_BASE_DIR}}/gentoo.qcow2"
# BOX_NBD_OVERLAY_QCOW: "/o/var/lib/libvirt/images/gentoo1.qcow2"
BOX_NBD_OVERLAY_QCOW: "/o/var/lib/libvirt/images/gentoo1.qcow2"
BOX_NBD_BASE_PUBKEY: "/root/.ssh/id_rsa-ansible.pub"
# libvirt overlay
BOX_NBD_OVERLAY_DIR: "/a/tmp/GentooImgr/create-vm"
BOX_NBD_LOGLEVEL: 10
BOX_NBD_OVERLAY_GB: "20"
BOX_NBD_OVERLAY_CPUS: 1
BOX_NBD_OVERLAY_RAM: 2048
BOX_NBD_OVERLAY_BR: virbr1
# unused?
BOX_NBD_OVERLAY_NETWORK: default
# plaintext
BOX_NBD_OVERLAY_PASS: "gentoo"
BOX_GENTOOIMGR_CONFIGFILE: "/g/Agile/tmp/Topics/GentooImgr/base.json"
vars:
@ -386,18 +377,25 @@ all:
BOX_NBD_MP: ""
BOX_NBD_FILES: ""
BOX_NBD_LOGLEVEL: 20
BOX_NBD_PORTAGE_FILE: "{{AGI_NBD_FILES}}/portage-20231223.tar.xz"
BOX_NBD_STAGE3_FILE: "{{AGI_NBD_FILES}}/stage3-amd64-openrc-20231217T170203Z.tar.xz"
BOX_NBD_KERNEL_DIR: /usr/src/linux
BOX_NBD_BASE_PROFILE: openrc
BOX_NBD_BASE_DIR: ""
BOX_NBD_BASE_QCOW: ""
BOX_NBD_BASE_PUBKEY: "/root/.ssh/id_rsa-ansible.pub"
BOX_NBD_BASE_PUBKEY: ""
# libvirt overlay
BOX_NBD_OVERLAY_QCOW: ""
BOX_NBD_OVERLAY_DIR: ""
BOX_NBD_OVERLAY_BR: ""
BOX_NBD_OVERLAY_GB: "20"
BOX_NBD_OVERLAY_NAME: "gentoo1"
BOX_NBD_OVERLAY_NAME: ""
BOX_NBD_OVERLAY_CPUS: 1
BOX_NBD_OVERLAY_RAM: 2048
# plaintext
BOX_NBD_OVERLAY_PASS: ""
BOX_GENTOOIMGR_CONFIGFILE: ""
# Controls what compression method is used for new-style ansible modules when
# they are sent to the remote system. The compression types depend on having

241
library/#ansible_gentooimgr.py# Executable file
View File

@ -0,0 +1,241 @@
#!/usr/bin/python3
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
import sys
import logging
from argparse import Namespace
import pathlib
import traceback
# in the library
mod_path = ''
if os.environ.get('PLAY_ANSIBLE_SRC',''):
# running from source
mod_path = os.environ.get('PLAY_ANSIBLE_SRC','')
mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
assert os.path.isdir(mod_path), f"parent {mod_path}"
assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
sys.path.append(mod_path)
else:
# in the library
mod_path = os.path.dirname(os.path.realpath('__file__'))
mod_path = os.path.join(mod_path, 'src', 'ansible_gentooimgr')
assert os.path.isdir(mod_path), f"parent {mod_path}"
assert os.path.isfile(os.path.join(mod_path, '__init__.py')),f"index {mod_path}"
assert os.path.isdir(os.path.join(mod_path, 'gentooimgr')), f"sub {mod_path}"
sys.path.append(mod_path)
try:
import gentooimgr
except Exception as e:
sys.stderr.write(f"{mod_path} {sys.path} {traceback.print_exc()}")
raise
import ansible
DOCUMENTATION = rf'''
---
module: gentooimgr
short_description: Gentoo Image Builder for Cloud and Turnkey ISO installers
version_added: "1.0.0"
description:
* This project enables easy access to building ``systemd`` or ``openrc`` -based images.
* Performs automatic download AND verification of the linux iso, stage3 tarball and portage.
* Caches the iso and stage3 .txt files for at most a day before redownloading and rechecking for new files
* Sane and readable cli commands to build, run and test.
* Step system to enable user to continue off at the same place if a step fails
* No heavy packages like rust included ** TODO
options:
action:
description: The action to be run by the image builder
choices:
- build
- run
- status
- install
- chroot
- unchroot
- command
- shrink
- kernel
required: true
# clean test
config:
default: cloud.json
description: init configuration file or or base.json or cloud.json
required: false
loglevel:
default: {logging.INFO}
description: python logging level <= 50, INFO=20
required: false
threads:
default: 1
description: Number of threads to use
required: false
profile:
default: openrc
description: The init system
choices:
- openrc
- systemd
required: false
kernel_dir:
default: /usr/src/linux
description: Where kernel is specified. By default uses the active linux kernel
required: false
portage:
description: Extract the specified portage tarball onto the filesystem
required: false
stage3:
description: Extract the specified stage3 package onto the filesystema
required: false
action_args:
default: []
description: Arguments for some of the actions - UNUSED!
required: false
temporary_dir:
description: Path to temporary directory for downloading files (20G)
required: false
qcow:
description: Path to file to serve as the base image
required: false
# Specify this value according to your collection
# in format of namespace.collection.doc_fragment_name
# extends_documentation_fragment:
# - my_namespace.my_collection.my_doc_fragment_name
author:
- Your Name (@yourGitHubHandle)
'''
#[-y DAYS]
# [-d DOWNLOAD_DIR]
# [-f]
# [--format FORMAT]
EXAMPLES = r'''
# Pass in a message
- name: Test with a message
my_namespace.my_collection.my_test:
name: hello world
# pass in a message and have changed true
- name: Test with a message and changed output
my_namespace.my_collection.my_test:
name: hello world
new: true
# fail the module
- name: Test failure of the module
my_namespace.my_collection.my_test:
name: fail me
'''
RETURN = r'''
# These are examples of possible return values, and in general should use other names for return values.
message:
description: The output message that the test module generates.
type: str
returned: always
sample: 'goodbye'
'''
from ansible.module_utils.basic import AnsibleModule
def run_module():
# define available arguments/parameters a user can pass to the module
#? default config from __file__ ?
if mod_path and os.path.isdir(mod_path):
def_config = os.path.join(mod_path, 'configs', 'base.json')
else:
# WARN:
def_config = 'base.json'
module_args = dict(
action=dict(type='str', required=True),
loglevel=dict(type='int', required=False, default=logging.INFO),
threads=dict(type='int', required=False, default=1),
config=dict(type='path', default=def_config, required=True),
profile=dict(type='str', required=False),
kernel_dir=dict(type='path', required=False),
portage=dict(type='path', required=False),
stage3=dict(type='path', required=False),
temporary_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
download_dir=dict(type='path', required=False, default=pathlib.Path(os.getcwd())),
qcow=dict(type='path', required=False),
)
# seed the result dict in the object
# we primarily care about changed and state
# changed is if this module effectively modified the target
# state will include any data that you want your module to pass back
# for consumption, for example, in a subsequent task
result = dict(
changed=False,
original_message='',
message=''
)
# the AnsibleModule object will be our abstraction working with Ansible
# this includes instantiation, a couple of common attr would be the
# args/params passed to the execution, as well as if the module
# supports check mode
module = AnsibleModule(
argument_spec=module_args,
supports_check_mode=True
)
# if the user is working with this module in only check mode we do not
# want to make any changes to the environment, just return the current
# state with no modifications
if module.check_mode:
module.exit_json(**result)
# manipulate or modify the state as needed (this is going to be the
# part where your module will do what it needs to do)
# if module.params.get('thirsty'):
oargs = Namespace(**module.params)
# during the execution of the module, if there is an exception or a
# conditional state that effectively causes a failure, run
# AnsibleModule.fail_json() to pass in the message and the result
result['original_message'] = ""
try:
from gentooimgr.__main__ import main
retval = main(oargs)
# should be 0
# is stdout already in result? how can it be?
except Exception as e:
result['message'] = str(e)
e = traceback.print_exc()
if e: result['original_message'] += f"{e}"
module.fail_json(msg='Exception', **result)
else:
result['message'] = str(retval)
# use whatever logic you need to determine whether or not this module
# made any modifications to your target
if dArgs['action'] in ['status', '']:
result['changed'] = False
else:
result['changed'] = True
# in the event of a successful module execution, you will want to
# simple AnsibleModule.exit_json(), passing the key/value results
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

1
library/.#ansible_gentooimgr.py Symbolic link
View File

@ -0,0 +1 @@
root@pentoo.152064:1703733868

14
library/ansible_gentooimgr.py
View File

@ -11,6 +11,7 @@ import pathlib
import traceback
# in the library
mod_path = ''
if os.environ.get('PLAY_ANSIBLE_SRC',''):
# running from source
mod_path = os.environ.get('PLAY_ANSIBLE_SRC','')
@ -152,11 +153,17 @@ from ansible.module_utils.basic import AnsibleModule
def run_module():
# define available arguments/parameters a user can pass to the module
#? default config from __file__ ?
if mod_path and os.path.isdir(mod_path):
def_config = os.path.join(mod_path, 'configs', 'base.json')
else:
# WARN:
def_config = 'base.json'
module_args = dict(
action=dict(type='str', required=True),
loglevel=dict(type='int', required=False, default=logging.INFO),
threads=dict(type='int', required=False, default=1),
config=dict(type='str', default='cloud.json', required=False),
config=dict(type='path', default=def_config, required=True),
profile=dict(type='str', required=False),
kernel_dir=dict(type='path', required=False),
portage=dict(type='path', required=False),
@ -204,6 +211,8 @@ def run_module():
try:
from gentooimgr.__main__ import main
retval = main(oargs)
# should be 0
# is stdout already in result? how can it be?
except Exception as e:
result['message'] = str(e)
e = traceback.print_exc()
@ -214,7 +223,7 @@ def run_module():
# use whatever logic you need to determine whether or not this module
# made any modifications to your target
if dArgs['action'] in ['status']:
if dArgs['action'] in ['status', '']:
result['changed'] = False
else:
result['changed'] = True
@ -227,7 +236,6 @@ def run_module():
def main():
run_module()
if __name__ == '__main__':
main()

3
roles/ansible-gentoo_install/defaults/main.yml
View File

@ -8,8 +8,11 @@ AGI_NBD_MP: "{{BOX_NBD_MP|default('/mnt/gentoo')}}"
AGI_NBD_FILES: "{{BOX_NBD_FILES}}"
AGI_GENTOO_FROM_MP: "{{BOX_GENTOO_FROM_MP}}"
AGI_GENTOOIMGR_CONFIGFILE: "{{AGI_NBD_FILES}}/base.json"
AGI_PROXY_MODE: "{{PROXY_MODE|default('')}}"
AGI_use_local_kernel: false
AGI_install_disklabel: msdos
AGI_install_timezone: UTC
AGI_install_locales:

8
roles/ansible-gentoo_install/tasks/copy.yml
View File

@ -103,12 +103,12 @@
shell: |
cd {{AGI_GENTOO_FROM_MP}} || exit 1
for dir in {{AGI_bootstrap_dirs}}; do
[ -d "{{AGI_NBD_MP}}/{{dir}}" ] && continue
mkdir -p "{{AGI_NBD_MP}}/{{dir}}"
[ -d "{{AGI_NBD_MP}}/$dir" ] && continue
mkdir -p "{{AGI_NBD_MP}}/$dir"
done
for file in {{AGI_bootstrap_files}}; do
[ -f "{{AGI_NBD_MP}}/{{file}}" ] && continue
cp -np "$file" "{{AGI_NBD_MP}}/{{file}}"
[ -f "{{AGI_NBD_MP}}/$file" ] && continue
cp -np "$file" "{{AGI_NBD_MP}}/$file"
done
# dracut

73
roles/ansible-gentoo_install/tasks/main.yml
View File

@ -9,13 +9,6 @@
- set_fact:
AGI_target: Gentoo2
- name: "ansible-gentoo_install"
set_fact:
ansible_check_mode: false
when:
- "'ansible-gentoo_install' in ROLES"
- ansible_check_mode is true
- name: look for nbd partitions
shell: |
grep nbd /proc/partitions | head -1|sed -e 's/.* //'
@ -40,7 +33,7 @@
[ -d '{{PLAY_ANSIBLE_SRC}}/src/ansible_gentooimgr/gentooimgr' ] || exit 3
[ -f '{{PLAY_ANSIBLE_SRC}}/src/ansible_gentooimgr/gentooimgr/__init__.py' ] || exit 4
{{ansible_python_interpreter}} \
-c "import os sys; sys.path.append('{{PLAY_ANSIBLE_SRC}}/src/ansible_gentooimgr'); import gentooimgr; print(os.path.dirname(gentooimgr.__file__))"
-c "import os, sys; sys.path.append('{{PLAY_ANSIBLE_SRC}}/src/ansible_gentooimgr'); import gentooimgr; print(os.path.dirname(gentooimgr.__file__))" || exit 5
register: gentooimgr_out
check_mode: false
ignore_errors: true
@ -49,18 +42,72 @@
- name: set AGI_gentooimgr_configs
set_fact:
AGI_gentooimgr_configs: "{{gentooimgr_out.stdout}}/configs"
AGI_gentooimgr_configs: "{{gentooimgr_out.stdout_lines[-1]}}/configs"
- name: ansible_gentooimgr nbd status
- name: check for a gentooimgr base config
shell: |
if [ -z "{{BOX_NBD_DEV}}" ] ; then
echo ERROR: empty "BOX_NBD_DEV" - define it in hosts.yml
exit 1
fi
if ! grep -q "{{BOX_NBD_DEV}}" /proc/partitions ; then
echo ERROR: not mounted BOX_NBD_DEV="{{BOX_NBD_DEV}}" - use qemu-nbd
exit 2
fi
exit 0
- name: make a gentooimgr base config
shell: |
base=base.json
tofile="{{AGI_GENTOOIMGR_CONFIGFILE}}"
todir=`dirname "$tofile"`
[ -d $todir ] || mkdir $todir
if [ ! -f "$tofile" ] ; then
if [ -f "{{AGI_gentooimgr_configs}}/$base" ] ; then
cp -p "{{AGI_gentooimgr_configs}}/$base" "$tofile" || exit 2
else
echo WARN: FNF "{{AGI_gentooimgr_configs}}/$base"
# use base.json
fi
fi
# should operate on json with jq
# "imgsize": "20G",
# "memory": 4096,
# "mountpoint": "/mnt/gentoo",
# "imagename": null,
# "initsys": "openrc",
# "iso": null,
# "portage": null,
# "stage3": null,
# "partition": 1
if [ -f "$tofile" ] ; then
# but this is crucial
# "disk": "/dev/sda",
sed -i -e 's@"disk": ".*"@"disk": "'{{BOX_NBD_DEV}}'"@' $tofile
grep {{BOX_NBD_DEV}} $tofile || exit 4
fi
- name: check for library/ansible_gentooimgr.py
shell: |
# find it
[ -f '{{PLAY_ANSIBLE_SRC}}/library/ansible_gentooimgr.py' ] || exit 1
# run it
echo '{}' | \
{{ansible_python_interpreter}} \
'{{PLAY_ANSIBLE_SRC}}/library/ansible_gentooimgr.py' 2>&1| \
grep "Unable to figure out what parameters were passed" && exit 0
echo WARN: ansible_gentooimgr.py BROKEN
- name: ansible_gentooimgr NBD STATUS
ansible_gentooimgr:
action: status
loglevel: "{{BOX_NBD_LOGLEVEL}}"
threads: 1
config: base.config
config: "{{AGI_GENTOOIMGR_CONFIGFILE}}" # base.json - bare filename in configs
profile: "{{BOX_NBD_BASE_PROFILE}}"
kernel_dir: "{{BOX_NBD_KERNEL_DIR}}"
portage: '{{BOX_NBD_PORTAGE_FILE}}'
stage3: '{{BOX_NBD_STAGE3_FILE }}'
portage: "{{BOX_NBD_PORTAGE_FILE}}"
stage3: "{{BOX_NBD_STAGE3_FILE }}"
temporary_dir: "{{BOX_NBD_BASE_DIR}}"
download_dir: "{{AGI_NBD_FILES}}"
register: ansible_gentooimgr_out

30
roles/ansible-gentoo_install/tasks/tarball.yml
View File

@ -12,20 +12,6 @@
[ -d "{{AGI_NBD_MP}}" ] || exit 3
check_mode: false
- name: gpg keys system
# Option --keyserver must be used to
environment: "{{proxy_env}}"
shell: |
/usr/bin/gpg --list-keys | grep "{{ item.uid }}" || \
/usr/bin/gpg --recv-keys \
--keyserver "{{ AGI_GPG_SERVER }}" "{{ item.uid }}"
with_items: "{{ agi_gpg_keys_system }}"
when:
- agi_gpg_keys_system|length > 0
- BASE_ARE_CONNECTED|default('') != ''
# FixMe:
ignore_errors: true
- name: check files dir
environment: "{{shell_env}}"
shell: |
@ -138,6 +124,20 @@
when:
- "ansible_distribution == 'Gentoo'"
- name: gpg keys system
# Option --keyserver must be used to
environment: "{{proxy_env}}"
shell: |
/usr/bin/gpg --list-keys | grep "{{ item.uid }}" || \
/usr/bin/gpg --recv-keys \
--keyserver "{{ AGI_GPG_SERVER }}" "{{ item.uid }}"
with_items: "{{ agi_gpg_keys_system }}"
when:
- agi_gpg_keys_system|length > 0
- BASE_ARE_CONNECTED|default('') != ''
# FixMe:
ignore_errors: true
- name: test portage tarball gpg our copy
environment: "{{shell_env}}"
shell: |
@ -145,6 +145,8 @@
gpg --list-keys | grep E1D6ABB63BFCFB4BA02FDF1CEC590EEAC9189250 || exit 2
gpg --verify "{{AGI_NBD_FILES}}/{{ latest_portage_tarball }}.gpgsig" \
"{{AGI_NBD_FILES}}/{{ latest_portage_tarball }}" || exit 3$?
# FixMe:
ignore_errors: true
- name: extract portage tarball
unarchive:

2
roles/ansible-gentoo_install/vars/target_Gentoo2.yml
View File

@ -69,6 +69,7 @@ AGI_bootstrap_pkgs:
- sys-libs/gpm
- app-portage/eix
- www-client/lynx
- linux-firmware
AGI_cloud_pkgs:
# get these from base.json
@ -87,7 +88,6 @@ AGI_cloud_pkgs:
- pciutils
- net-misc/ntp
- net-fs/nfs-utils
- linux-firmware
# get these from config.json
- app-emulation/cloud-init
- sys-block/open-iscsi

20
src/ansible_gentooimgr/README.md
View File

@ -2,16 +2,18 @@ GentooImgr: Gentoo Image Builder for Cloud and Turnkey ISO installers
=====================================================================
**This is a modified version of https://github.com/NucleaPeon/gentooimgr/
where we've modified the code a little to do use Python logging. We can
still use it for the build stage, but we think the install stage is better
done using ansible, hence the libvirt_cloud playbook.
where we've modified the code a little to do use Python logging and maybe fixed a bug or two. We can still use it for the build stage, but we think the install stage is better done using ansible, hence the libvirt_cloud playbook; you don't check return values in your install steps, and I'm confused on the precedence of the config json file.
I think you should merge args into the configjson object and have the code work only on the merged object: the precedence shouldn't be down in the code.
The code is a WIP so look to see if the repo is active. It's being supported as an ansible module using library/ansible_gentooimgr.py which is a work in progress; the idea is to use it for the build and status test? actions, but handle the install tasks using ansible.
Instead of using cloud-init, our approach is to build a base image with qemu-guest-agent installed which means you don't need cloud-init. That means you don't need 384 crates of opaque googled code. Apart from making the base smaller, it makes it auditable, I can't imaging why the libvirt base images on the Internet don't include qemu-guest-agent.........
Take any of the code you want back into your repo: I think the changes are all logging and bulletproofing. If you use ansible you can maintain the library file. If you don't maybe it a good time to look at it, because writing the install in Python will be painfully limiting, and it may take you a while to write tests.py.
Please wrap the code after if __name__ == "__main__" in __main__.py into a function that returns an int, 0 on success so it's callable from an import.
The code is in overlay/Linux/usr/local/src/ansible_gentooimgr
for reasons to do with how the ansible role transfers files.
The code is being supported as an ansible module using
library/ansible_gentooimgr.py which is a work in progress; the idea
is to use it for the build and status actions, but handle the install
tasks using ansible.
**