added logging
This commit is contained in:
parent
f7303dce15
commit
06cffbdbd7
174
.gitignore
vendored
Normal file
174
.gitignore
vendored
Normal file
@ -0,0 +1,174 @@
|
|||||||
|
# ---> Python
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
*.diff
|
||||||
|
*.good
|
||||||
|
*.bad
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py,cover
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
#Pipfile.lock
|
||||||
|
|
||||||
|
# poetry
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
|
#poetry.lock
|
||||||
|
|
||||||
|
# pdm
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||||
|
#pdm.lock
|
||||||
|
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||||
|
# in version control.
|
||||||
|
# https://pdm.fming.dev/#use-with-ide
|
||||||
|
.pdm.toml
|
||||||
|
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||||
|
__pypackages__/
|
||||||
|
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule
|
||||||
|
celerybeat.pid
|
||||||
|
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
|
||||||
|
# mypy
|
||||||
|
.mypy_cache/
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
|
||||||
|
# PyCharm
|
||||||
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
#.idea/
|
||||||
|
|
||||||
|
.pylint.err
|
||||||
|
.pylint.log
|
||||||
|
.pylint.out
|
||||||
|
|
||||||
|
*.dst
|
||||||
|
|
||||||
|
*~
|
||||||
|
.rsync.sh
|
||||||
|
.rsync.sh
|
@ -1,2 +1,5 @@
|
|||||||
import os
|
import os
|
||||||
HERE = os.path.abspath(os.path.dirname(__file__))
|
HERE = os.path.abspath(os.path.dirname(__file__))
|
||||||
|
import logging
|
||||||
|
LOG = logging.getLogger('GI ')
|
||||||
|
logging.basicConfig(level=logging.INFO) # oArgs.loglevel) #
|
||||||
|
@ -4,6 +4,16 @@ import json
|
|||||||
import argparse
|
import argparse
|
||||||
import pathlib
|
import pathlib
|
||||||
import copy
|
import copy
|
||||||
|
import logging
|
||||||
|
|
||||||
|
try:
|
||||||
|
import coloredlogs
|
||||||
|
os.environ['COLOREDLOGS_LEVEL_STYLES'] = 'spam=22;debug=28;verbose=34;notice=220;warning=202;success=118,bold;error=124;critical=background=red'
|
||||||
|
except ImportError as e:
|
||||||
|
logging.log(logging.DEBUG, f"coloredlogs not available: {e}")
|
||||||
|
coloredlogs = None
|
||||||
|
|
||||||
|
from gentooimgr import LOG
|
||||||
import gentooimgr.common
|
import gentooimgr.common
|
||||||
import gentooimgr.config
|
import gentooimgr.config
|
||||||
import gentooimgr.configs
|
import gentooimgr.configs
|
||||||
@ -13,7 +23,9 @@ def main(args):
|
|||||||
'''Gentoo Cloud Image Builder Utility'''
|
'''Gentoo Cloud Image Builder Utility'''
|
||||||
import gentooimgr.config
|
import gentooimgr.config
|
||||||
configjson = gentooimgr.config.determine_config(args)
|
configjson = gentooimgr.config.determine_config(args)
|
||||||
|
prefix = args.temporary_dir
|
||||||
|
|
||||||
|
LOG.info(f'Gentoo Cloud Image Builder Utility {args.action}')
|
||||||
if args.action == "build":
|
if args.action == "build":
|
||||||
import gentooimgr.builder
|
import gentooimgr.builder
|
||||||
gentooimgr.builder.build(args, configjson)
|
gentooimgr.builder.build(args, configjson)
|
||||||
@ -23,9 +35,11 @@ def main(args):
|
|||||||
gentooimgr.run.run(args, configjson)
|
gentooimgr.run.run(args, configjson)
|
||||||
|
|
||||||
elif args.action == "test":
|
elif args.action == "test":
|
||||||
|
# empty
|
||||||
import gentooimgr.test
|
import gentooimgr.test
|
||||||
|
|
||||||
elif args.action == "clean":
|
elif args.action == "clean":
|
||||||
|
# empty
|
||||||
import gentooimgr.clean
|
import gentooimgr.clean
|
||||||
|
|
||||||
elif args.action == "status":
|
elif args.action == "status":
|
||||||
@ -56,6 +70,7 @@ def main(args):
|
|||||||
elif args.action == "kernel":
|
elif args.action == "kernel":
|
||||||
import gentooimgr.kernel
|
import gentooimgr.kernel
|
||||||
gentooimgr.kernel.build_kernel(args, configjson)
|
gentooimgr.kernel.build_kernel(args, configjson)
|
||||||
|
return 0
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
"""Gentoo Cloud Image Builder Utility"""
|
"""Gentoo Cloud Image Builder Utility"""
|
||||||
@ -68,9 +83,13 @@ if __name__ == "__main__":
|
|||||||
help="Use a minimal base Gentoo configuration")
|
help="Use a minimal base Gentoo configuration")
|
||||||
|
|
||||||
parser.add_argument("-t", "--temporary-dir", nargs='?', type=pathlib.Path,
|
parser.add_argument("-t", "--temporary-dir", nargs='?', type=pathlib.Path,
|
||||||
default=os.getcwd(), help="Path to temporary directory for downloading files")
|
default=pathlib.Path(os.getcwd()), help="Path to temporary directory for downloading files")
|
||||||
parser.add_argument("-j", "--threads", type=int, default=gentooimgr.config.THREADS,
|
parser.add_argument("-j", "--threads", type=int, default=gentooimgr.config.THREADS,
|
||||||
help="Number of threads to use for building and emerging software")
|
help="Number of threads to use for building and emerging software")
|
||||||
|
parser.add_argument("-l", "--loglevel", type=int, default=logging.INFO,
|
||||||
|
help="python logging level <= 50, INFO=20")
|
||||||
|
parser.add_argument("-y", "--days", type=int, default=7, # gentooimgr.config.DAYS
|
||||||
|
help="Number of days before the files are redownloaded")
|
||||||
parser.add_argument("-d", "--download-dir", type=pathlib.Path, default=os.getcwd(),
|
parser.add_argument("-d", "--download-dir", type=pathlib.Path, default=os.getcwd(),
|
||||||
help="Path to the desired download directory (default: current)")
|
help="Path to the desired download directory (default: current)")
|
||||||
parser.add_argument("--openrc", dest="profile", action="store_const", const="openrc",
|
parser.add_argument("--openrc", dest="profile", action="store_const", const="openrc",
|
||||||
@ -145,9 +164,21 @@ if __name__ == "__main__":
|
|||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
assert args.loglevel < 59
|
||||||
|
if coloredlogs:
|
||||||
|
# https://pypi.org/project/coloredlogs/
|
||||||
|
coloredlogs.install(level=args.loglevel,
|
||||||
|
logger=LOG,
|
||||||
|
# %(asctime)s,%(msecs)03d %(hostname)s [%(process)d]
|
||||||
|
fmt='%(name)s %(levelname)s %(message)s'
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logging.basicConfig(level=args.loglevel) # logging.INFO
|
||||||
|
logging.basicConfig(level=args.loglevel)
|
||||||
|
|
||||||
isos = gentooimgr.common.find_iso(args.download_dir)
|
isos = gentooimgr.common.find_iso(args.download_dir)
|
||||||
if args.action == "run" and args.iso is None and len(isos) > 1:
|
if args.action == "run" and args.iso is None and len(isos) > 1:
|
||||||
print(f"Error: multiple iso files were found in {args.download_dir}, please specify one using `--iso [iso]`")
|
LOG.error(f"Error: multiple iso files were found in {args.download_dir}, please specify one using `--iso [iso]`")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,5 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
import argparse
|
import argparse
|
||||||
|
|
||||||
|
from gentooimgr import LOG
|
||||||
import gentooimgr.config as config
|
import gentooimgr.config as config
|
||||||
import gentooimgr.download as download
|
import gentooimgr.download as download
|
||||||
import gentooimgr.qemu as qemu
|
import gentooimgr.qemu as qemu
|
||||||
@ -7,6 +9,7 @@ import gentooimgr.common
|
|||||||
import requests
|
import requests
|
||||||
|
|
||||||
def build(args: argparse.Namespace, config: dict) -> None:
|
def build(args: argparse.Namespace, config: dict) -> None:
|
||||||
|
LOG.info(": build")
|
||||||
|
|
||||||
iso = config.get("iso") or download.download(args)
|
iso = config.get("iso") or download.download(args)
|
||||||
stage3 = config.get("stage3") or download.download_stage3(args)
|
stage3 = config.get("stage3") or download.download_stage3(args)
|
||||||
@ -17,5 +20,7 @@ def build(args: argparse.Namespace, config: dict) -> None:
|
|||||||
raise Exception(f"Image {image} does not exist")
|
raise Exception(f"Image {image} does not exist")
|
||||||
|
|
||||||
is_default = os.path.basename(image) == filename
|
is_default = os.path.basename(image) == filename
|
||||||
print(image)
|
LOG.info(image)
|
||||||
print(f"Image {image} build successfully.\nRun `python -m gentooimgr run{' ' + image if not is_default else ''} --iso {iso}`")
|
LOG.info(f"Image {image} build successfully.\nRun `python -m gentooimgr run{' ' + image if not is_default else ''} --iso {iso}`")
|
||||||
|
|
||||||
|
return image
|
||||||
|
@ -6,12 +6,15 @@ import json
|
|||||||
from subprocess import Popen, PIPE
|
from subprocess import Popen, PIPE
|
||||||
|
|
||||||
import gentooimgr.config
|
import gentooimgr.config
|
||||||
|
from gentooimgr import LOG
|
||||||
|
|
||||||
def older_than_a_day(fullpath):
|
def older_than_a_day(fullpath):
|
||||||
if not os.path.exists(fullpath):
|
if not os.path.exists(fullpath):
|
||||||
return True # Don't fail on missing files
|
return True # Don't fail on missing files
|
||||||
filetime = os.path.getmtime(fullpath)
|
filetime = os.path.getmtime(fullpath)
|
||||||
return time.time() - filetime > gentooimgr.config.DAY_IN_SECONDS
|
return time.time() - filetime > (gentooimgr.config.DAY_IN_SECONDS *
|
||||||
|
gentooimgr.config.DAYS)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def find_iso(download_dir):
|
def find_iso(download_dir):
|
||||||
@ -65,9 +68,9 @@ def portage_from_dir(d, filename=None):
|
|||||||
found.append(f)
|
found.append(f)
|
||||||
|
|
||||||
if len(found) > 1:
|
if len(found) > 1:
|
||||||
sys.stderr.write("\tEE: More than one portage file exists, please specify the exact portage file with --portage [file] or remove all others\n")
|
LOG.error("\tEE: More than one portage file exists, please specify the exact portage file with --portage [file] or remove all others\n")
|
||||||
sys.stderr.write(''.join([f"\t{f}\n" for f in found]))
|
LOG.error(''.join([f"\t{f}\n" for f in found]))
|
||||||
sys.stderr.write(f"in {d}\n")
|
LOG.error(f"in {d}\n")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
return found[0] if found else None
|
return found[0] if found else None
|
||||||
@ -89,9 +92,9 @@ def stage3_from_dir(d, filename=None):
|
|||||||
found.append(f)
|
found.append(f)
|
||||||
|
|
||||||
if len(found) > 1:
|
if len(found) > 1:
|
||||||
sys.stderr.write("More than one stage3 file exists, please specify the exact stage3 file or remove all others\n")
|
LOG.error("More than one stage3 file exists, please specify the exact stage3 file or remove all others\n")
|
||||||
sys.stderr.write(''.join([f"\t{f}\n" for f in found]))
|
LOG.error(''.join([f"\t{f}\n" for f in found]))
|
||||||
sys.stderr.write(f"in {d}\n")
|
LOG.error(f"in {d}\n")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return found[0] if found else None
|
return found[0] if found else None
|
||||||
|
@ -2,11 +2,15 @@ import os
|
|||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import argparse
|
import argparse
|
||||||
|
|
||||||
|
from gentooimgr import LOG
|
||||||
import gentooimgr.configs
|
import gentooimgr.configs
|
||||||
import multiprocessing
|
import multiprocessing
|
||||||
|
|
||||||
# A day in seconds:
|
# A day in seconds:
|
||||||
DAY_IN_SECONDS = 60*60*24
|
DAY_IN_SECONDS = 60*60*24
|
||||||
|
# days until the iso is old
|
||||||
|
DAYS = 1
|
||||||
# Define threads to compile packages with
|
# Define threads to compile packages with
|
||||||
THREADS = multiprocessing.cpu_count()
|
THREADS = multiprocessing.cpu_count()
|
||||||
# URL to latest image text file, defaults to amd64. This is parsed to find latest iso to download
|
# URL to latest image text file, defaults to amd64. This is parsed to find latest iso to download
|
||||||
@ -37,9 +41,14 @@ CLOUD_MODULES = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
def load_config(path):
|
def load_config(path):
|
||||||
|
assert path, "load config called with nothing"
|
||||||
if os.path.exists(path):
|
if os.path.exists(path):
|
||||||
with open(path, 'r') as f:
|
with open(path, 'r') as f:
|
||||||
return json.loads(f.read())
|
try:
|
||||||
|
return json.loads(f.read())
|
||||||
|
except Exception as e:
|
||||||
|
LOG.error(f"ERROR loading {path}")
|
||||||
|
raise
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
def load_default_config(config_name):
|
def load_default_config(config_name):
|
||||||
@ -51,9 +60,14 @@ def load_default_config(config_name):
|
|||||||
if not name in gentooimgr.configs.KNOWN_CONFIGS:
|
if not name in gentooimgr.configs.KNOWN_CONFIGS:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
with open(os.path.join(gentooimgr.configs.CONFIG_DIR, config_name), 'r') as f:
|
json_file = os.path.join(gentooimgr.configs.CONFIG_DIR, config_name)
|
||||||
return json.loads(f.read())
|
ret = {}
|
||||||
|
with open(json_file, 'r') as f:
|
||||||
|
try:
|
||||||
|
ret = json.loads(f.read())
|
||||||
|
except Exception as e:
|
||||||
|
LOG.error(f"loading {json_file} {e}")
|
||||||
|
return ret
|
||||||
|
|
||||||
def inherit_config(config: dict) -> dict:
|
def inherit_config(config: dict) -> dict:
|
||||||
"""Returns the json file that the inherit key specifies; will recursively update if inherit values are set.
|
"""Returns the json file that the inherit key specifies; will recursively update if inherit values are set.
|
||||||
@ -95,10 +109,10 @@ def determine_config(args: argparse.Namespace) -> dict:
|
|||||||
|
|
||||||
# Check custom configuration
|
# Check custom configuration
|
||||||
configuration = load_default_config(args.config or 'base.json')
|
configuration = load_default_config(args.config or 'base.json')
|
||||||
if not configuration:
|
if not configuration and args.config:
|
||||||
configuration = load_config(args.config)
|
configuration = load_config(args.config)
|
||||||
if not configuration:
|
if not configuration:
|
||||||
sys.stderr.write(f"\tWW: Warning: Configuration {args.config} is empty\n")
|
LOG.error(f"\tWW: Warning: Configuration {args.config} is empty\n")
|
||||||
else:
|
else:
|
||||||
if configuration.get("inherit"):
|
if configuration.get("inherit"):
|
||||||
# newpkgs = configuration.get("packages", {})
|
# newpkgs = configuration.get("packages", {})
|
||||||
|
@ -45,6 +45,7 @@
|
|||||||
"sys-block/gpart",
|
"sys-block/gpart",
|
||||||
"net-misc/ntp",
|
"net-misc/ntp",
|
||||||
"net-fs/nfs-utils",
|
"net-fs/nfs-utils",
|
||||||
|
"app-emulation/qemu-guest-agent",
|
||||||
"linux-firmware"
|
"linux-firmware"
|
||||||
],
|
],
|
||||||
"additional": ["app-editors/vim"],
|
"additional": ["app-editors/vim"],
|
||||||
@ -72,6 +73,7 @@
|
|||||||
"cronie": "default",
|
"cronie": "default",
|
||||||
"acpid": "default",
|
"acpid": "default",
|
||||||
"ntp": "default"
|
"ntp": "default"
|
||||||
|
"qemu-guest-agent": "default"
|
||||||
},
|
},
|
||||||
"iso": null,
|
"iso": null,
|
||||||
"portage": null,
|
"portage": null,
|
||||||
|
@ -3,7 +3,9 @@
|
|||||||
"packages": {
|
"packages": {
|
||||||
"additional": [
|
"additional": [
|
||||||
"app-emulation/cloud-init",
|
"app-emulation/cloud-init",
|
||||||
"sys-block/open-iscsi"
|
"sys-block/open-iscsi",
|
||||||
|
"app-editors/mg",
|
||||||
|
"net-analyzer/openbsd-netcat"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"disk": "/dev/vda",
|
"disk": "/dev/vda",
|
||||||
|
@ -14,6 +14,8 @@ import hashlib
|
|||||||
import progressbar
|
import progressbar
|
||||||
from urllib.request import urlretrieve
|
from urllib.request import urlretrieve
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|
||||||
|
from gentooimgr import LOG
|
||||||
import gentooimgr.config as config
|
import gentooimgr.config as config
|
||||||
from gentooimgr.common import older_than_a_day
|
from gentooimgr.common import older_than_a_day
|
||||||
|
|
||||||
@ -75,12 +77,14 @@ def verify(args, _type: str, baseurl: str, hashpattern, filename: str) -> bool:
|
|||||||
Whether iso was verified using the specified hash
|
Whether iso was verified using the specified hash
|
||||||
|
|
||||||
"""
|
"""
|
||||||
digest = hashlib.file_digest(open(os.path.join(args.download_dir, filename), 'rb'), _type.lower())
|
thefile = os.path.join(args.download_dir, filename)
|
||||||
|
LOG.info(f"verifying hash of {thefile}")
|
||||||
|
digest = hashlib.file_digest(open(thefile, 'rb'), _type.lower())
|
||||||
filename = filename+f".{_type.lower()}" # Update to hash file
|
filename = filename+f".{_type.lower()}" # Update to hash file
|
||||||
hashfile = os.path.join(baseurl, filename)
|
hashfile = os.path.join(baseurl, filename)
|
||||||
fullpath = os.path.join(args.download_dir, os.path.basename(hashfile))
|
fullpath = os.path.join(args.download_dir, os.path.basename(hashfile))
|
||||||
if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
|
if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
|
||||||
print(f"Downloading {filename}")
|
LOG.info(f"Downloading {filename}")
|
||||||
urlretrieve(hashfile, fullpath, DownloadProgressBar())
|
urlretrieve(hashfile, fullpath, DownloadProgressBar())
|
||||||
|
|
||||||
hd = digest.hexdigest()
|
hd = digest.hexdigest()
|
||||||
@ -110,7 +114,7 @@ def download_stage3(args, url=None) -> str:
|
|||||||
filename = latest
|
filename = latest
|
||||||
fullpath = os.path.join(args.download_dir, filename)
|
fullpath = os.path.join(args.download_dir, filename)
|
||||||
if not os.path.exists(fullpath) or args.redownload:
|
if not os.path.exists(fullpath) or args.redownload:
|
||||||
print(f"Downloading {filename}")
|
LOG.info(f"Downloading {filename}")
|
||||||
url = os.path.join(
|
url = os.path.join(
|
||||||
config.GENTOO_BASE_STAGE_SYSTEMD_URL if args.profile == "systemd" else \
|
config.GENTOO_BASE_STAGE_SYSTEMD_URL if args.profile == "systemd" else \
|
||||||
config.GENTOO_BASE_STAGE_OPENRC_URL,
|
config.GENTOO_BASE_STAGE_OPENRC_URL,
|
||||||
@ -145,7 +149,7 @@ def download_portage(args, url=None) -> str:
|
|||||||
fullpath = os.path.join(args.download_dir, filename)
|
fullpath = os.path.join(args.download_dir, filename)
|
||||||
# Portage is always "latest" in this case, so definitely check if older than a day and redownload.
|
# Portage is always "latest" in this case, so definitely check if older than a day and redownload.
|
||||||
if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
|
if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
|
||||||
print(f"Downloading {filename} ({base})")
|
LOG.info(f"Downloading {filename} ({base})")
|
||||||
urlretrieve(url, fullpath, DownloadProgressBar())
|
urlretrieve(url, fullpath, DownloadProgressBar())
|
||||||
|
|
||||||
return fullpath
|
return fullpath
|
||||||
@ -169,7 +173,7 @@ def download(args, url=None) -> str:
|
|||||||
filename = os.path.basename(url)
|
filename = os.path.basename(url)
|
||||||
fullpath = os.path.join(args.download_dir, filename)
|
fullpath = os.path.join(args.download_dir, filename)
|
||||||
if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
|
if not os.path.exists(fullpath) or args.redownload or older_than_a_day(fullpath):
|
||||||
print(f"Downloading {filename}")
|
LOG.info(f"Downloading {fullpath}")
|
||||||
urlretrieve(url, fullpath, DownloadProgressBar())
|
urlretrieve(url, fullpath, DownloadProgressBar())
|
||||||
|
|
||||||
hashtype, latest, size = parse_latest_iso_text(fullpath)
|
hashtype, latest, size = parse_latest_iso_text(fullpath)
|
||||||
@ -178,8 +182,8 @@ def download(args, url=None) -> str:
|
|||||||
# Download the iso file
|
# Download the iso file
|
||||||
filename = latest
|
filename = latest
|
||||||
fullpath = os.path.join(args.download_dir, filename)
|
fullpath = os.path.join(args.download_dir, filename)
|
||||||
if not os.path.exists(fullpath) or args.redownload :
|
if not os.path.exists(fullpath) or args.redownload:
|
||||||
print(f"Downloading {filename}")
|
LOG.info(f"Downloading {filename}")
|
||||||
url = os.path.join(config.GENTOO_BASE_ISO_URL, filename)
|
url = os.path.join(config.GENTOO_BASE_ISO_URL, filename)
|
||||||
urlretrieve(url, fullpath, DownloadProgressBar())
|
urlretrieve(url, fullpath, DownloadProgressBar())
|
||||||
|
|
||||||
|
@ -9,11 +9,15 @@ import sys
|
|||||||
import shutil
|
import shutil
|
||||||
import configparser
|
import configparser
|
||||||
from subprocess import Popen, PIPE
|
from subprocess import Popen, PIPE
|
||||||
|
import logging
|
||||||
|
import traceback
|
||||||
|
|
||||||
import gentooimgr.config
|
import gentooimgr.config
|
||||||
import gentooimgr.configs
|
import gentooimgr.configs
|
||||||
import gentooimgr.common
|
import gentooimgr.common
|
||||||
import gentooimgr.chroot
|
import gentooimgr.chroot
|
||||||
import gentooimgr.kernel
|
import gentooimgr.kernel
|
||||||
|
from gentooimgr import LOG
|
||||||
from gentooimgr import HERE
|
from gentooimgr import HERE
|
||||||
|
|
||||||
from gentooimgr.configs import *
|
from gentooimgr.configs import *
|
||||||
@ -21,7 +25,7 @@ from gentooimgr.configs import *
|
|||||||
FILES_DIR = os.path.join(HERE, "..")
|
FILES_DIR = os.path.join(HERE, "..")
|
||||||
|
|
||||||
def step1_diskprep(args, cfg):
|
def step1_diskprep(args, cfg):
|
||||||
print("\t:: Step 1: Disk Partitioning")
|
LOG.info("\t:: Step 1: Disk Partitioning")
|
||||||
# http://rainbow.chard.org/2013/01/30/how-to-align-partitions-for-best-performance-using-parted/
|
# http://rainbow.chard.org/2013/01/30/how-to-align-partitions-for-best-performance-using-parted/
|
||||||
# http://honglus.blogspot.com/2013/06/script-to-automatically-partition-new.html
|
# http://honglus.blogspot.com/2013/06/script-to-automatically-partition-new.html
|
||||||
cmds = [
|
cmds = [
|
||||||
@ -37,13 +41,13 @@ def step1_diskprep(args, cfg):
|
|||||||
completestep(1, "diskprep")
|
completestep(1, "diskprep")
|
||||||
|
|
||||||
def step2_mount(args, cfg):
|
def step2_mount(args, cfg):
|
||||||
print(f'\t:: Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}')
|
LOG.info(f'\t:: Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}')
|
||||||
proc = Popen(["mount", f'{cfg.get("disk")}{cfg.get("partition")}', cfg.get("mountpoint")])
|
proc = Popen(["mount", f'{cfg.get("disk")}{cfg.get("partition")}', cfg.get("mountpoint")])
|
||||||
proc.communicate()
|
proc.communicate()
|
||||||
completestep(2, "mount")
|
completestep(2, "mount")
|
||||||
|
|
||||||
def step3_stage3(args, cfg):
|
def step3_stage3(args, cfg):
|
||||||
print(f'\t:: Step 3: Stage3 Tarball')
|
LOG.info(f'\t:: Step 3: Stage3 Tarball')
|
||||||
|
|
||||||
stage3 = cfg.get("stage3") or args.stage3 # FIXME: auto detect stage3 images in mountpoint and add here
|
stage3 = cfg.get("stage3") or args.stage3 # FIXME: auto detect stage3 images in mountpoint and add here
|
||||||
if not stage3:
|
if not stage3:
|
||||||
@ -56,12 +60,12 @@ def step3_stage3(args, cfg):
|
|||||||
completestep(3, "stage3")
|
completestep(3, "stage3")
|
||||||
|
|
||||||
def step4_binds(args, cfg):
|
def step4_binds(args, cfg):
|
||||||
print(f'\t:: Step 4: Binding Filesystems')
|
LOG.info(f'\t:: Step 4: Binding Filesystems')
|
||||||
gentooimgr.chroot.bind(verbose=False)
|
gentooimgr.chroot.bind(verbose=False)
|
||||||
completestep(4, "binds")
|
completestep(4, "binds")
|
||||||
|
|
||||||
def step5_portage(args, cfg):
|
def step5_portage(args, cfg):
|
||||||
print(f'\t:: Step 5: Portage')
|
LOG.info(f'\t:: Step 5: Portage')
|
||||||
portage = cfg.get("portage") or args.portage
|
portage = cfg.get("portage") or args.portage
|
||||||
if not portage:
|
if not portage:
|
||||||
portage = gentooimgr.common.portage_from_dir(FILES_DIR)
|
portage = gentooimgr.common.portage_from_dir(FILES_DIR)
|
||||||
@ -80,7 +84,7 @@ def step5_portage(args, cfg):
|
|||||||
completestep(5, "portage")
|
completestep(5, "portage")
|
||||||
|
|
||||||
def step6_licenses(args, cfg):
|
def step6_licenses(args, cfg):
|
||||||
print(f'\t:: Step 6: Licenses')
|
LOG.info(f'\t:: Step 6: Licenses')
|
||||||
license_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.license')
|
license_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'package.license')
|
||||||
os.makedirs(license_path, exist_ok=True)
|
os.makedirs(license_path, exist_ok=True)
|
||||||
for f, licenses in cfg.get("licensefiles", {}).items():
|
for f, licenses in cfg.get("licensefiles", {}).items():
|
||||||
@ -89,7 +93,7 @@ def step6_licenses(args, cfg):
|
|||||||
completestep(6, "license")
|
completestep(6, "license")
|
||||||
|
|
||||||
def step7_repos(args, cfg):
|
def step7_repos(args, cfg):
|
||||||
print(f'\t:: Step 7: Repo Configuration')
|
LOG.info(f'\t:: Step 7: Repo Configuration')
|
||||||
repo_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'repos.conf')
|
repo_path = os.path.join(cfg.get("mountpoint"), 'etc', 'portage', 'repos.conf')
|
||||||
os.makedirs(repo_path, exist_ok=True)
|
os.makedirs(repo_path, exist_ok=True)
|
||||||
# Copy from template
|
# Copy from template
|
||||||
@ -115,7 +119,7 @@ def step7_repos(args, cfg):
|
|||||||
completestep(7, "repos")
|
completestep(7, "repos")
|
||||||
|
|
||||||
def step8_resolv(args, cfg):
|
def step8_resolv(args, cfg):
|
||||||
print(f'\t:: Step 8: Resolv')
|
LOG.info(f'\t:: Step 8: Resolv')
|
||||||
proc = Popen(["cp", "--dereference", "/etc/resolv.conf", os.path.join(cfg.get("mountpoint"), 'etc')])
|
proc = Popen(["cp", "--dereference", "/etc/resolv.conf", os.path.join(cfg.get("mountpoint"), 'etc')])
|
||||||
proc.communicate()
|
proc.communicate()
|
||||||
# Copy all step files and python module to new chroot
|
# Copy all step files and python module to new chroot
|
||||||
@ -124,20 +128,20 @@ def step8_resolv(args, cfg):
|
|||||||
completestep(8, "resolv")
|
completestep(8, "resolv")
|
||||||
|
|
||||||
def step9_sync(args, cfg):
|
def step9_sync(args, cfg):
|
||||||
print(f"\t:: Step 9: sync")
|
LOG.info(f"\t:: Step 9: sync")
|
||||||
print("\t\t:: Entering chroot")
|
LOG.info("\t\t:: Entering chroot")
|
||||||
os.chroot(cfg.get("mountpoint"))
|
os.chroot(cfg.get("mountpoint"))
|
||||||
os.chdir(os.sep)
|
os.chdir(os.sep)
|
||||||
os.system("source /etc/profile")
|
os.system("source /etc/profile")
|
||||||
proc = Popen(["emerge", "--sync", "--quiet"])
|
proc = Popen(["emerge", "--sync", "--quiet"])
|
||||||
proc.communicate()
|
proc.communicate()
|
||||||
print("\t\t:: Emerging base")
|
LOG.info("\t\t:: Emerging base")
|
||||||
proc = Popen(["emerge", "--update", "--deep", "--newuse", "--keep-going", "@world"])
|
proc = Popen(["emerge", "--update", "--deep", "--newuse", "--keep-going", "@world"])
|
||||||
proc.communicate()
|
proc.communicate()
|
||||||
completestep(9, "sync")
|
completestep(9, "sync")
|
||||||
|
|
||||||
def step10_emerge_pkgs(args, cfg):
|
def step10_emerge_pkgs(args, cfg):
|
||||||
print(f"\t:: Step 10: emerge pkgs")
|
LOG.info(f"\t:: Step 10: emerge pkgs")
|
||||||
packages = cfg.get("packages", {})
|
packages = cfg.get("packages", {})
|
||||||
for oneshot_up in packages.get("oneshots", []):
|
for oneshot_up in packages.get("oneshots", []):
|
||||||
proc = Popen(["emerge", "--oneshot", "--update", oneshot_up])
|
proc = Popen(["emerge", "--oneshot", "--update", oneshot_up])
|
||||||
@ -147,7 +151,7 @@ def step10_emerge_pkgs(args, cfg):
|
|||||||
proc = Popen(["emerge", "-j1", single])
|
proc = Popen(["emerge", "-j1", single])
|
||||||
proc.communicate()
|
proc.communicate()
|
||||||
|
|
||||||
print("KERNEL PACKAGES", packages.get("kernel"))
|
LOG.info("KERNEL PACKAGES", packages.get("kernel"))
|
||||||
if packages.get("kernel", []):
|
if packages.get("kernel", []):
|
||||||
cmd = ["emerge", "-j", str(args.threads)] + packages.get("kernel", [])
|
cmd = ["emerge", "-j", str(args.threads)] + packages.get("kernel", [])
|
||||||
proc = Popen(cmd)
|
proc = Popen(cmd)
|
||||||
@ -162,14 +166,14 @@ def step10_emerge_pkgs(args, cfg):
|
|||||||
cmd += packages.get("base", [])
|
cmd += packages.get("base", [])
|
||||||
cmd += packages.get("additional", [])
|
cmd += packages.get("additional", [])
|
||||||
cmd += packages.get("bootloader", [])
|
cmd += packages.get("bootloader", [])
|
||||||
print(cmd)
|
LOG.info(cmd)
|
||||||
proc = Popen(cmd)
|
proc = Popen(cmd)
|
||||||
proc.communicate()
|
proc.communicate()
|
||||||
completestep(10, "pkgs")
|
completestep(10, "pkgs")
|
||||||
|
|
||||||
def step11_kernel(args, cfg):
|
def step11_kernel(args, cfg):
|
||||||
# at this point, genkernel will be installed
|
# at this point, genkernel will be installed
|
||||||
print(f"\t:: Step 11: kernel")
|
LOG.info(f"\t:: Step 11: kernel")
|
||||||
proc = Popen(["eselect", "kernel", "set", "1"])
|
proc = Popen(["eselect", "kernel", "set", "1"])
|
||||||
proc.communicate()
|
proc.communicate()
|
||||||
if not args.kernel_dist:
|
if not args.kernel_dist:
|
||||||
@ -180,7 +184,7 @@ def step11_kernel(args, cfg):
|
|||||||
completestep(11, "kernel")
|
completestep(11, "kernel")
|
||||||
|
|
||||||
def step12_grub(args, cfg):
|
def step12_grub(args, cfg):
|
||||||
print(f"\t:: Step 12: kernel")
|
LOG.info(f"\t:: Step 12: kernel")
|
||||||
proc = Popen(["grub-install", cfg.get('disk')])
|
proc = Popen(["grub-install", cfg.get('disk')])
|
||||||
proc.communicate()
|
proc.communicate()
|
||||||
code = proc.returncode
|
code = proc.returncode
|
||||||
@ -196,13 +200,13 @@ def step12_grub(args, cfg):
|
|||||||
completestep(12, "grub")
|
completestep(12, "grub")
|
||||||
|
|
||||||
def step13_serial(args, cfg):
|
def step13_serial(args, cfg):
|
||||||
print(f"\t:: Step 13: Serial")
|
LOG.info(f"\t:: Step 13: Serial")
|
||||||
os.system("sed -i 's/^#s0:/s0:/g' /etc/inittab")
|
os.system("sed -i 's/^#s0:/s0:/g' /etc/inittab")
|
||||||
os.system("sed -i 's/^#s1:/s1:/g' /etc/inittab")
|
os.system("sed -i 's/^#s1:/s1:/g' /etc/inittab")
|
||||||
completestep(13, "serial")
|
completestep(13, "serial")
|
||||||
|
|
||||||
def step14_services(args, cfg):
|
def step14_services(args, cfg):
|
||||||
print(f"\t:: Step 14: Services")
|
LOG.info(f"\t:: Step 14: Services")
|
||||||
for service in ["acpid", "syslog-ng", "cronie", "sshd", "cloud-init-local", "cloud-init", "cloud-config",
|
for service in ["acpid", "syslog-ng", "cronie", "sshd", "cloud-init-local", "cloud-init", "cloud-config",
|
||||||
"cloud-final", "ntpd", "nfsclient"]:
|
"cloud-final", "ntpd", "nfsclient"]:
|
||||||
if args.profile == "systemd":
|
if args.profile == "systemd":
|
||||||
@ -214,11 +218,11 @@ def step14_services(args, cfg):
|
|||||||
completestep(14, "services")
|
completestep(14, "services")
|
||||||
|
|
||||||
def step15_ethnaming(args, cfg):
|
def step15_ethnaming(args, cfg):
|
||||||
print(f"\t:: Step 15: Eth Naming")
|
LOG.info(f"\t:: Step 15: Eth Naming")
|
||||||
completestep(15, "networking")
|
completestep(15, "networking")
|
||||||
|
|
||||||
def step16_sysconfig(args, cfg):
|
def step16_sysconfig(args, cfg):
|
||||||
print(f"\t:: Step 16: Sysconfig")
|
LOG.info(f"\t:: Step 16: Sysconfig")
|
||||||
with open("/etc/timezone", "w") as f:
|
with open("/etc/timezone", "w") as f:
|
||||||
f.write("UTC")
|
f.write("UTC")
|
||||||
proc = Popen(["emerge", "--config", "sys-libs/timezone-data"])
|
proc = Popen(["emerge", "--config", "sys-libs/timezone-data"])
|
||||||
@ -270,7 +274,7 @@ def step16_sysconfig(args, cfg):
|
|||||||
completestep(16, "sysconfig")
|
completestep(16, "sysconfig")
|
||||||
|
|
||||||
def step17_fstab(args, cfg):
|
def step17_fstab(args, cfg):
|
||||||
print(f"\t:: Step 17: fstab")
|
LOG.info(f"\t:: Step 17: fstab")
|
||||||
with open(os.path.join(os.sep, 'etc', 'fstab'), 'a') as fstab:
|
with open(os.path.join(os.sep, 'etc', 'fstab'), 'a') as fstab:
|
||||||
fstab.write(f"{cfg.get('disk')}\t/\text4\tdefaults,noatime\t0 1\n")
|
fstab.write(f"{cfg.get('disk')}\t/\text4\tdefaults,noatime\t0 1\n")
|
||||||
|
|
||||||
|
@ -45,9 +45,9 @@ def run_image(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if isinstance(iso, list):
|
if isinstance(iso, list):
|
||||||
|
assert len(iso), f"iso list is empty {iso}"
|
||||||
iso = iso[0]
|
iso = iso[0]
|
||||||
|
|
||||||
|
|
||||||
image = gentooimgr.common.get_image_name(args, config)
|
image = gentooimgr.common.get_image_name(args, config)
|
||||||
qmounts = []
|
qmounts = []
|
||||||
mounts.extend(args.mounts)
|
mounts.extend(args.mounts)
|
||||||
|
@ -1,9 +1,12 @@
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
|
from gentooimgr import LOG
|
||||||
import gentooimgr.config
|
import gentooimgr.config
|
||||||
import gentooimgr.qemu
|
import gentooimgr.qemu
|
||||||
import gentooimgr.common
|
import gentooimgr.common
|
||||||
|
|
||||||
def run(args, config: dict):
|
def run(args, config: dict) -> None:
|
||||||
|
LOG.info(": run")
|
||||||
mounts = args.mounts
|
mounts = args.mounts
|
||||||
# Specified image or look for gentoo.{img,qcow2}
|
# Specified image or look for gentoo.{img,qcow2}
|
||||||
image = config.get("imagename") or args.image or gentooimgr.qemu.create_image()
|
image = config.get("imagename") or args.image or gentooimgr.qemu.create_image()
|
||||||
@ -17,12 +20,13 @@ def run(args, config: dict):
|
|||||||
".."
|
".."
|
||||||
))
|
))
|
||||||
|
|
||||||
print(args)
|
assert os.path.isfile(main_iso), f"iso not found {main_iso}"
|
||||||
print(main_iso)
|
LOG.info(args)
|
||||||
|
LOG.info(f'iso={args.iso}')
|
||||||
gentooimgr.qemu.run_image(
|
gentooimgr.qemu.run_image(
|
||||||
args,
|
args,
|
||||||
config,
|
config,
|
||||||
# Add our generated mount and livecd (assumed)
|
# Add our generated mount and livecd (assumed)
|
||||||
mounts=[main_iso]
|
mounts=[main_iso]
|
||||||
)
|
)
|
||||||
print("done")
|
LOG.info("done")
|
||||||
|
@ -1,8 +1,31 @@
|
|||||||
|
"""Step 1: Disk Partitioning
|
||||||
|
Step 2: Mounting {gentooimgr.config.GENTOO_MOUNT}
|
||||||
|
Step 3: Stage3 Tarball
|
||||||
|
Step 4: Binding Filesystems
|
||||||
|
Step 5: Portage
|
||||||
|
Step 6: Licenses
|
||||||
|
Step 7: Repo Configuration
|
||||||
|
Step 8: Resolv
|
||||||
|
Step 9: sync
|
||||||
|
Step 10: emerge pkgs
|
||||||
|
Step 11: kernel
|
||||||
|
Step 12: kernel
|
||||||
|
Step 13: Serial
|
||||||
|
Step 14: Services
|
||||||
|
Step 15: Eth Naming
|
||||||
|
Step 16: Sysconfig
|
||||||
|
Step 17: fstab
|
||||||
|
"""
|
||||||
|
import os
|
||||||
import json
|
import json
|
||||||
|
|
||||||
|
# from gentooimgr import LOG
|
||||||
import gentooimgr.config
|
import gentooimgr.config
|
||||||
import gentooimgr.configs
|
import gentooimgr.configs
|
||||||
|
|
||||||
def print_template(args, configjson):
|
def print_template(args, configjson):
|
||||||
|
print(__doc__)
|
||||||
|
print(f"the last step to succeed is {install.getlaststep(prefix)}\n")
|
||||||
print(f"""------------------------ STATUS ------------------------
|
print(f"""------------------------ STATUS ------------------------
|
||||||
|
|
||||||
CPU_THREADS = {args.threads or 1}
|
CPU_THREADS = {args.threads or 1}
|
||||||
|
@ -2,3 +2,4 @@ pydantic
|
|||||||
typing
|
typing
|
||||||
urllib
|
urllib
|
||||||
progressbar
|
progressbar
|
||||||
|
requests
|
||||||
|
Loading…
Reference in New Issue
Block a user