Move package management to yum tool
This commit introduces new tool named YYOOM that handles package management. It uses yum python module (and so is GPLv2+ licensed) and print work result in JSON to stdout. Anvil uses it to install packages and query which packages are install or available. We also log all installed packages (including deps) via tracewriter, which allows to use tracereader for clean and complete uninstall. Fixes: bug 1189707 Change-Id: Ib6d13b2dc816a3d2f8875aa23779e34fa685cd31
This commit is contained in:
Ivan A. Melnikov
parent
5796566830
commit
07eb723261
|
|
@ -3,6 +3,7 @@
|
|||
*.pyc
|
||||
/build/
|
||||
/dist/
|
||||
/tools/yyoomc
|
||||
.coverage
|
||||
*.egg-info
|
||||
|
||||
|
|
|
|||
10
README.rst
10
README.rst
|
|
@ -2,3 +2,13 @@ We want more information!
|
|||
=========================
|
||||
|
||||
Please check out: http://anvil.readthedocs.org.
|
||||
|
||||
Licensing
|
||||
=========
|
||||
|
||||
Anvil is licensed under the Apache License, Version 2.0 (the "License"); you
|
||||
may not use this file except in compliance with the License. You may obtain a
|
||||
copy of the License at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Some tools are licensed under different terms; see tools/README.rst for
|
||||
more information.
|
||||
|
|
|
|||
|
|
@ -14,47 +14,40 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# See http://yum.baseurl.org/api/yum-3.2.26/yum-module.html
|
||||
from yum import YumBase
|
||||
import sys
|
||||
import json
|
||||
|
||||
from yum.packages import PackageObject
|
||||
from anvil import log as logging
|
||||
from anvil import shell as sh
|
||||
|
||||
|
||||
class Requirement(object):
|
||||
def __init__(self, name, version):
|
||||
self.name = str(name)
|
||||
self.version = version
|
||||
|
||||
def __str__(self):
|
||||
name = self.name
|
||||
if self.version is not None:
|
||||
name += "-%s" % (self.version)
|
||||
return name
|
||||
|
||||
@property
|
||||
def package(self):
|
||||
# Form a 'fake' rpm package that
|
||||
# can be used to compare against
|
||||
# other rpm packages using the
|
||||
# standard rpm routines
|
||||
my_pkg = PackageObject()
|
||||
my_pkg.name = self.name
|
||||
if self.version is not None:
|
||||
my_pkg.version = str(self.version)
|
||||
return my_pkg
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Helper(object):
|
||||
# Cache of yumbase object
|
||||
_yum_base = None
|
||||
|
||||
def __init__(self):
|
||||
self._installed = None
|
||||
self._available = None
|
||||
|
||||
@staticmethod
|
||||
def _get_yum_base():
|
||||
if Helper._yum_base is None:
|
||||
_yum_base = YumBase()
|
||||
_yum_base.setCacheDir(force=True)
|
||||
Helper._yum_base = _yum_base
|
||||
return Helper._yum_base
|
||||
def _yyoom(arglist):
|
||||
executable = sh.which("yyoom", ["tools/"])
|
||||
cmdline = [executable]
|
||||
if LOG.logger.isEnabledFor(logging.DEBUG):
|
||||
cmdline.append('--verbose')
|
||||
cmdline.extend(arglist)
|
||||
out = sh.execute(cmdline, stderr_fh=sys.stderr)[0].strip()
|
||||
if out:
|
||||
return json.loads(out)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _trace_installed_packages(tracewriter, data):
|
||||
if tracewriter is None or not data:
|
||||
return
|
||||
for action in data:
|
||||
if action['action_type'] == 'install':
|
||||
tracewriter.package_installed(action['name'])
|
||||
|
||||
def is_installed(self, name):
|
||||
if len(self.get_installed(name)):
|
||||
|
|
@ -63,18 +56,38 @@ class Helper(object):
|
|||
return False
|
||||
|
||||
def get_available(self):
|
||||
base = Helper._get_yum_base()
|
||||
pkgs = base.doPackageLists(showdups=True)
|
||||
avail = list(pkgs.available)
|
||||
avail.extend(pkgs.installed)
|
||||
return avail
|
||||
if self._available is None:
|
||||
self._available = self._yyoom(['list', 'available'])
|
||||
return self._available
|
||||
|
||||
def get_installed(self, name):
|
||||
base = Helper._get_yum_base()
|
||||
pkgs = base.doPackageLists(pkgnarrow='installed',
|
||||
ignore_case=True, patterns=[name])
|
||||
if pkgs.installed:
|
||||
whats_installed = list(pkgs.installed)
|
||||
else:
|
||||
whats_installed = []
|
||||
return whats_installed
|
||||
if self._installed is None:
|
||||
self._installed = self._yyoom(['list', 'installed'])
|
||||
return [item for item in self._installed
|
||||
if item['name'] == name]
|
||||
|
||||
def builddep(self, srpm_path, tracewriter=None):
|
||||
data = self._yyoom(['builddep', srpm_path])
|
||||
self._trace_installed_packages(tracewriter, data)
|
||||
|
||||
def clean(self):
|
||||
self._yyoom(['cleanall'])
|
||||
|
||||
def transaction(self, install_pkgs=(), remove_pkgs=(), tracewriter=None):
|
||||
if not install_pkgs and not remove_pkgs:
|
||||
return
|
||||
|
||||
# reset the caches:
|
||||
self._installed = None
|
||||
self._available = None
|
||||
|
||||
cmdline = ['transaction']
|
||||
for pkg in install_pkgs:
|
||||
cmdline.append('--install')
|
||||
cmdline.append(pkg)
|
||||
for pkg in remove_pkgs:
|
||||
cmdline.append('--erase')
|
||||
cmdline.append(pkg)
|
||||
|
||||
data = self._yyoom(cmdline)
|
||||
self._trace_installed_packages(tracewriter, data)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@
|
|||
|
||||
import collections
|
||||
import contextlib
|
||||
import os
|
||||
import pkg_resources
|
||||
import sys
|
||||
|
||||
|
|
@ -25,7 +24,6 @@ import rpm
|
|||
import tarfile
|
||||
|
||||
from anvil import colorizer
|
||||
from anvil import env
|
||||
from anvil import exceptions as excp
|
||||
from anvil import log as logging
|
||||
from anvil.packaging import base
|
||||
|
|
@ -103,18 +101,6 @@ class YumDependencyHandler(base.DependencyHandler):
|
|||
self.anvil_repo_dir = sh.joinpths(self.root_dir, "repo")
|
||||
self._no_remove = None
|
||||
|
||||
@property
|
||||
def no_remove(self):
|
||||
if self._no_remove is not None:
|
||||
return self._no_remove
|
||||
packages = env.get_key('REQUIRED_PACKAGES', default_value='').split()
|
||||
own_details = pip_helper.get_directory_details(os.getcwd())
|
||||
required_pips = own_details['dependencies']
|
||||
no_remove = self._convert_names_python2rpm(required_pips)
|
||||
no_remove.extend(packages)
|
||||
self._no_remove = no_remove
|
||||
return self._no_remove
|
||||
|
||||
def py2rpm_start_cmdline(self):
|
||||
cmdline = [
|
||||
self.py2rpm_executable,
|
||||
|
|
@ -178,20 +164,13 @@ class YumDependencyHandler(base.DependencyHandler):
|
|||
sh.move(filename, target_dir, force=True)
|
||||
|
||||
def build_binary(self):
|
||||
|
||||
def _install_build_requirements():
|
||||
build_requires = self.requirements["build-requires"]
|
||||
if build_requires:
|
||||
utils.log_iterable(sorted(build_requires),
|
||||
header=("Installing %s build requirements" % len(build_requires)),
|
||||
logger=LOG)
|
||||
cmdline = ["yum", "install", "-y"] + list(build_requires)
|
||||
sh.execute(cmdline)
|
||||
|
||||
def _is_src_rpm(filename):
|
||||
return filename.endswith('.src.rpm')
|
||||
|
||||
_install_build_requirements()
|
||||
LOG.info("Installing build requirements")
|
||||
self.helper.transaction(
|
||||
install_pkgs=self.requirements["build-requires"],
|
||||
tracewriter=self.tracewriter)
|
||||
|
||||
for repo_name in self.REPOS:
|
||||
repo_dir = sh.joinpths(self.anvil_repo_dir, repo_name)
|
||||
|
|
@ -269,8 +248,8 @@ class YumDependencyHandler(base.DependencyHandler):
|
|||
def _get_yum_available(self):
|
||||
yum_map = collections.defaultdict(list)
|
||||
for pkg in self.helper.get_available():
|
||||
for provides in pkg.provides:
|
||||
yum_map[provides[0]].append((pkg.version, pkg.repo))
|
||||
for provides in pkg['provides']:
|
||||
yum_map[provides[0]].append((pkg['version'], pkg['repo']))
|
||||
return dict(yum_map)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -638,39 +617,18 @@ class YumDependencyHandler(base.DependencyHandler):
|
|||
|
||||
def install(self):
|
||||
super(YumDependencyHandler, self).install()
|
||||
self.helper.clean()
|
||||
|
||||
# Erase conflicting packages
|
||||
cmdline = []
|
||||
for p in self.requirements["conflicts"]:
|
||||
if self.helper.is_installed(p):
|
||||
cmdline.append(p)
|
||||
|
||||
if cmdline:
|
||||
cmdline = ["yum", "erase", "-y"] + cmdline
|
||||
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
|
||||
|
||||
cmdline = ["yum", "clean", "all"]
|
||||
sh.execute(cmdline)
|
||||
|
||||
rpm_names = self._all_rpm_names()
|
||||
if rpm_names:
|
||||
cmdline = ["yum", "install", "-y"] + rpm_names
|
||||
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
|
||||
remove_pkgs = [pkg_name
|
||||
for pkg_name in self.requirements["conflicts"]
|
||||
if self.helper.is_installed(pkg_name)]
|
||||
self.helper.transaction(install_pkgs=self._all_rpm_names(),
|
||||
remove_pkgs=remove_pkgs,
|
||||
tracewriter=self.tracewriter)
|
||||
|
||||
def uninstall(self):
|
||||
super(YumDependencyHandler, self).uninstall()
|
||||
|
||||
scan_packages = self._all_rpm_names()
|
||||
rpm_names = []
|
||||
for p in scan_packages:
|
||||
if p in self.no_remove:
|
||||
continue
|
||||
if self.helper.is_installed(p):
|
||||
rpm_names.append(p)
|
||||
|
||||
if rpm_names:
|
||||
cmdline = ["yum", "remove", "--remove-leaves", "-y"]
|
||||
for p in self.no_remove:
|
||||
cmdline.append("--exclude=%s" % (p))
|
||||
cmdline.extend(sorted(set(rpm_names)))
|
||||
sh.execute(cmdline, stdout_fh=sys.stdout, stderr_fh=sys.stderr)
|
||||
if self.tracereader.exists():
|
||||
remove_pkgs = self.tracereader.packages_installed()
|
||||
self.helper.transaction(remove_pkgs=remove_pkgs)
|
||||
|
|
|
|||
|
|
@ -95,6 +95,20 @@ builds RPMs (current directory is used by default)::
|
|||
...
|
||||
|
||||
|
||||
yyoom
|
||||
-----
|
||||
|
||||
`yyoom` uses yum API to provide nice command-line interface to package
|
||||
management. It is able to install and remove packages in the same
|
||||
transaction (see `yyoom transaction --help`), list available or installed
|
||||
packages and a bit more. It writes results of its work to standard output
|
||||
in JSON.
|
||||
|
||||
`yyoom` is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
git-changelog
|
||||
-------------
|
||||
This tool generates a pretty software's changelog from git history.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,291 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Library General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
# Copyright 2005 Duke University
|
||||
# Parts Copyright 2007 Red Hat, Inc
|
||||
|
||||
"""YYOOM: a package management utility
|
||||
|
||||
Using Yum API instead of /usr/bin/yum provides several interesting
|
||||
capabilities, some of which we are desperate to use, including:
|
||||
- installing and removing packages in same transaction;
|
||||
- JSON output.
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import yum
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
LOG = logging.getLogger('yum-tool')
|
||||
OUTPUT = None
|
||||
|
||||
|
||||
def _setup_output():
|
||||
"""Do some nasty manipulations with fds
|
||||
|
||||
Yum internals may sometimes write to stdout, just out of a sudden.
|
||||
To prevent this output form interfering with our JSON, we save
|
||||
current stdout to other fd via os.dup, and replace fd 1 with
|
||||
/dev/null opened for writing.
|
||||
"""
|
||||
global OUTPUT
|
||||
# save current stdout for later use
|
||||
OUTPUT = os.fdopen(os.dup(sys.stdout.fileno()), 'wb')
|
||||
# close the stream
|
||||
sys.stdout.close()
|
||||
# open /dev/null -- all writes to stdout from now on will go there
|
||||
devnull_fd = os.open(os.devnull, os.O_WRONLY)
|
||||
if devnull_fd != 1:
|
||||
os.dup2(devnull_fd, 1)
|
||||
os.close(devnull_fd)
|
||||
sys.stdout = os.fdopen(1, 'w')
|
||||
|
||||
|
||||
def _write_output(data):
|
||||
"""Dump given object as pretty json"""
|
||||
OUTPUT.write(json.dumps(data, indent=4,
|
||||
separators=(',', ': '),
|
||||
sort_keys=True) + '\n')
|
||||
|
||||
|
||||
def _package_info(pkg, **kwargs):
|
||||
if isinstance(pkg, basestring):
|
||||
result = dict(name=pkg, **kwargs)
|
||||
else:
|
||||
result = dict(
|
||||
name=pkg.name,
|
||||
epoch=pkg.epoch,
|
||||
version=pkg.version,
|
||||
release=pkg.release,
|
||||
provides=pkg.provides,
|
||||
repo=str(pkg.repo),
|
||||
arch=pkg.arch,
|
||||
**kwargs
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class _RPMCallback(yum.rpmtrans.RPMBaseCallback):
|
||||
"""Listen to events from RPM transactions"""
|
||||
|
||||
def __init__(self):
|
||||
self.seen = []
|
||||
|
||||
def event(self, package, action, te_current, te_total,
|
||||
ts_current, ts_total):
|
||||
pass
|
||||
|
||||
def scriptout(self, package, msg):
|
||||
if not msg or not LOG.isEnabledFor(logging.INFO):
|
||||
return
|
||||
for line in msg.splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
LOG.info("%s: %s", package, line)
|
||||
|
||||
def errorlog(self, msg):
|
||||
LOG.error("%s", msg)
|
||||
|
||||
def filelog(self, package, action):
|
||||
action_data = _package_info(package, action_code=action)
|
||||
if action in yum.constants.TS_INSTALL_STATES:
|
||||
action_data['action_type'] = 'install'
|
||||
elif action in yum.constants.TS_REMOVE_STATES:
|
||||
action_data['action_type'] = 'erase'
|
||||
else:
|
||||
action_data['action_type'] = 'other'
|
||||
self.seen.append(action_data)
|
||||
LOG.info("Performed %(action_type)s (code %(action_code)s) on %(name)s"
|
||||
% action_data)
|
||||
|
||||
|
||||
def _run(yum_base, options):
|
||||
"""Handler of `transaction` command
|
||||
|
||||
Installs and erases packages, prints what was done in JSON
|
||||
"""
|
||||
LOG.debug('Erasing packages: %s', options.erase)
|
||||
LOG.debug('Installing packages: %s', options.install)
|
||||
callback = _RPMCallback()
|
||||
with _transaction(yum_base, callback):
|
||||
for name in options.erase or ():
|
||||
yum_base.remove(name=name)
|
||||
for name in options.install or ():
|
||||
yum_base.install(name=name)
|
||||
_write_output(callback.seen)
|
||||
|
||||
|
||||
def _list(yum_base, options):
|
||||
"""Handler of `list` command"""
|
||||
pkgnarrow = options.what[0] if len(options.what) == 1 else 'all'
|
||||
lists = yum_base.doPackageLists(pkgnarrow=pkgnarrow, showdups=True)
|
||||
LOG.debug("Got packages for '%s': %s installed, %s available,"
|
||||
"%s available for reinstall, %s extras",
|
||||
pkgnarrow, len(lists.installed), len(lists.available),
|
||||
len(lists.reinstall_available), len(lists.extras))
|
||||
|
||||
result = []
|
||||
if 'installed' in options.what:
|
||||
result.extend(_package_info(pkg, status='installed')
|
||||
for pkg in lists.installed)
|
||||
if 'available' in options.what:
|
||||
result.extend(_package_info(pkg, status='available')
|
||||
for pkg in lists.available)
|
||||
result.extend(_package_info(pkg, status='available')
|
||||
for pkg in lists.reinstall_available)
|
||||
if 'extras' in options.what:
|
||||
result.extend(_package_info(pkg, status='installed')
|
||||
for pkg in lists.extras)
|
||||
_write_output(result)
|
||||
|
||||
|
||||
def _cleanall(yum_base, options):
|
||||
"""Handler of `cleanall` command"""
|
||||
LOG.info("Running yum cleanup")
|
||||
code = sum((
|
||||
_run_yum_api('packages clean up', yum_base.cleanPackages),
|
||||
_run_yum_api('headers clean up', yum_base.cleanHeaders),
|
||||
_run_yum_api('metadata clean up', yum_base.cleanMetadata),
|
||||
_run_yum_api('sqlite clean up', yum_base.cleanSqlite),
|
||||
_run_yum_api('rpm db clean up', yum_base.cleanRpmDB),
|
||||
))
|
||||
return code
|
||||
|
||||
|
||||
def _builddep(yum_base, options):
|
||||
"""Handler of `builddep` command
|
||||
|
||||
Installs build dependencies for given package, prints what was done
|
||||
in JSON.
|
||||
"""
|
||||
LOG.info("Installing build dependencies for package %s", options.srpm)
|
||||
srpm = yum.packages.YumLocalPackage(yum_base.ts, options.srpm)
|
||||
callback = _RPMCallback()
|
||||
with _transaction(yum_base, callback):
|
||||
for req in srpm.requiresList():
|
||||
LOG.debug('Processing dependency: %s', req)
|
||||
if not (
|
||||
req.startswith('rpmlib(') or
|
||||
yum_base.returnInstalledPackagesByDep(req)
|
||||
):
|
||||
pkg = yum_base.returnPackageByDep(req)
|
||||
LOG.debug('Installing %s', pkg)
|
||||
yum_base.install(pkg)
|
||||
_write_output(callback.seen)
|
||||
|
||||
|
||||
def _parse_arguments(args):
|
||||
parser = argparse.ArgumentParser(prog=args[0])
|
||||
parser.add_argument('--verbose', '-v', action='store_true',
|
||||
help='verbose operation')
|
||||
# TODO(imelnikov): --format
|
||||
subparsers = parser.add_subparsers(title='subcommands')
|
||||
|
||||
parser_list = subparsers.add_parser('list', help='list packages')
|
||||
parser_list.add_argument('what', nargs='+',
|
||||
choices=('installed', 'available', 'extras'),
|
||||
help='what packages to list')
|
||||
parser_list.set_defaults(func=_list)
|
||||
|
||||
parser_run = subparsers.add_parser('transaction',
|
||||
help='install or remove packages')
|
||||
parser_run.set_defaults(func=_run)
|
||||
parser_run.add_argument('--install', '-i', action='append',
|
||||
metavar='package',
|
||||
help='install package')
|
||||
parser_run.add_argument('--erase', '-e', action='append',
|
||||
metavar='package',
|
||||
help='erase package')
|
||||
|
||||
parser_builddep = subparsers.add_parser(
|
||||
'builddep', help='install build dependencies of srpm')
|
||||
parser_builddep.add_argument('srpm', help='path to source RPM package')
|
||||
parser_builddep.set_defaults(func=_builddep)
|
||||
|
||||
parser_cleanall = subparsers.add_parser('cleanall', help='clean all')
|
||||
parser_cleanall.set_defaults(func=_cleanall)
|
||||
return parser.parse_args(args[1:])
|
||||
|
||||
|
||||
def _setup_logging(verbose=True):
|
||||
"""Initialize logging"""
|
||||
# setup logging -- put messages to stderr
|
||||
handler = logging.StreamHandler(sys.stderr)
|
||||
handler.setFormatter(logging.Formatter('YYOOM %(levelname)s: %(message)s'))
|
||||
root_logger = logging.getLogger()
|
||||
root_logger.addHandler(handler)
|
||||
root_logger.setLevel(logging.DEBUG if verbose else logging.INFO)
|
||||
|
||||
|
||||
def _get_yum_base():
|
||||
base = yum.YumBase()
|
||||
base.setCacheDir(force=True)
|
||||
return base
|
||||
|
||||
|
||||
def _run_yum_api(name, func, ok_codes=(0,), *args, **kwargs):
|
||||
code, results = func(*args, **kwargs)
|
||||
for msg in results:
|
||||
LOG.debug(msg)
|
||||
if code not in ok_codes:
|
||||
LOG.error('%s failed', name.title())
|
||||
return code
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _transaction(base, callback):
|
||||
"""Manage Yum transactions
|
||||
|
||||
Locks and unlocks Yum database, builds and processes transaction
|
||||
on __exit__.
|
||||
"""
|
||||
try:
|
||||
base.doLock()
|
||||
yield
|
||||
code = _run_yum_api('building transaction',
|
||||
base.buildTransaction, ok_codes=(0, 2))
|
||||
if code == 0:
|
||||
LOG.debug('Nothing to do')
|
||||
elif code == 2:
|
||||
base.processTransaction(rpmTestDisplay=callback,
|
||||
rpmDisplay=callback)
|
||||
else:
|
||||
raise RuntimeError("Transaction failed: %s" % code)
|
||||
finally:
|
||||
del base.tsInfo
|
||||
del base.ts
|
||||
base.doUnlock()
|
||||
|
||||
|
||||
def main(args):
|
||||
options = _parse_arguments(args)
|
||||
try:
|
||||
_setup_output()
|
||||
_setup_logging(options.verbose)
|
||||
return options.func(_get_yum_base(), options) or 0
|
||||
except Exception as e:
|
||||
if options.verbose:
|
||||
raise # let python runtime write stacktrace
|
||||
sys.stderr.write("Failed: %s\n" % e)
|
||||
return 1
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv))
|
||||
Loading…
Reference in New Issue