Rework package-installs to collapse on build host

Instead of doing the work in the image of parsing through the element's
package-install declarations, we can squash it on hostside, where we
have both YAML and JSON available to us, and then emit a single
pre-processed file into the target to be used later.

Change-Id: I3f182aa3aae0a79b2f3ea4e66c1878ad12878b0a
This commit is contained in:
Monty Taylor 2014-12-10 08:19:28 -08:00 committed by Gregory Haynes
parent 1273cd5532
commit 98d008c6af
10 changed files with 136 additions and 121 deletions

View File

@ -0,0 +1,82 @@
#!/usr/bin/env python
# Copyright 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import collections
import functools
import json
import os
import yaml
def collect_data(data, filename, element_name):
try:
objs = json.load(open(filename))
except ValueError:
objs = yaml.load(open(filename))
for pkg_name, params in objs.items():
if not params:
params = {}
phase = params.get('phase', 'install.d')
install = "install"
if 'uninstall' in params:
install = "uninstall"
data[phase][install].append((pkg_name, element_name))
return data
def main():
parser = argparse.ArgumentParser(
description="Produce a single packages-installs file from all of"
" the available package-installs files")
parser.add_argument('--elements', required=True,
help="Which elements to squash")
parser.add_argument('--path', required=True,
help="Elements path to search for elements")
parser.add_argument('outfile', help="Location of the output file")
args = parser.parse_args()
# Replicate the logic of finding the first element, because we can't
# operate on the post-copied hooks dir, since we lose element context
element_dirs = list()
for element_name in args.elements.split():
for elements_dir in args.path.split(':'):
potential_path = os.path.join(elements_dir, element_name)
if os.path.exists(potential_path):
element_dirs.append((elements_dir, element_name))
# Collect the merge of all of the existing install files in the elements
# that are the first on the ELEMENT_PATH
final_dict = collections.defaultdict(
functools.partial(collections.defaultdict, list))
for (elements_dir, element_name) in element_dirs:
for file_type in ('json', 'yaml'):
target_file = os.path.join(
elements_dir, element_name, "package-installs.%s" % file_type)
if not os.path.exists(target_file):
continue
final_dict = collect_data(final_dict, target_file, element_name)
# Write the resulting file
with open(args.outfile, 'w') as outfile:
json.dump(
final_dict, outfile,
indent=True, separators=(',', ': '), sort_keys=False)
if __name__ == '__main__':
main()

View File

@ -15,131 +15,73 @@
# under the License.
import argparse
from json import load as json_load
import os
import json
import subprocess
import sys
from yaml import load as yaml_load
class PackageInstalls(object):
@classmethod
def phase_to_attr(cls, phase):
return phase.replace('.', '_').replace('-', '_')
@classmethod
def from_yaml_path(cls, path):
with open(path) as fp:
return PackageInstalls.from_native_objs(yaml_load(fp))
@classmethod
def from_json_path(cls, path):
with open(path) as fp:
return PackageInstalls.from_native_objs(json_load(fp))
@classmethod
def from_native_objs(cls, objs):
init_args = {}
for pkg_name, params in objs.items():
uninstall = False
phase = "install.d"
try:
phase = params["phase"]
except (KeyError, TypeError):
pass
try:
uninstall = bool(params["uninstall"])
except (KeyError, TypeError):
pass
init_arg = PackageInstalls.phase_to_attr(phase)
if uninstall:
init_arg = init_arg + '_uninst'
init_args[init_arg] = init_args.get(init_arg, []) + [pkg_name]
return PackageInstalls(**init_args)
def __init__(self, **phase_installs):
for phase, pkgs in phase_installs.items():
setattr(self, phase, pkgs)
class PackageInstallsController(object):
def __init__(self, path='/usr/share/package-installs'):
self.path = path
def package_installs(self):
for yaml_path in os.listdir(self.path):
full_path = os.path.join(self.path, yaml_path)
if full_path.endswith('.yaml'):
pi = PackageInstalls.from_yaml_path(full_path)
elif full_path.endswith('.json'):
pi = PackageInstalls.from_json_path(full_path)
else:
print("No decoder known for %s, skipping" % full_path)
continue
yield (yaml_path[:-5], pi)
def main():
parser = argparse.ArgumentParser(
description="Install or uninstall packages for a specific phase based"
" on package-installs files.")
parser.add_argument('--phase',
parser.add_argument('--phase', required=True,
help="Install phase to filter on. Valid options are"
" 'install.d' or pre-install.d")
parser.add_argument('--uninstall', action="store_true",
help="Only show packages to uninstall. By default only"
" packages to install are shown")
args, extra = parser.parse_known_args()
parser.add_argument('-n', '--noop', action="store_true",
help="Don't actually install, just print the command")
parser.add_argument('infile', help="File to process")
args = parser.parse_args()
if not args.phase:
print("Please specify an install phase.")
sys.exit(1)
packages = json.load(open(args.infile))
if args.uninstall:
install = "uninstall"
else:
install = "install"
pi_c = PackageInstallsController()
pkgs = []
for element, pi in pi_c.package_installs():
installs_attr = PackageInstalls.phase_to_attr(args.phase)
if args.uninstall:
installs_attr += '_uninst'
pkgs = list()
if args.phase in packages and install in packages[args.phase]:
install_packages = packages[args.phase][install]
else:
print("Nothing to %s" % install)
sys.exit(0)
for (pkg, element) in install_packages:
print("%sing %s from %s" % (install, pkg, element))
pkg_map_args = ["pkg-map", "--missing-ok", "--element", element]
pkg_map_args.append(pkg)
try:
phase_installs = getattr(pi, installs_attr)
except AttributeError:
continue
for pkg in phase_installs:
print("Installing %s from %s" % (phase_installs, element))
pkg_map_args = ["pkg-map", "--missing-ok", "--element", element]
pkg_map_args += phase_installs
try:
map_output = subprocess.check_output(pkg_map_args)
except subprocess.CalledProcessError as e:
if e.returncode == 1:
map_output = subprocess.check_output(
pkg_map_args, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if e.returncode == 1:
if args.noop:
pkgs.append(pkg)
continue
else:
print("pkg-map failed with error %s" % e.output)
sys.exit(1)
elif e.returncode == 2:
pkgs += phase_installs
continue
pkgs += map_output.strip().split('\n')
elif e.returncode == 2:
pkgs.append(pkg)
continue
pkgs.extend(map_output.strip().split('\n'))
install_args = ["install-packages"]
if args.uninstall:
install_args.append("-e")
install_args.extend(pkgs)
install_args.extend(list(set(pkgs)))
try:
subprocess.check_output(install_args)
except subprocess.CalledProcessError as e:
print("install failed with error %s" % e.output)
sys.exit(1)
if args.noop:
print(" ".join(install_args))
else:
try:
subprocess.check_output(install_args)
except subprocess.CalledProcessError as e:
print("install failed with error %s" % e.output)
sys.exit(1)
if __name__ == '__main__':

View File

@ -1,14 +0,0 @@
#!/bin/bash
set -eux
set -o pipefail
dest_dir="$TMP_MOUNT_PATH/usr/share/package-installs/"
sudo mkdir -p $dest_dir
for ELEMENT in $IMAGE_ELEMENT ; do
for DIR in ${ELEMENTS_PATH//:/ }; do
if [ -f "$DIR/$ELEMENT/package-installs.yaml" ]; then
sudo cp $DIR/$ELEMENT/package-installs.yaml $dest_dir/$ELEMENT.yaml
fi
done
done

View File

@ -0,0 +1,5 @@
#!/bin/bash
set -eux
set -o pipefail
sudo $(dirname $0)/../bin/package-installs-squash --elements="$IMAGE_ELEMENT" --path=$ELEMENTS_PATH $TMP_MOUNT_PATH/tmp/package-installs.json

View File

@ -4,4 +4,4 @@ set -eux
set -o pipefail
package-installs -d $(dirname $0)
package-installs-v2 --phase install.d
package-installs-v2 --phase install.d /tmp/package-installs.json

View File

@ -4,4 +4,4 @@ set -eux
set -o pipefail
package-uninstalls -d $(dirname $0)
package-installs-v2 --phase install.d --uninstall
package-installs-v2 --phase install.d --uninstall /tmp/package-installs.json

View File

@ -4,4 +4,4 @@ set -eux
set -o pipefail
package-installs -d $(dirname $0)
package-installs-v2 --phase post-install.d
package-installs-v2 --phase post-install.d /tmp/package-installs.json

View File

@ -4,4 +4,4 @@ set -eux
set -o pipefail
package-uninstalls -d $(dirname $0)
package-installs-v2 --phase post-install.d --uninstall
package-installs-v2 --phase post-install.d --uninstall /tmp/package-installs.json

View File

@ -4,4 +4,4 @@ set -eux
set -o pipefail
package-installs -d $(dirname $0)
package-installs-v2 --phase pre-install.d
package-installs-v2 --phase pre-install.d /tmp/package-installs.json

View File

@ -4,4 +4,4 @@ set -eux
set -o pipefail
package-uninstalls -d $(dirname $0)
package-installs-v2 --phase pre-install.d --uninstall
package-installs-v2 --phase pre-install.d --uninstall /tmp/package-installs.json