Pass all flake8 tests - run in CI too

This commit is contained in:
Clint Byrum 2013-03-15 10:47:02 -07:00
parent 57a2265ebb
commit 1db777fd9e
8 changed files with 266 additions and 194 deletions

View File

@ -2,8 +2,8 @@ language: python
python:
- "2.7"
# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
install: pip install --use-mirrors pystache nose argparse
install: pip install --use-mirrors pystache nose argparse flake8
# # command to run tests, e.g. python setup.py test
script: nosetests
script: ./run_tests.sh
notifications:
irc: "irc.freenode.org#tripleo"

View File

@ -1,2 +1,2 @@
class ConfigException(Exception):
pass
pass

View File

@ -4,101 +4,125 @@ import logging
import os
import pystache
import sys
import tempfile
from argparse import ArgumentParser
from pystache.context import KeyNotFoundError
from subprocess import Popen, PIPE
from value_types import *
from config_exception import *
from tempfile import NamedTemporaryFile
from value_types import ensure_type
from config_exception import ConfigException
TEMPLATES_DIR = os.environ.get('OS_CONFIG_APPLIER_TEMPLATES',
'/opt/stack/os-config-applier/templates')
def install_config(config_path, template_root, output_path, validate, subhash=None):
config = strip_hash( read_config(config_path), subhash)
tree = build_tree( template_paths(template_root), config )
if not validate:
for path, contents in tree.items():
write_file( os.path.join(output_path, strip_prefix('/', path)), contents)
def install_config(config_path, template_root,
output_path, validate, subhash=None):
config = strip_hash(read_config(config_path), subhash)
tree = build_tree(template_paths(template_root), config)
if not validate:
for path, contents in tree.items():
write_file(os.path.join(
output_path, strip_prefix('/', path)), contents)
def print_key(config_path, key, type_name):
config = read_config(config_path)
keys = key.split('.')
for key in keys:
try:
config = config[key]
except KeyError:
raise KeyError('key %s does not exist in %s' % (key, config_path))
ensure_type(config, type_name)
print config
config = read_config(config_path)
keys = key.split('.')
for key in keys:
try:
config = config[key]
except KeyError:
raise KeyError('key %s does not exist in %s' % (key, config_path))
ensure_type(config, type_name)
print config
def write_file(path, contents):
logger.info("writing %s", path)
d = os.path.dirname(path)
os.path.exists(d) or os.makedirs(d)
with NamedTemporaryFile(dir=d, delete=False) as newfile:
newfile.write(contents)
os.rename(newfile.name, path)
logger.info("writing %s", path)
d = os.path.dirname(path)
os.path.exists(d) or os.makedirs(d)
with NamedTemporaryFile(dir=d, delete=False) as newfile:
newfile.write(contents)
os.rename(newfile.name, path)
# return a map of filenames->filecontents
def build_tree(templates, config):
res = {}
for in_file, out_file in templates:
res[out_file] = render_template(in_file, config)
return res
res = {}
for in_file, out_file in templates:
res[out_file] = render_template(in_file, config)
return res
def render_template(template, config):
if is_executable(template):
return render_executable(template, config)
else:
try:
return render_moustache(open(template).read(), config)
except KeyNotFoundError as e:
raise ConfigException("key '%s' from template '%s' does not exist in metadata file." % (e.key, template))
except Exception as e:
raise ConfigException("could not render moustache template %s" % template)
if is_executable(template):
return render_executable(template, config)
else:
try:
return render_moustache(open(template).read(), config)
except KeyNotFoundError as e:
raise ConfigException(
"key '%s' from template '%s' does not exist in metadata file."
% (e.key, template))
except Exception as e:
raise ConfigException(
"could not render moustache template %s" % template)
def is_executable(path):
return os.path.isfile(path) and os.access(path, os.X_OK)
return os.path.isfile(path) and os.access(path, os.X_OK)
def render_moustache(text, config):
r = pystache.Renderer(missing_tags = 'strict')
return r.render(text, config)
r = pystache.Renderer(missing_tags='strict')
return r.render(text, config)
def render_executable(path, config):
p = Popen([path], stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(json.dumps(config))
p.wait()
if p.returncode != 0: raise ConfigException("config script failed: %s\n\nwith output:\n\n%s" % (path, stdout + stderr))
return stdout
p = Popen([path], stdin=PIPE, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(json.dumps(config))
p.wait()
if p.returncode != 0:
raise ConfigException(
"config script failed: %s\n\nwith output:\n\n%s" %
(path, stdout + stderr))
return stdout
def read_config(path):
try:
return json.loads(open(path).read())
except:
raise ConfigException("invalid metadata file: %s" % path)
try:
return json.loads(open(path).read())
except:
raise ConfigException("invalid metadata file: %s" % path)
def template_paths(root):
res = []
for cur_root, subdirs, files in os.walk(root):
for f in files:
inout = ( os.path.join(cur_root, f), os.path.join(strip_prefix(root, cur_root), f) )
res.append(inout)
return res
res = []
for cur_root, subdirs, files in os.walk(root):
for f in files:
inout = (os.path.join(cur_root, f), os.path.join(
strip_prefix(root, cur_root), f))
res.append(inout)
return res
def strip_prefix(prefix, s):
return s[len(prefix):] if s.startswith(prefix) else s
return s[len(prefix):] if s.startswith(prefix) else s
def strip_hash(h, keys):
if not keys: return h
for k in keys.split('.'):
if k in h and isinstance(h[k], dict):
h = h[k]
else:
raise ConfigException("key '%s' does not correspond to a hash in the metadata file" % keys)
return h
if not keys:
return h
for k in keys.split('.'):
if k in h and isinstance(h[k], dict):
h = h[k]
else:
raise ConfigException(
"key '%s' does not correspond to a hash in the metadata file"
% keys)
return h
def parse_opts(argv):
parser = ArgumentParser()
@ -107,58 +131,68 @@ def parse_opts(argv):
%(default)s)""",
default=TEMPLATES_DIR)
parser.add_argument('-o', '--output', metavar='OUT_DIR',
help='root directory for output (default: %(default)s)',
help='root directory for output (default:%(default)s)',
default='/')
parser.add_argument('-m', '--metadata', metavar='METADATA_FILE',
help='path to metadata file (default: %(default)s)',
default='/var/lib/cloud/data/cfn-init-data')
parser.add_argument('-v', '--validate', help='validate only. do not write files',
default=False, action='store_true')
parser.add_argument('--print-templates', default=False, action='store_true',
help='Print templates root and exit.')
parser.add_argument(
'-v', '--validate', help='validate only. do not write files',
default=False, action='store_true')
parser.add_argument(
'--print-templates', default=False, action='store_true',
help='Print templates root and exit.')
parser.add_argument('-s', '--subhash',
help='use the sub-hash named by this key, instead of the full metadata hash')
help='use the sub-hash named by this key,'
' instead of the full metadata hash')
parser.add_argument('--key', metavar='KEY', default=None,
help='print the specified key and exit. (may be used with --type)')
help='print the specified key and exit.'
' (may be used with --type)')
parser.add_argument('--type', default='default',
help='exit with error if the specified --key does not match type. Valid types are <int|default|raw>')
help='exit with error if the specified --key does not'
' match type. Valid types are <int|default|raw>')
opts = parser.parse_args(argv[1:])
return opts
def main(argv=sys.argv):
opts = parse_opts(argv)
if opts.print_templates:
print(opts.templates)
return 0
opts = parse_opts(argv)
if opts.print_templates:
print(opts.templates)
return 0
try:
if opts.templates is None:
raise ConfigException('missing option --templates')
try:
if opts.templates is None:
raise ConfigException('missing option --templates')
if opts.key:
print_key(opts.metadata, opts.key, opts.type)
else:
if not os.access(opts.output, os.W_OK):
raise ConfigException("you don't have permission to write to '%s'" % opts.output)
install_config(opts.metadata, opts.templates, opts.output,
opts.validate, opts.subhash)
logger.info("success")
except ConfigException as e:
logger.error(e)
sys.exit(1)
sys.exit(0)
if opts.key:
print_key(opts.metadata, opts.key, opts.type)
else:
if not os.access(opts.output, os.W_OK):
raise ConfigException(
"you don't have permission to write to '%s'" % opts.output)
install_config(opts.metadata, opts.templates, opts.output,
opts.validate, opts.subhash)
logger.info("success")
except ConfigException as e:
logger.error(e)
sys.exit(1)
sys.exit(0)
# logginig
LOG_FORMAT = '[%(asctime)s] [%(levelname)s] %(message)s'
DATE_FORMAT = '%Y/%m/%d %I:%M:%S %p'
def add_handler(logger, handler):
handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT))
logger.addHandler(handler)
handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT))
logger.addHandler(handler)
logger = logging.getLogger('os-config-applier')
logger.setLevel(logging.INFO)
add_handler(logger, logging.StreamHandler(sys.stdout))
if os.geteuid() == 0: add_handler(logger, logging.FileHandler('/var/log/os-config-applier.log'))
if os.geteuid() == 0:
add_handler(logger, logging.FileHandler('/var/log/os-config-applier.log'))
if __name__ == '__main__':
main(sys.argv)
main(sys.argv)

View File

@ -2,14 +2,17 @@ import re
from config_exception import ConfigException
TYPES = {
"int": "^[0-9]+$",
"default": "^[A-Za-z0-9]+$",
"raw": "."
"int": "^[0-9]+$",
"default": "^[A-Za-z0-9]+$",
"raw": "."
}
def ensure_type(string_value, type_name='default'):
if type_name not in TYPES:
raise ValueError("requested validation of unknown type: %s" % type_name)
if not re.match(TYPES[type_name], string_value):
raise ConfigException("cannot interpret value '%s' as type %s" % (string_value, type_name))
return string_value
if type_name not in TYPES:
raise ValueError(
"requested validation of unknown type: %s" % type_name)
if not re.match(TYPES[type_name], string_value):
raise ConfigException("cannot interpret value '%s' as type %s" % (
string_value, type_name))
return string_value

2
run_tests.sh Executable file
View File

@ -0,0 +1,2 @@
flake8 --verbose `find . -name '*.py'`
nosetests

View File

@ -16,7 +16,8 @@ config = {
'install_requires': ['pystache', 'anyjson'],
# 'long_description': open('README.md').read(),
'entry_points': {
'console_scripts': ['os-config-applier = os_config_applier.os_config_applier:main']
'console_scripts': [
'os-config-applier = os_config_applier.os_config_applier:main']
}
}

View File

@ -1,157 +1,186 @@
import json
import os
import sys
import subprocess
import tempfile
from StringIO import StringIO
from nose.tools import *
from os_config_applier.config_exception import *
from os_config_applier.os_config_applier import *
from nose.tools import assert_equal, assert_equals, assert_raises, raises
from os_config_applier.config_exception import ConfigException
from os_config_applier.os_config_applier import (
main, TEMPLATES_DIR, strip_hash, read_config, template_paths,
render_executable, render_template, render_moustache, install_config,
build_tree)
# example template tree
TEMPLATES = os.path.join(os.path.dirname(__file__), 'templates')
TEMPLATE_PATHS = [
"/etc/glance/script.conf",
"/etc/keystone/keystone.conf"
]
]
# config for example tree
CONFIG = {
"x": "foo",
"database": {
"x": "foo",
"database": {
"url": "sqlite:///blah"
}
}
}
# config for example tree - with subhash
CONFIG_SUBHASH = {
"OpenStack::Config": {
"OpenStack::Config": {
"x": "foo",
"database": {
"url": "sqlite:///blah"
"url": "sqlite:///blah"
}
}
}
}
# expected output for example tree
OUTPUT = {
"/etc/glance/script.conf": "foo\n",
"/etc/keystone/keystone.conf": "[foo]\ndatabase = sqlite:///blah\n"
"/etc/glance/script.conf": "foo\n",
"/etc/keystone/keystone.conf": "[foo]\ndatabase = sqlite:///blah\n"
}
def setup():
pass
pass
def teardown():
pass
pass
def main_path():
return os.path.dirname(os.path.realpath(__file__)) + '/../os_config_applier/os_config_applier.py'
return (
os.path.dirname(os.path.realpath(__file__)) +
'/../os_config_applier/os_config_applier.py')
def template(relpath):
return os.path.join(TEMPLATES, relpath[1:])
return os.path.join(TEMPLATES, relpath[1:])
def test_install_config():
t = tempfile.NamedTemporaryFile()
t.write(json.dumps(CONFIG))
t.flush()
tmpdir = tempfile.mkdtemp()
install_config(t.name, TEMPLATES, tmpdir, False)
for path, contents in OUTPUT.items():
full_path = os.path.join(tmpdir, path[1:])
assert os.path.exists(full_path)
assert_equal( open(full_path).read(), contents )
t = tempfile.NamedTemporaryFile()
t.write(json.dumps(CONFIG))
t.flush()
tmpdir = tempfile.mkdtemp()
install_config(t.name, TEMPLATES, tmpdir, False)
for path, contents in OUTPUT.items():
full_path = os.path.join(tmpdir, path[1:])
assert os.path.exists(full_path)
assert_equal(open(full_path).read(), contents)
def test_install_config_subhash():
t = tempfile.NamedTemporaryFile()
t.write(json.dumps(CONFIG_SUBHASH))
t.flush()
tmpdir = tempfile.mkdtemp()
install_config(t.name, TEMPLATES, tmpdir, False, 'OpenStack::Config')
for path, contents in OUTPUT.items():
full_path = os.path.join(tmpdir, path[1:])
assert os.path.exists(full_path)
assert_equal( open(full_path).read(), contents )
t = tempfile.NamedTemporaryFile()
t.write(json.dumps(CONFIG_SUBHASH))
t.flush()
tmpdir = tempfile.mkdtemp()
install_config(t.name, TEMPLATES, tmpdir, False, 'OpenStack::Config')
for path, contents in OUTPUT.items():
full_path = os.path.join(tmpdir, path[1:])
assert os.path.exists(full_path)
assert_equal(open(full_path).read(), contents)
def test_print_key():
t = tempfile.NamedTemporaryFile()
t.write(json.dumps(CONFIG))
t.flush()
out = subprocess.check_output([main_path(), '--metadata', t.name, '--key',
'database.url', '--type', 'raw'],
stderr=subprocess.STDOUT)
assert_equals(CONFIG['database']['url'], out.rstrip())
t = tempfile.NamedTemporaryFile()
t.write(json.dumps(CONFIG))
t.flush()
out = subprocess.check_output([main_path(), '--metadata', t.name, '--key',
'database.url', '--type', 'raw'],
stderr=subprocess.STDOUT)
assert_equals(CONFIG['database']['url'], out.rstrip())
@raises(subprocess.CalledProcessError)
def test_print_key_missing():
t = tempfile.NamedTemporaryFile()
t.write(json.dumps(CONFIG))
t.flush()
out = subprocess.check_output([main_path(), '--metadata', t.name, '--key',
'does.not.exist'], stderr=subprocess.STDOUT)
t = tempfile.NamedTemporaryFile()
t.write(json.dumps(CONFIG))
t.flush()
subprocess.check_output([main_path(), '--metadata', t.name, '--key',
'does.not.exist'], stderr=subprocess.STDOUT)
@raises(subprocess.CalledProcessError)
def test_print_key_wrong_type():
t = tempfile.NamedTemporaryFile()
t.write(json.dumps(CONFIG))
t.flush()
out = subprocess.check_output([main_path(), '--metadata', t.name, '--key',
'x', '--type', 'int'],
stderr=subprocess.STDOUT)
t = tempfile.NamedTemporaryFile()
t.write(json.dumps(CONFIG))
t.flush()
subprocess.check_output([main_path(), '--metadata', t.name, '--key',
'x', '--type', 'int'], stderr=subprocess.STDOUT)
def test_build_tree():
assert_equals( build_tree(template_paths(TEMPLATES), CONFIG), OUTPUT )
assert_equals(build_tree(template_paths(TEMPLATES), CONFIG), OUTPUT)
def test_render_template():
# execute executable files, moustache non-executables
assert render_template(template("/etc/glance/script.conf"), {"x": "abc"}) == "abc\n"
assert_raises(ConfigException, render_template, template("/etc/glance/script.conf"), {})
# execute executable files, moustache non-executables
assert render_template(template(
"/etc/glance/script.conf"), {"x": "abc"}) == "abc\n"
assert_raises(ConfigException, render_template, template(
"/etc/glance/script.conf"), {})
def test_render_moustache():
assert_equals( render_moustache("ab{{x.a}}cd", {"x": {"a": "123"}}), "ab123cd" )
assert_equals(render_moustache("ab{{x.a}}cd", {
"x": {"a": "123"}}), "ab123cd")
@raises(Exception)
def test_render_moustache_bad_key():
render_moustache("{{badkey}}", {})
render_moustache("{{badkey}}", {})
def test_render_executable():
params = {"x": "foo"}
assert render_executable(template("/etc/glance/script.conf"), params) == "foo\n"
params = {"x": "foo"}
assert render_executable(template(
"/etc/glance/script.conf"), params) == "foo\n"
@raises(ConfigException)
def test_render_executable_failure():
render_executable(template("/etc/glance/script.conf"), {})
render_executable(template("/etc/glance/script.conf"), {})
def test_template_paths():
expected = map(lambda p: (template(p), p), TEMPLATE_PATHS)
actual = template_paths(TEMPLATES)
expected.sort(key=lambda tup: tup[1])
actual.sort(key=lambda tup: tup[1])
assert_equals( actual , expected)
expected = map(lambda p: (template(p), p), TEMPLATE_PATHS)
actual = template_paths(TEMPLATES)
expected.sort(key=lambda tup: tup[1])
actual.sort(key=lambda tup: tup[1])
assert_equals(actual, expected)
def test_read_config():
with tempfile.NamedTemporaryFile() as t:
d = {"a": {"b": ["c", "d"] } }
t.write(json.dumps(d))
t.flush()
assert_equals( read_config(t.name), d )
with tempfile.NamedTemporaryFile() as t:
d = {"a": {"b": ["c", "d"]}}
t.write(json.dumps(d))
t.flush()
assert_equals(read_config(t.name), d)
@raises(ConfigException)
def test_read_config_bad_json():
with tempfile.NamedTemporaryFile() as t:
t.write("{{{{")
t.flush()
read_config(t.name)
with tempfile.NamedTemporaryFile() as t:
t.write("{{{{")
t.flush()
read_config(t.name)
@raises(Exception)
def test_read_config_no_file():
read_config("/nosuchfile")
read_config("/nosuchfile")
def test_strip_hash():
h = {'a': {'b': {'x': 'y'} }, "c": [1, 2, 3] }
assert_equals( strip_hash(h, 'a.b'), {'x': 'y'})
assert_raises(ConfigException, strip_hash, h, 'a.nonexistent')
assert_raises(ConfigException, strip_hash, h, 'a.c')
h = {'a': {'b': {'x': 'y'}}, "c": [1, 2, 3]}
assert_equals(strip_hash(h, 'a.b'), {'x': 'y'})
assert_raises(ConfigException, strip_hash, h, 'a.nonexistent')
assert_raises(ConfigException, strip_hash, h, 'a.c')
def test_print_templates():
save_stdout = sys.stdout

View File

@ -1,18 +1,21 @@
from nose.tools import *
from os_config_applier.os_config_applier import *
from os_config_applier.config_exception import *
from os_config_applier.value_types import *
from nose.tools import assert_equals, raises
from os_config_applier.config_exception import ConfigException
from os_config_applier.value_types import ensure_type
@raises(ValueError)
def test_unknown_type():
ensure_type("foo", "badtype")
ensure_type("foo", "badtype")
def test_int():
assert_equals("123", ensure_type("123", "int"))
assert_equals("123", ensure_type("123", "int"))
def test_defualt():
assert_equals("foobar", ensure_type("foobar", "default"))
assert_equals("foobar", ensure_type("foobar", "default"))
@raises(ConfigException)
def test_default_bad():
ensure_type("foo\nbar", "default")
ensure_type("foo\nbar", "default")