Standardize Pegleg directory structure

This patch set standardizes the Pegleg directory
structure because of the following reasons:

1) src/bin/pegleg is not necessary and only makes
building (e.g. documentation building) and running
of tox targets unnecessarily difficult.
2) src/bin/pegleg is a Java-like standard that
bears no relevance to Python.

Change-Id: I37d39d3d6186b92f8fbfe234221c9e44da48cf10
This commit is contained in:
Felipe Monteiro 2018-09-20 17:33:54 +01:00
parent 2c963948e2
commit 893ea9f4bb
42 changed files with 208 additions and 76 deletions

View File

@ -66,7 +66,7 @@
timeout: 300
nodeset: airship-pegleg-single-node
irrelevant-files:
- ^src/bin/pegleg/tests/.*$
- ^tests/.*$
- ^setup.cfg$
- job:

View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
PEGLEG_BUILD_CTX ?= src/bin/pegleg
PEGLEG_BUILD_CTX ?= pegleg
IMAGE_NAME ?= pegleg
IMAGE_PREFIX ?= airshipit
DOCKER_REGISTRY ?= quay.io
@ -45,12 +45,12 @@ tests: run_tests
.PHONY: security
security:
tox -c src/bin/pegleg/tox.ini -e bandit
tox -e bandit
# Run all unit tests under src/bin/pegleg
# Run all unit tests under pegleg
.PHONY: run_tests
run_tests:
tox -c src/bin/pegleg/tox.ini -e py35
tox -e py35
# Perform Linting
.PHONY: lint
@ -83,7 +83,7 @@ endif
.PHONY: docs
docs: clean
tox -edocs
tox -e docs
.PHONY: clean
clean:
@ -91,8 +91,8 @@ clean:
.PHONY: py_lint
py_lint:
cd src/bin/pegleg;tox -e pep8
cd pegleg;tox -e pep8
.PHONY: py_format
py_format:
cd src/bin/pegleg;tox -e fmt
cd pegleg;tox -e fmt

View File

@ -18,7 +18,8 @@
#
import os
import sys
sys.path.insert(0, os.path.abspath('../../src/bin/pegleg'))
sys.path.insert(0, os.path.abspath('.'))
sys.path.insert(0, os.path.abspath('../../'))
import sphinx_rtd_theme
@ -105,7 +106,7 @@ html_static_path = []
# -- Options for HTMLHelp output ------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'ucpintdoc'
htmlhelp_basename = 'peglegdoc'
# -- Options for LaTeX output ---------------------------------------------
@ -128,3 +129,32 @@ latex_elements = {
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'pegleg.tex', u'Pegleg Documentation',
u'Pegleg Authors', 'manual'),
]
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'pegleg', u'Pegleg Documentation',
[author], 1)
]
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'pegleg', u'Pegleg Documentation',
author, 'pegleg', 'Document-based aggregation and linting service.',
'Miscellaneous'),
]

View File

@ -72,7 +72,7 @@ Pegleg strives to conform to the `Airship coding conventions`_.
Python
------
The Pegleg code base lives under ``/src/bin/pegleg``. Pegleg supports py35
The Pegleg code base lives under ``pegleg``. Pegleg supports py35 and py36
interpreters.
Docker
@ -91,7 +91,7 @@ target: ``run_pegleg``.
Testing
=======
All Pegleg tests are nested under ``/src/bin/pegleg/tests``.
All Pegleg tests are nested under ``tests``.
Pegleg comes equipped with a number of `tox`_ targets for running unit tests,
as well as `pep8`_ and `Bandit`_ checks.

View File

@ -4,10 +4,8 @@ FROM ${FROM}
VOLUME /var/pegleg
WORKDIR /var/pegleg
ARG ctx_base=src/bin/pegleg
COPY ${ctx_base}/requirements.txt /opt/pegleg/requirements.txt
COPY requirements.txt /opt/pegleg/requirements.txt
RUN pip3 install --no-cache-dir -r /opt/pegleg/requirements.txt
COPY ${ctx_base} /opt/pegleg
COPY . /opt/pegleg
RUN pip3 install -e /opt/pegleg

135
pegleg/engine/site.py Normal file
View File

@ -0,0 +1,135 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import csv
import json
import logging
import os
import click
import yaml
from pegleg.engine import util
__all__ = ('collect', 'list_', 'show', 'render')
LOG = logging.getLogger(__name__)
def _read_and_format_yaml(filename):
with open(filename) as f:
lines_to_write = f.readlines()
if lines_to_write[0] != '---\n':
lines_to_write = ['---\n'] + lines_to_write
if lines_to_write[-1] != '...\n':
lines_to_write.append('...\n')
return lines_to_write or []
def _collect_to_stdout(site_name):
"""Collects all documents related to ``site_name`` and outputs them to
stdout via ``output_stream``.
"""
try:
for repo_base, filename in util.definition.site_files_by_repo(
site_name):
for line in _read_and_format_yaml(filename):
# This code is a pattern to convert \r\n to \n.
click.echo("\n".join(line.splitlines()))
except Exception as ex:
raise click.ClickException("Error printing output: %s" % str(ex))
def _collect_to_file(site_name, save_location):
"""Collects all documents related to ``site_name`` and outputs them to
the file denoted by ``save_location``.
"""
if not os.path.exists(save_location):
LOG.debug("Collection save location %s does not exist. Creating "
"automatically.", save_location)
os.makedirs(save_location)
# In case save_location already exists and isn't a directory.
if not os.path.isdir(save_location):
raise click.ClickException('save_location %s already exists, but must '
'be a directory' % save_location)
save_files = dict()
try:
for repo_base, filename in util.definition.site_files_by_repo(
site_name):
repo_name = os.path.normpath(repo_base).split(os.sep)[-1]
save_file = os.path.join(save_location, repo_name + '.yaml')
if repo_name not in save_files:
save_files[repo_name] = open(save_file, "w")
LOG.debug("Collecting file %s to file %s" % (filename, save_file))
save_files[repo_name].writelines(_read_and_format_yaml(filename))
except Exception as ex:
raise click.ClickException("Error saving output: %s" % str(ex))
finally:
for f in save_files.values():
f.close()
def collect(site_name, save_location):
if save_location:
_collect_to_file(site_name, save_location)
else:
_collect_to_stdout(site_name)
def render(site_name, output_stream):
documents = []
for filename in util.definition.site_files(site_name):
with open(filename) as f:
documents.extend(list(yaml.safe_load_all(f)))
rendered_documents, errors = util.deckhand.deckhand_render(
documents=documents)
err_msg = ''
if errors:
for err in errors:
if isinstance(err, tuple) and len(err) > 1:
err_msg += ': '.join(err) + '\n'
else:
err_msg += str(err) + '\n'
raise click.ClickException(err_msg)
yaml.dump_all(rendered_documents, output_stream, default_flow_style=False)
def list_(output_stream):
"""List site names for a given repository."""
# TODO(felipemonteiro): This should output a formatted table, not rows of
# data without delimited columns.
fieldnames = ['site_name', 'site_type', 'repositories']
writer = csv.DictWriter(
output_stream, fieldnames=fieldnames, delimiter=' ')
for site_name in util.files.list_sites():
params = util.definition.load_as_params(site_name)
# TODO(felipemonteiro): This is a temporary hack around legacy manifest
# repositories containing the name of a directory that symbolizes a
# repository. Once all these manifest repositories migrate over to Git
# references instead, remove this hack.
# NOTE(felipemonteiro): The 'revision' information can instead be
# computed using :func:`process_site_repository` and storing into
# a configuration via a "set_site_revision" function, for example.
if 'revision' in params:
params.pop('revision')
writer.writerow(params)
def show(site_name, output_stream):
data = util.definition.load_as_params(site_name)
data['files'] = list(util.definition.site_files(site_name))
json.dump(data, output_stream, indent=2, sort_keys=True)

View File

@ -1,37 +0,0 @@
[tox]
envlist = py35, py36, pep8
skipsdist = True
[testenv]
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
basepython=python3
whitelist_externals =
find
commands =
find . -type f -name "*.pyc" -delete
pytest \
{posargs}
[testenv:fmt]
basepython=python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
yapf --style=pep8 -ir {toxinidir}/pegleg {toxinidir}/tests
[testenv:pep8]
basepython=python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
flake8 {toxinidir}/pegleg
bandit -r pegleg -n 5
[testenv:bandit]
basepython=python3
commands = bandit -r pegleg -n 5
[flake8]
ignore = E125,E251,W503

52
tox.ini
View File

@ -10,40 +10,46 @@ setenv = VIRTUAL_ENV={envdir}
LANGUAGE=en_US
LC_ALL=en_US.utf-8
passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
[testenv:venv]
commands = {posargs}
[testenv:py35]
basepython = python3.5
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
basepython=python3
whitelist_externals =
find
commands =
# Run all unit tests under src/bin/pegleg
tox -c src/bin/pegleg/tox.ini -e py35
whitelist_externals = tox
[testenv:py36]
basepython = python3.6
commands =
# Run all unit tests under src/bin/pegleg
tox -c src/bin/pegleg/tox.ini -e py36
whitelist_externals = tox
find . -type f -name "*.pyc" -delete
pytest \
{posargs}
[testenv:fmt]
basepython = python3
basepython=python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
tox -c src/bin/pegleg/tox.ini -e fmt
whitelist_externals = tox
yapf --style=pep8 -ir {toxinidir}/pegleg {toxinidir}/tests
[testenv:pep8]
basepython = python3
basepython=python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
tox -c src/bin/pegleg/tox.ini -e pep8
whitelist_externals = tox
flake8 {toxinidir}/pegleg
bandit -r pegleg -n 5
[testenv:docs]
basepython = python3
deps =
-r{toxinidir}/src/bin/pegleg/requirements.txt
-r{toxinidir}/requirements.txt
-r{toxinidir}/doc/requirements.txt
commands =
{toxinidir}/tools/gate/build-docs.sh
[testenv:bandit]
basepython=python3
commands = bandit -r pegleg -n 5
[testenv:venv]
commands = {posargs}
[flake8]
ignore = E125,E251,W503