Fix tox settings and relevant bugs

I found problem as follows in current tox.ini

- test-requirements can not be read correctly
- some os level command is not in white_list
- one rst file does not follow rst syntac
- fix lack of gitreview file

Purpose of this review request is fixing above problems.

Change-Id: I1d2b29bbc7cdc506f5187c2ef6ee03fd2b908f58
This commit is contained in:
Keiichi Hikita 2018-04-13 11:35:05 +09:00
parent eb49a9225a
commit 7444891218
13 changed files with 87 additions and 69 deletions

5
.gitreview Normal file
View File

@ -0,0 +1,5 @@
[gerrit]
host=review.openstack.org
port=29418
project=openstack/slogging.git
defaultbranch=master

6
bindep.txt Normal file
View File

@ -0,0 +1,6 @@
# This is a cross-platform list tracking distribution packages needed by tests;
# see http://docs.openstack.org/infra/bindep/ for additional information.
liberasurecode-dev [platform:dpkg]
# There's no library in CentOS 7 but Fedora and openSUSE have it.
liberasurecode-devel [platform:rpm !platform:centos]

View File

@ -2,7 +2,7 @@
# of appearance. Changing the order has an impact on the overall integration # of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later. # process, which may cause wedges in the gate later.
openstackdocstheme>=1.18.1 # Apache-2.0 openstackdocstheme # Apache-2.0
sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD sphinx!=1.6.6,!=1.6.7 # BSD
reno>=2.5.0 # Apache-2.0 reno # Apache-2.0
sphinxcontrib-httpdomain>=1.3.0 # BSD sphinxcontrib-httpdomain # BSD

View File

@ -4,18 +4,29 @@ How to Build to RPM Packages
#. Make sure you have rpm-build installed:: #. Make sure you have rpm-build installed::
- on Fedora/CentOS/RedHat::
sudo yum install rpm-build sudo yum install rpm-build
- on OpenSUSE::
sudo zypper install rpm-build Fedora/CentOS/RedHat
~~~~~~~~~~~~~~~~~~~~
#. Thsn type following command at the top of slogging directory::
#. Type following command at the top of slogging directory::
sudo python setup.py bdist_rpm sudo python setup.py bdist_rpm
#. Check if the RPM package has built:: #. Check if the RPM package has built::
ls dist/slogging-[slogging-version] ls dist/slogging-[slogging-version]
OpenSUSE
~~~~~~~~
#. Type following command at the top of slogging directory::
sudo zypper install rpm-build
#. Check if the RPM package has built::
ls dist/slogging-[slogging-version]

View File

@ -1,4 +1,6 @@
pbr!=2.1.0,>=2.0.0 # Apache-2.0 # The order of packages is significant, because pip processes them in the order
iptools>=0.4 # of appearance. Changing the order has an impact on the overall integration
tzlocal # process, which may cause wedges in the gate later.
swift>=2.14.0 pbr!=2.1.0 # Apache-2.0
pytz
netaddr

View File

@ -1,5 +1,4 @@
#!/usr/bin/python # Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
# Copyright (c) 2010-2011 OpenStack, LLC.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
@ -14,15 +13,17 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools import setuptools
# In python < 2.7.4, a lazy loading of package `pbr` will break
# setuptools if some other modules registered functions in `atexit`.
# solution from: http://bugs.python.org/issue15881#msg170215
try: try:
import multiprocessing # noqa import multiprocessing # noqa
except ImportError: except ImportError:
pass pass
setuptools.setup( setuptools.setup(
setup_requires=['pbr'], setup_requires=['pbr>=2.0.0'],
pbr=True, pbr=True)
)

View File

@ -16,20 +16,19 @@
from datetime import datetime from datetime import datetime
import pytz import pytz
from slogging import common from slogging import common
from tzlocal import get_localzone
from urllib import unquote from urllib import unquote
from urlparse import urlparse from urlparse import urlparse
# conditionalize the return_ips method based on whether or not iptools # conditionalize the return_ips method based on whether or not iptools
# is present in the system. Without iptools, you will lack CIDR support. # is present in the system. Without iptools, you will lack CIDR support.
try: try:
from iptools import IpRangeList from netaddr import IPSet
CIDR_support = True CIDR_support = True
def return_ips(conf, conf_tag): def return_ips(conf, conf_tag):
return set(k for k in IpRangeList(*[ return IPSet([
x.strip() for x in conf.get(conf_tag, '').split(',') x.strip() for x in conf.get(conf_tag, '').split(',')
if x.strip()])) if x.strip()])
def sanitize_ips(line_data): def sanitize_ips(line_data):
for x in ['lb_ip', 'client_ip', 'log_source']: for x in ['lb_ip', 'client_ip', 'log_source']:
@ -48,7 +47,7 @@ from swift.common import utils
month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split() month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split()
LISTING_PARAMS = set( LISTING_PARAMS = set(
'path limit format delimiter marker end_marker prefix'.split()) 'path limit format delimiter marker end_marker prefix'.split())
local_zone = get_localzone() local_zone = common.get_localzone()
class AccessLogProcessor(object): class AccessLogProcessor(object):
@ -63,7 +62,7 @@ class AccessLogProcessor(object):
self.warn_percent = float(conf.get('warn_percent', '0.8')) self.warn_percent = float(conf.get('warn_percent', '0.8'))
self.logger = utils.get_logger(conf, log_route='access-processor') self.logger = utils.get_logger(conf, log_route='access-processor')
self.time_zone = common.get_time_zone(conf, self.logger, 'time_zone', self.time_zone = common.get_time_zone(conf, self.logger, 'time_zone',
str(local_zone)) local_zone)
def log_line_parser(self, raw_log): def log_line_parser(self, raw_log):
'''given a raw access log line, return a dict of the good parts''' '''given a raw access log line, return a dict of the good parts'''

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import pytz import pytz
@ -44,3 +45,14 @@ def get_format_type(conf, logger, key, default):
logger.warning(msg) logger.warning(msg)
format_type = default format_type = default
return format_type return format_type
def get_localzone():
"""Get time_zone value."""
try:
str_time_zone = '/'.join(
os.readlink('/etc/localtime').split('/')[-2:])
except Exception:
str_time_zone = 'UTC'
return str_time_zone

View File

@ -29,11 +29,10 @@ from swift.container.backend import ContainerBroker
from swift.container.server import DATADIR as container_server_data_dir from swift.container.server import DATADIR as container_server_data_dir
import time import time
from tzlocal import get_localzone
import urllib import urllib
local_zone = get_localzone() local_zone = common.get_localzone()
class DatabaseStatsCollector(Daemon): class DatabaseStatsCollector(Daemon):
@ -57,7 +56,7 @@ class DatabaseStatsCollector(Daemon):
self.logger = utils.get_logger(stats_conf, self.logger = utils.get_logger(stats_conf,
log_route='%s-stats' % stats_type) log_route='%s-stats' % stats_type)
self.time_zone = common.get_time_zone(stats_conf, self.logger, self.time_zone = common.get_time_zone(stats_conf, self.logger,
'time_zone', str(local_zone)) 'time_zone', local_zone)
def run_once(self, *args, **kwargs): def run_once(self, *args, **kwargs):
self.logger.info(_("Gathering %s stats") % self.stats_type) self.logger.info(_("Gathering %s stats") % self.stats_type)

View File

@ -25,11 +25,10 @@ from slogging import log_common
from swift.common.daemon import Daemon from swift.common.daemon import Daemon
from swift.common import utils from swift.common import utils
import time import time
from tzlocal import get_localzone
now = datetime.datetime.now now = datetime.datetime.now
local_zone = get_localzone() local_zone = common.get_localzone()
class LogProcessor(log_common.LogProcessorCommon): class LogProcessor(log_common.LogProcessorCommon):
@ -137,7 +136,7 @@ class LogProcessorDaemon(Daemon):
self._keylist_mapping = None self._keylist_mapping = None
self.processed_files_filename = 'processed_files.pickle.gz' self.processed_files_filename = 'processed_files.pickle.gz'
self.time_zone = common.get_time_zone(c, self.logger, 'time_zone', self.time_zone = common.get_time_zone(c, self.logger, 'time_zone',
str(local_zone)) local_zone)
self.format_type = common.get_format_type(c, self.logger, self.format_type = common.get_format_type(c, self.logger,
'format_type', 'csv') 'format_type', 'csv')

View File

@ -3,11 +3,12 @@
# process, which may cause wedges in the gate later. # process, which may cause wedges in the gate later.
# Hacking already pins down pep8, pyflakes and flake8 # Hacking already pins down pep8, pyflakes and flake8
hacking>=0.11.0,<0.12 # Apache-2.0 hacking<0.12,>=0.11.0 # Apache-2.0
coverage>=3.6 # Apache-2.0 coverage!=4.4 # Apache-2.0
nose # LGPL nose # LGPL
nosexcover # BSD nosexcover # BSD
nosehtmloutput>=0.0.3 # Apache-2.0 nosehtmloutput # Apache-2.0
flake8<=2.5.5
# Security checks # Security checks
bandit>=1.1.0 # Apache-2.0 bandit>=1.1.0 # Apache-2.0

View File

@ -15,7 +15,7 @@
import datetime import datetime
import iptools import netaddr
import unittest import unittest
@ -25,15 +25,15 @@ class TestAccessProcessorSpeed(unittest.TestCase):
line = 'Sep 16 20:00:02 srv testsrv 192.%s.119.%s - ' \ line = 'Sep 16 20:00:02 srv testsrv 192.%s.119.%s - ' \
'16/Sep/2012/20/00/02 GET /v1/a/c/o HTTP/1.0 ' \ '16/Sep/2012/20/00/02 GET /v1/a/c/o HTTP/1.0 ' \
'200 - StaticWeb - - 17005 - txn - 0.0095 -' '200 - StaticWeb - - 17005 - txn - 0.0095 -'
ips1 = iptools.IpRangeList(*[x.strip() for x in ips1 = netaddr.IPSet([x.strip() for x in
'127.0.0.1,192.168/16,10/24'.split(',') '127.0.0.1,192.168/16,10/24'.split(',')
if x.strip()]) if x.strip()])
ips2 = iptools.IpRangeList(*[x.strip() for x in ips2 = netaddr.IPSet([x.strip() for x in
'172.168/16,10/30'.split(',') '172.168/16,10/30'.split(',')
if x.strip()]) if x.strip()])
ips3 = iptools.IpRangeList(*[x.strip() for x in ips3 = netaddr.IPSet([x.strip() for x in
'127.0.0.1,11/24'.split(',') '127.0.0.1,11/24'.split(',')
if x.strip()]) if x.strip()])
orig_start = datetime.datetime.utcnow() orig_start = datetime.datetime.utcnow()
hit = 0 hit = 0
@ -49,9 +49,9 @@ class TestAccessProcessorSpeed(unittest.TestCase):
self.assertEqual(hit, 255) self.assertEqual(hit, 255)
# now, let's check the speed with sets # now, let's check the speed with sets
set1 = set(k for k in ips1) set1 = ips1
set2 = set(k for k in ips2) set2 = ips2
set3 = set(k for k in ips3) set3 = ips3
new_start = datetime.datetime.utcnow() new_start = datetime.datetime.utcnow()
hit = 0 hit = 0

27
tox.ini
View File

@ -2,6 +2,7 @@
envlist = py27,docs,pep8 envlist = py27,docs,pep8
[testenv] [testenv]
usedevelop = True
setenv = VIRTUAL_ENV={envdir} setenv = VIRTUAL_ENV={envdir}
NOSE_WITH_OPENSTACK=1 NOSE_WITH_OPENSTACK=1
NOSE_OPENSTACK_COLOR=1 NOSE_OPENSTACK_COLOR=1
@ -13,55 +14,37 @@ deps = -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/re
commands= commands=
py27: {[unit_tests]commands} py27: {[unit_tests]commands}
py35: {[unit_tests]commands} py35: {[unit_tests]commands}
whitelist_externals = find
rm
[testenv:pep8] [testenv:pep8]
deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz
-r{toxinidir}/test-requirements.txt
commands = flake8 {posargs} commands = flake8 {posargs}
[testenv:cover]
setenv = VIRTUAL_ENV={envdir}
NOSE_WITH_COVERAGE=1
NOSE_COVER_HTML_DIR={toxinidir}/cover
NOSE_COVER_HTML=1
NOSE_COVER_ERASE=1
commands =
{[unit_tests]commands}
[testenv:py27] [testenv:py27]
basepython = python2.7 basepython = python2.7
deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz
-r{toxinidir}/test-requirements.txt
setenv = SWIFT_TEST_CONFIG_FILE={toxinidir}/test/sample.conf setenv = SWIFT_TEST_CONFIG_FILE={toxinidir}/test/sample.conf
SWIFT_PROXY_TEST_CONFIG_FILE={toxinidir}/test/sample.proxy-server.conf SWIFT_PROXY_TEST_CONFIG_FILE={toxinidir}/test/sample.proxy-server.conf
commands = commands =
{[unit_tests]commands} {[unit_tests]commands}
[unit_tests] [unit_tests]
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands = find . -type f -name "*.py[c|o]" -delete commands = find . -type f -name "*.py[c|o]" -delete
find . -type d -name "__pycache__" -delete find . -type d -name "__pycache__" -delete
nosetests {posargs:test/unit} nosetests {posargs:test/unit}
[testenv:venv]
commands = {posargs}
[flake8] [flake8]
exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,dash_template exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,dash_template
max-complexity = 20 max-complexity = 20
import-order-style = pep8
[testenv:docs] [testenv:docs]
deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz
-r{toxinidir}/doc/requirements.txt -r{toxinidir}/doc/requirements.txt
commands = python setup.py build_sphinx commands = python setup.py build_sphinx
[doc8] [doc8]
# File extensions to check # File extensions to check
extensions = .rst, .yaml extensions = .rst, .yaml