From db1fb068b89478dce182dc4a67d0c59e530b49a2 Mon Sep 17 00:00:00 2001 From: Alexander Kislitsky Date: Tue, 20 Oct 2015 19:23:19 +0300 Subject: [PATCH] Fuel-stats prepared for OpenStack CI We have 3 services: analytics, collector, migration. Migration is frozen and going to be removed. Also migration requires Elasticsearch server for tests, thus it can't be tested on OpenStack CI. Pep8 checked on all services. Project works only on python2.7, thus gate-fuel-stats-python26 should be removed. Change-Id: Ic5b6f15903918b4b0850ce0cdc12543925de0cfa Closes-Bug: #1500073 --- MANIFEST.in | 7 ++ .../fuel_analytics/api/common/__init__.py | 1 - analytics/fuel_analytics/test/base.py | 1 + collector/collector/test/base.py | 1 + migration/migration/test/test_es_mapping.py | 4 +- requirements.txt | 11 ++++ setup.py | 64 +++++++++++++++++++ test-requirements.txt | 7 ++ tools/prepare_ci_config.sh | 6 ++ tools/prepare_database.sh | 22 +++++++ tox.ini | 57 +++++++++++++++++ 11 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 MANIFEST.in create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 test-requirements.txt create mode 100755 tools/prepare_ci_config.sh create mode 100755 tools/prepare_database.sh create mode 100644 tox.ini diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e602741 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include *.txt +include collector/collector/test/logs +graft analytics/static +graft collector/collector/api/db/migrations +recursive-include collector/api/schemas *.json +prune analytics/static/bower_components +prune analytics/static/node_modules diff --git a/analytics/fuel_analytics/api/common/__init__.py b/analytics/fuel_analytics/api/common/__init__.py index aee8d92..b5ff96e 100644 --- a/analytics/fuel_analytics/api/common/__init__.py +++ b/analytics/fuel_analytics/api/common/__init__.py @@ -11,4 +11,3 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - diff --git a/analytics/fuel_analytics/test/base.py b/analytics/fuel_analytics/test/base.py index c5a5809..943cb9b 100644 --- a/analytics/fuel_analytics/test/base.py +++ b/analytics/fuel_analytics/test/base.py @@ -25,6 +25,7 @@ from fuel_analytics.api.log import init_logger # Configuring app for the test environment app.config.from_object('fuel_analytics.api.config.Testing') +app.config.from_envvar('ANALYTICS_SETTINGS', silent=True) init_logger() diff --git a/collector/collector/test/base.py b/collector/collector/test/base.py index 4475a19..d1d888e 100644 --- a/collector/collector/test/base.py +++ b/collector/collector/test/base.py @@ -27,6 +27,7 @@ flask_migrate.Migrate(app, db) # Configuring app for the test environment app.config.from_object('collector.api.config.Testing') +app.config.from_envvar('COLLECTOR_SETTINGS', silent=True) init_logger() diff --git a/migration/migration/test/test_es_mapping.py b/migration/migration/test/test_es_mapping.py index e11da13..7dc8a5a 100644 --- a/migration/migration/test/test_es_mapping.py +++ b/migration/migration/test/test_es_mapping.py @@ -1,13 +1,13 @@ # Copyright 2014 Mirantis, Inc. # -# Licensed under the Apache License, Version 2.0 (the 'License'); you may +# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2ebd159 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +alembic==0.6.7 +Flask==0.10.1 +Flask-JsonSchema==0.1.1 +Flask-Migrate==1.2.0 +Flask-Script==2.0.5 +Flask-SQLAlchemy==2.0 +psycopg2==2.5.4 +python-dateutil==2.2 +PyYAML==3.11 +six>=1.8.0 +SQLAlchemy==0.9.8 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c0e1e91 --- /dev/null +++ b/setup.py @@ -0,0 +1,64 @@ +# Copyright 2014 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from setuptools import find_packages +from setuptools import setup + + +def parse_requirements_txt(): + root = os.path.dirname(os.path.abspath(__file__)) + requirements = [] + with open(os.path.join(root, 'requirements.txt'), 'r') as f: + for line in f.readlines(): + line = line.rstrip() + if not line or line.startswith('#'): + continue + requirements.append(line) + return requirements + + +setup( + name='fuel-stats', + version='0.0.1', + description="Service of collecting usage statistics", + long_description="""Service of collecting usage statistics""", + license="http://www.apache.org/licenses/LICENSE-2.0", + classifiers=[ + "License :: OSI Approved :: Apache Software License", + "Development Status :: 3 - Alpha", + "Programming Language :: Python", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", + ], + author='Mirantis Inc.', + author_email='product@mirantis.com', + url='https://mirantis.com', + keywords='fuel statistics collector mirantis', + package_dir={'collector': 'collector/collector', + 'fuel_analytics': 'analytics/fuel_analytics', + 'migration': 'migration/migration'}, + packages=find_packages(where='collector') + + find_packages(where='analytics') + + find_packages(where='migration'), + zip_safe=False, + install_requires=parse_requirements_txt(), + include_package_data=True, + scripts=[ + 'analytics/manage_analytics.py', + 'collector/manage_collector.py', + 'migration/manage_migration.py', + ] +) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..cc0618c --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,7 @@ +-r requirements.txt +hacking==0.9.2 +mock==1.0.1 +nose==1.3.4 +nose2==0.4.7 +tox==1.8.0 +unittest2==0.5.1 diff --git a/tools/prepare_ci_config.sh b/tools/prepare_ci_config.sh new file mode 100755 index 0000000..4de10d0 --- /dev/null +++ b/tools/prepare_ci_config.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +cat > ${FUELSTAT_CI_CONFIG} < ${FUELSTAT_DB_ROOTPGPASS} +chmod 600 ${FUELSTAT_DB_ROOTPGPASS} + +export PGPASSFILE=${FUELSTAT_DB_ROOTPGPASS} + +echo "Trying to find out if role ${FUELSTAT_DB_USER} exists" +root_roles=$(psql -h 127.0.0.1 -U ${FUELSTAT_DB_ROOT} -t -c "SELECT 'HERE' from pg_roles where rolname='${FUELSTAT_DB_USER}'") +if [[ ${root_roles} == *HERE ]];then + echo "Role ${FUELSTAT_DB_USER} exists. Setting password ${FUELSTAT_DB_PW}" + psql -h 127.0.0.1 -U ${FUELSTAT_DB_ROOT} -c "ALTER ROLE ${FUELSTAT_DB_USER} WITH SUPERUSER LOGIN PASSWORD '${FUELSTAT_DB_PW}'" +else + echo "Creating role ${FUELSTAT_DB_USER} with password ${FUELSTAT_DB_PASSWD}" + psql -h 127.0.0.1 -U ${FUELSTAT_DB_ROOT} -c "CREATE ROLE ${FUELSTAT_DB_USER} WITH SUPERUSER LOGIN PASSWORD '${FUELSTAT_DB_PW}'" +fi + +echo "Dropping database ${FUELSTAT_DB} if exists" +psql -h 127.0.0.1 -U ${FUELSTAT_DB_ROOT} -c "DROP DATABASE IF EXISTS ${FUELSTAT_DB}" +echo "Creating database ${FUELSTAT_DB}" +psql -h 127.0.0.1 -U ${FUELSTAT_DB_ROOT} -c "CREATE DATABASE ${FUELSTAT_DB} OWNER ${FUELSTAT_DB_USER}" diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..22ac1ab --- /dev/null +++ b/tox.ini @@ -0,0 +1,57 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = py27,pep8 + +[testenv] +usedevelop = True +whitelist_externals = bash +install_command = pip install {packages} +setenv = VIRTUAL_ENV={envdir} + FUELSTAT_CI_CONFIG={toxinidir}/config_for_ci + COLLECTOR_SETTINGS={toxinidir}/config_for_ci + ANALYTICS_SETTINGS={toxinidir}/config_for_ci + FUELSTAT_DB=openstack_citest + FUELSTAT_DB_USER=openstack_citest + FUELSTAT_DB_PW=openstack_citest + FUELSTAT_DB_ROOT=postgres + FUELSTAT_DB_ROOTPW=insecure_slave + FUELSTAT_DB_ROOTPGPASS={toxinidir}/pgpass +deps = + -r{toxinidir}/test-requirements.txt +commands = + bash "{toxinidir}/tools/prepare_database.sh" + bash "{toxinidir}/tools/prepare_ci_config.sh" + python {toxinidir}/collector/manage_collector.py --mode test db upgrade \ + -d {toxinidir}/collector/collector/api/db/migrations/ + nosetests {posargs:collector/collector/test analytics/fuel_analytics/test} + +[tox:jenkins] +downloadcache = ~/cache/pip + +[testenv:pep8] +deps = hacking==0.7 +usedevelop = False +commands = + flake8 {posargs:analytics/fuel_analytics collector/collector \ + migration/migration} + +[testenv:cover] +setenv = NOSE_WITH_COVERAGE=1 + +[testenv:venv] +commands = {posargs:} + +[testenv:devenv] +envdir = devenv +usedevelop = True + +[flake8] +ignore = H302 +exclude = .venv,.git,.tox,dist,doc,*egg,build,docs +show-pep8 = True +show-source = True +count = True + +[hacking] +import_exceptions = testtools.matchers