fix: migrate CI to jammy

fix: change tox.ini

fix: change queries for list_dimension_names and list_dimension_values because of influxdb time filter problem

fix: remove build_sphinx group from setup.cfg

fix: handle hashlib security problem

Change-Id: I0d31a8db5ed71c70e7b878ce5e7940e041d0fa43

Change-Id: I6f7066da10e834550cbf0c053c7bf425ac0ead93

Change-Id: If9575aee73d600bbc84fcdf58deb1c57b508d9c2

Change-Id: If515eaeee7539da3ca49997e88785dc65572b334
(cherry picked from commit a24fd834d0)
This commit is contained in:
Hasan Acar 2024-01-02 12:42:36 +00:00 committed by Gökhan
parent 9b270faeb2
commit 3f355890a9
16 changed files with 145 additions and 75 deletions

View File

@ -1,7 +1,8 @@
openjdk-8-jdk # dist:xenial,bionic,focal
openjdk-8-jre-headless # dist:bionic,focal
maven # dist:xenial,bionic,focal
jq # dist:xenial,bionic,focal
openjdk-8-jdk # dist:xenial,bionic,focal,jammy
openjdk-8-jre-headless # dist:bionic,focal,jammy
maven # dist:xenial,bionic,focal,jammy
jq # dist:xenial,bionic,focal,jammy
python-dev # dist:xenial,bionic,focal
build-essential # dist:xenial,bionic,focal
mailutils # dist:xenial,bionic,focal
build-essential # dist:xenial,bionic,focal,jammy
mailutils # dist:xenial,bionic,focal,jammy
python-is-python3 # dist:focal,jammy

View File

@ -309,7 +309,7 @@ function install_logstash {
tar xzf ${logstash_dest} -C $DEST
sudo chown -R $STACK_USER $DEST/logstash-${LOGSTASH_VERSION}
ln -sf $DEST/logstash-${LOGSTASH_VERSION} $LOGSTASH_DIR
sudo ln -sf $DEST/logstash-${LOGSTASH_VERSION} $LOGSTASH_DIR
sudo mkdir -p $LOGSTASH_DATA_DIR
sudo chown $STACK_USER:monasca $LOGSTASH_DATA_DIR
@ -339,7 +339,7 @@ function install_elasticsearch {
tar xzf ${es_dest} -C $DEST
sudo chown -R $STACK_USER $DEST/elasticsearch-${ELASTICSEARCH_VERSION}
ln -sf $DEST/elasticsearch-${ELASTICSEARCH_VERSION} $ELASTICSEARCH_DIR
sudo ln -sf $DEST/elasticsearch-${ELASTICSEARCH_VERSION} $ELASTICSEARCH_DIR
fi
}
@ -364,7 +364,7 @@ function configure_elasticsearch {
s|%ES_LOG_DIR%|$ELASTICSEARCH_LOG_DIR|g;
" -i $ELASTICSEARCH_CFG_DIR/elasticsearch.yml
ln -sf $ELASTICSEARCH_CFG_DIR/elasticsearch.yml $GATE_CONFIGURATION_DIR/elasticsearch.yml
sudo ln -sf $ELASTICSEARCH_CFG_DIR/elasticsearch.yml $GATE_CONFIGURATION_DIR/elasticsearch.yml
echo "[Service]" | sudo tee --append /etc/systemd/system/devstack\@elasticsearch.service > /dev/null
echo "LimitNOFILE=$LIMIT_NOFILE" | sudo tee --append /etc/systemd/system/devstack\@elasticsearch.service > /dev/null
@ -420,7 +420,7 @@ function install_kibana {
local kibana_version_name
kibana_version_name=`_get_kibana_version_name`
sudo chown -R $STACK_USER $DEST/${kibana_version_name}
ln -sf $DEST/${kibana_version_name} $KIBANA_DIR
sudo ln -sf $DEST/${kibana_version_name} $KIBANA_DIR
fi
}
@ -443,7 +443,7 @@ function configure_kibana {
s|%KEYSTONE_AUTH_URI%|$KEYSTONE_AUTH_URI|g;
" -i $KIBANA_CFG_DIR/kibana.yml
ln -sf $KIBANA_CFG_DIR/kibana.yml $GATE_CONFIGURATION_DIR/kibana.yml
sudo ln -sf $KIBANA_CFG_DIR/kibana.yml $GATE_CONFIGURATION_DIR/kibana.yml
fi
}
@ -522,7 +522,7 @@ function build_kibana_plugin {
yarn --cwd $KIBANA_DEV_DIR kbn bootstrap
yarn --cwd $plugin_dir build
local get_version_script="import json; obj = json.load(open('$plugin_dir/package.json')); print obj['version']"
local get_version_script="import json; obj = json.load(open('$plugin_dir/package.json')); print(obj['version'])"
local monasca_kibana_plugin_version
monasca_kibana_plugin_version=$(python -c "$get_version_script")
local pkg="$plugin_dir/build/monasca-kibana-plugin-$monasca_kibana_plugin_version.zip"

View File

@ -1154,7 +1154,7 @@ function install_monasca_agent {
if is_service_enabled monasca-agent; then
echo_summary "Install Monasca monasca_agent"
apt_get install python-yaml libxml2-dev libxslt1-dev
apt_get install python3-yaml libxml2-dev libxslt1-dev
MONASCA_AGENT_EXTRAS="kafka_plugin"
if is_service_enabled nova && [ "$VIRT_DRIVER" = "libvirt" ]; then
@ -1262,7 +1262,7 @@ function clean_monasca_agent {
apt_get purge libxslt1-dev
apt_get purge libxml2-dev
apt_get purge python-yaml
apt_get purge python3-yaml
fi
}
@ -1271,7 +1271,7 @@ function clean_monasca_agent {
function install_nodejs {
echo_summary "Install Node.js"
curl -sL https://deb.nodesource.com/setup_10.x | sudo bash -
curl -sL https://deb.nodesource.com/setup_18.x | sudo bash -
apt_get install nodejs
npm config set registry "http://registry.npmjs.org/"; \

View File

@ -4,3 +4,4 @@ os-api-ref>=1.4.0 # Apache-2.0
reno>=3.1.0 # Apache-2.0
openstackdocstheme>=2.2.1 # Apache-2.0
SQLAlchemy>=1.3.0 # MIT
oslo.config>=6.8.0 # Apache-2.0

View File

@ -14,5 +14,3 @@ Modules
.. toctree::
:maxdepth: 2
api/autoindex.rst

View File

@ -113,6 +113,29 @@ Administrating
admin/index
Glossary
-------------
.. toctree::
:maxdepth: 2
glossary
Installation
------------
.. toctree::
:maxdepth: 2
install/index
User
------------
.. toctree::
:maxdepth: 2
user/index
Configuration
-------------
@ -124,7 +147,3 @@ Configuration
admin/index
cli/index
configuration/sample
.. only:: html
glossary
install/index
user/index

View File

@ -8,7 +8,7 @@ ARG CONSTRAINTS_BRANCH=master
# Extra Python3 dependencies.
# gevent is not in upper constrains and v1.3.6 is not working with
# older greenlet.
ARG EXTRA_DEPS="gunicorn gevent==1.5.0 python-memcached influxdb"
ARG EXTRA_DEPS="gunicorn gevent>=21.12.0 python-memcached influxdb"
# Always start from `monasca-base` image and use specific tag of it.
ARG BASE_TAG=master

View File

@ -41,7 +41,7 @@ def do_detect_revision():
fingerprint = Fingerprint(sql_repository.get_engine())
if fingerprint.revision is None:
print(_FP_NOREVISION % fingerprint.sha1)
print(_FP_NOREVISION % fingerprint.sha256)
sys.exit(1)
else:
print(fingerprint.revision)
@ -52,7 +52,7 @@ def do_fingerprint():
if CONF.command.raw:
print(fingerprint.schema_raw, end="")
else:
print(fingerprint.sha1)
print(fingerprint.sha256)
def do_stamp():
@ -71,7 +71,7 @@ def do_stamp():
else:
fp = Fingerprint(engine)
if fp.revision is None:
print(_FP_NOREVISION % fp.sha1)
print(_FP_NOREVISION % fp.sha256)
sys.exit(1)
rev = fp.revision
@ -110,7 +110,7 @@ def do_version():
def add_command_parsers(subparsers):
parser = subparsers.add_parser('fingerprint',
help="Compute SHA1 fingerprint of "
help="Compute SHA256 fingerprint of "
"current database schema ")
parser.add_argument('-r', '--raw', action='store_true',
help='Print raw schema dump used for '

View File

@ -214,6 +214,20 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
return query
def _build_select_all_query(self, dimensions, name, tenant_id,
region, start_timestamp, end_timestamp,
no_record_check_dim):
from_clause = self._build_from_clause(dimensions, name, tenant_id,
region, start_timestamp,
end_timestamp)
if no_record_check_dim is not None:
query = 'select *' + from_clause + " and {} != ''".format(no_record_check_dim)
else:
query = 'select *' + from_clause
return query
def _build_statistics_query(self, dimensions, name, tenant_id,
region, start_timestamp, end_timestamp,
statistics, period, offset, group_by, limit):
@ -263,7 +277,7 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
# replace ' with \' to make query parsable
clean_name = name.replace("'", "\\'") if PY3 \
else name.replace("'", "\\'").encode('utf-8')
where_clause += ' from "{}" '.format(clean_name)
where_clause += ' from "{}"'.format(clean_name)
# region
where_clause += " where _region = '{}'".format(region)
@ -889,9 +903,25 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
start_timestamp,
end_timestamp)
result = self.query_tenant_db(query, tenant_id)
json_dim_name_list = self._build_serie_dimension_values(
json_dim_value_list = self._build_serie_dimension_values(
result, dimension_name)
return json_dim_name_list
json_dim_value_list_filtered = list()
for serie in result.raw['series']:
for dim_value_dict in json_dim_value_list:
query = self._build_select_all_query(
dimensions={dimension_name: dim_value_dict['dimension_value']},
name=serie['name'],
tenant_id=tenant_id,
region=region,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
no_record_check_dim=None)
result = self.query_tenant_db(query, tenant_id)
if len(result.raw['series']) > 0:
json_dim_value_list_filtered.append(dim_value_dict)
json_dim_value_list_filtered = sorted(json_dim_value_list_filtered, key=lambda
dim_value_dict: dim_value_dict['dimension_value'])
return json_dim_value_list_filtered
except Exception as ex:
LOG.exception(ex)
raise exceptions.RepositoryException(ex)
@ -905,7 +935,26 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
end_timestamp)
result = self.query_tenant_db(query, tenant_id)
json_dim_name_list = self._build_serie_dimension_names(result)
return json_dim_name_list
if metric_name is not None:
json_dim_name_list_filtered = list()
for dim_name_dict in json_dim_name_list:
query = self._build_select_all_query(
dimensions=None,
name=metric_name,
tenant_id=tenant_id,
region=region,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
no_record_check_dim=dim_name_dict['dimension_name'])
result = self.query_tenant_db(query, tenant_id)
if len(result.raw['series']) > 0:
json_dim_name_list_filtered.append(dim_name_dict)
json_dim_name_list_filtered = sorted(json_dim_name_list_filtered, key=lambda
dim_name_dict: dim_name_dict['dimension_name'])
return json_dim_name_list_filtered
else:
return json_dim_name_list
except Exception as ex:
LOG.exception(ex)
raise exceptions.RepositoryException(ex)
@ -915,7 +964,7 @@ class MetricsRepository(metrics_repository.AbstractMetricsRepository):
uri = 'http://{0}:{1}/ping'.format(CONF.influxdb.ip_address,
CONF.influxdb.port)
try:
resp = requests.head(url=uri)
resp = requests.head(url=uri, timeout=5)
except Exception as ex:
LOG.exception(str(ex))
return False, str(ex)

View File

@ -21,7 +21,7 @@ from sqlalchemy.orm import sessionmaker
LOG = log.getLogger(__name__)
# Map of SHA1 fingerprints to alembic revisions. Note that this is
# Map of SHA256 fingerprints to alembic revisions. Note that this is
# used in the pre-alembic case and does not need to be updated if a
# new revision is introduced.
_REVS = {"43e5913b0272077321ab6f25ffbcda7149b6284b": "00597b5c8325",
@ -43,8 +43,8 @@ class Fingerprint(object):
def __init__(self, engine):
metadata = self._get_metadata(engine)
self.schema_raw = self._get_schema_raw(metadata)
self.sha1 = self._get_schema_sha1(self.schema_raw)
self.revision = self._get_revision(metadata, engine, self.sha1)
self.sha256 = self._get_schema_sha256(self.schema_raw)
self.revision = self._get_revision(metadata, engine, self.sha256)
@staticmethod
def _get_metadata(engine):
@ -75,18 +75,18 @@ class Fingerprint(object):
return "\n".join(schema_strings)
@staticmethod
def _get_schema_sha1(schema_raw):
return hashlib.sha1(encodeutils.to_utf8(schema_raw)).hexdigest()
def _get_schema_sha256(schema_raw):
return hashlib.sha256(encodeutils.to_utf8(schema_raw)).hexdigest()
@staticmethod
def _get_revision(metadata, engine, sha1):
def _get_revision(metadata, engine, sha256):
# Alembic stores the current version in the DB so check that first
# and fall back to the lookup table for the pre-alembic case.
versions_table = metadata.tables.get('alembic_version')
if versions_table is not None:
return Fingerprint._lookup_version_from_db(versions_table, engine)
elif sha1:
return Fingerprint._lookup_version_from_table(sha1)
elif sha256:
return Fingerprint._lookup_version_from_table(sha256)
@staticmethod
def _get_db_session(engine):
@ -102,9 +102,9 @@ class Fingerprint(object):
return session.query(versions_table).one()[0]
@staticmethod
def _lookup_version_from_table(sha1):
revision = _REVS.get(sha1)
def _lookup_version_from_table(sha256):
revision = _REVS.get(sha256)
if not revision:
LOG.warning("Fingerprint: {} does not match any revisions."
.format(sha1))
.format(sha256))
return revision

View File

@ -35,11 +35,11 @@ class TestFingerprint(base.BaseTestCase):
# table of fingerprints. Since we use a dummy schema, we insert a dummy
# entry into the lookup table.
fingerprint._REVS[
hashlib.sha1(b'dummy_schema_raw').hexdigest()] = 'dummy_revision'
hashlib.sha256(b'dummy_schema_raw').hexdigest()] = 'dummy_revision'
f = fingerprint.Fingerprint('mock_engine')
self.assertEqual(f.schema_raw, 'dummy_schema_raw')
self.assertEqual(f.sha1, hashlib.sha1(b'dummy_schema_raw').hexdigest())
self.assertEqual(f.sha256, hashlib.sha256(b'dummy_schema_raw').hexdigest())
self.assertEqual(f.revision, 'dummy_revision')
@mock.patch('monasca_api.db.fingerprint.Fingerprint._get_db_session')
@ -61,5 +61,5 @@ class TestFingerprint(base.BaseTestCase):
f = fingerprint.Fingerprint('mock_engine')
self.assertEqual(f.schema_raw, 'dummy_schema_raw')
self.assertEqual(f.sha1, hashlib.sha1(b'dummy_schema_raw').hexdigest())
self.assertEqual(f.sha256, hashlib.sha256(b'dummy_schema_raw').hexdigest())
self.assertEqual(f.revision, 'dummy_revision')

View File

@ -221,18 +221,20 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase):
self.assertEqual(result, [{u'dimension_value': hostname}])
query = ('show tag values from "{metric}"'
' with key = "{column}"'
query = ('select * from "{metric}"'
' where _region = \'{region}\''
.format(region=region, metric=metric, column=column))
.format(region=region, metric=metric))
query += ('' if db_per_tenant else ' and _tenant_id = \'{tenant_id}\''
.format(tenant_id=tenant_id))
query += (' and "{column}" = \'{hostname}\''
.format(column=column,
hostname=hostname))
query += (' and time >= {start_timestamp}000000u'
' and time < {end_timestamp}000000u'
.format(start_timestamp=start_timestamp,
end_timestamp=end_timestamp)
if timestamp else '')
mock_client.query.assert_called_once_with(query, database=database)
mock_client.query.assert_called_with(query, database=database)
def test_list_dimension_values_with_timestamp(self):
self.test_list_dimension_values(timestamp=True)
@ -275,18 +277,19 @@ class TestRepoMetricsInfluxDB(base.BaseTestCase):
{u'dimension_name': u'service'}
])
query = ('show tag keys from "{metric}"'
' where _region = \'{region}\''
.format(region=region, metric=metric))
query += ('' if db_per_tenant else ' and _tenant_id = \'{tenant_id}\''
.format(tenant_id=tenant_id))
query += (' and time >= {start_timestamp}000000u'
' and time < {end_timestamp}000000u'
.format(start_timestamp=start_timestamp,
end_timestamp=end_timestamp)
if timestamp else '')
query_last = ('select * from "{metric}"'
' where _region = \'{region}\''
.format(region=region, metric=metric))
query_last += ('' if db_per_tenant else ' and _tenant_id = \'{tenant_id}\''
.format(tenant_id=tenant_id))
query_last += (' and time >= {start_timestamp}000000u'
' and time < {end_timestamp}000000u'
.format(start_timestamp=start_timestamp,
end_timestamp=end_timestamp)
if timestamp else '')
query_last += (' and service != \'\'')
mock_client.query.assert_called_once_with(query, database=database)
mock_client.query.assert_called_with(query_last, database=database)
def test_list_dimension_names_with_timestamp(self):
self.test_list_dimension_names(timestamp=True)

View File

@ -3,4 +3,4 @@ Yoga Series Release Notes
=========================
.. release-notes::
:branch: stable/yoga
:branch: unmaintained/yoga

View File

@ -7,7 +7,7 @@ description_file =
author = OpenStack
author_email = openstack-discuss@lists.openstack.org
home_page = https://docs.openstack.org/monasca-api/latest/
python_requires = >=3.6
python_requires = >=3.8
classifier =
Environment :: OpenStack
Intended Audience :: Information Technology
@ -17,9 +17,10 @@ classifier =
Programming Language :: Python
Programming Language :: Python :: Implementation :: CPython
Programming Language :: Python :: 3 :: Only
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
Programming Language :: Python :: 3.11
[files]
packages =
@ -65,8 +66,3 @@ autodoc_exclude_modules =
monasca_tempest_tests.*
api_doc_dir = contributor/api
[build_sphinx]
all_files = 1
build-dir = doc/build
source-dir = doc/source
warning-is-error = 1

View File

@ -23,3 +23,4 @@ testtools>=2.2.0 # MIT
tempest>=17.1.0 # Apache-2.0
doc8>=0.6.0 # Apache-2.0
oslo.config>=6.8.0 # Apache-2.0

14
tox.ini
View File

@ -1,7 +1,8 @@
[tox]
envlist = py38,pep8,cover
envlist = py3,pep8,cover
minversion = 2.7
skipsdist = True
ignore_basepython_conflict = True
[testenv]
basepython = python3
@ -17,9 +18,10 @@ deps =
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/requirements.txt
.[influxdb,cassandra]
whitelist_externals = bash
find
allowlist_externals = bash
find
rm
make
commands =
find . -type f -name "*.pyc" -delete
stestr run {posargs}
@ -49,7 +51,7 @@ skip_install = True
usedevelop = False
commands =
# B101(assert_ussed) - API uses asserts because of performance reasons
# B303 - Fingerprint class uses SHA1 to map fingerprints to alembic revisions.
# B303 - Fingerprint class uses SHA256 to map fingerprints to alembic revisions.
bandit -r monasca_api -n5 -s B101,B303 -x monasca_api/tests
[testenv:bashate]
@ -76,7 +78,7 @@ commands =
[testenv:pdf-docs]
deps = {[testenv:docs]deps}
envdir = {toxworkdir}/docs
whitelist_externals =
allowlist_externals =
make
rm
commands =
@ -107,7 +109,7 @@ description = Builds developer documentation
commands =
rm -rf doc/build doc/source/contributor/api
{[testenv:checkjson]commands}
python setup.py build_sphinx
sphinx-build -W -b html doc/source/ doc/build/html
[testenv:checkniceness]
skip_install = True