diff --git a/congress/datasources/json_ingester/exec_api.py b/congress/datasources/json_ingester/exec_api.py index aeed7afc2..55342a86f 100644 --- a/congress/datasources/json_ingester/exec_api.py +++ b/congress/datasources/json_ingester/exec_api.py @@ -65,15 +65,7 @@ class ExecApiManager(object): 'supported.'.auth_config['type']) name = config['name'] - port = config.get('api_endpoint_port') - if port is not None: - port_str = ':' + str(port) - else: - port_str = '' - self._exec_api_endpoints[name] = ( - config.get('api_endpoint_host', '').rstrip('/') - + port_str + '/' - + config.get('api_endpoint_path', '').lstrip('/')) + self._exec_api_endpoints[name] = config['api_endpoint'] self._exec_api_sessions[name] = session @lockutils.synchronized('congress_json_ingester_exec_api') diff --git a/congress/datasources/json_ingester/json_ingester.py b/congress/datasources/json_ingester/json_ingester.py index d194897b2..1aa2df601 100644 --- a/congress/datasources/json_ingester/json_ingester.py +++ b/congress/datasources/json_ingester/json_ingester.py @@ -74,15 +74,7 @@ class JsonIngester(datasource_driver.PollingDataSourceDriver): self._create_schema_and_tables() self.poll_time = self._config.get('poll_interval', 60) self._setup_table_key_sets() - port = self._config.get('api_endpoint_port') - if port is not None: - port_str = ':' + str(port) - else: - port_str = '' - self._api_endpoint = ( - self._config.get('api_endpoint_host', '').rstrip('/') - + port_str + '/' - + self._config.get('api_endpoint_path', '').lstrip('/')) + self._api_endpoint = self._config.get('api_endpoint') self._initialize_session() self._initialize_update_methods() if len(self.update_methods) > 0: @@ -251,8 +243,9 @@ class JsonIngester(datasource_driver.PollingDataSourceDriver): else: LOG.error('authentication type %s not supported.', auth_config.get['type']) - raise exception.BadConfig('authentication type {} not ' - 'supported.'.auth_config['type']) + raise exception.BadConfig( + 'authentication type {} not supported.'.format( + auth_config['type'])) def _initialize_update_methods(self): for table_name in self._config['tables']: @@ -263,17 +256,23 @@ class JsonIngester(datasource_driver.PollingDataSourceDriver): # variables in closure def update_method( table_name=table_name, table_info=table_info): - full_path = self._api_endpoint.rstrip( - '/') + '/' + table_info['api_path'].lstrip('/') - result = self._session.get(full_path).json() - # FIXME: generalize to other verbs? + try: + full_path = self._api_endpoint.rstrip( + '/') + '/' + table_info['api_path'].lstrip('/') + result = self._session.get(full_path).json() + # FIXME: generalize to other verbs? - jsonpath_expr = parser.parse(table_info['jsonpath']) - ingest_data = [match.value for match in - jsonpath_expr.find(result)] - self.state[table_name] = set( - [json.dumps(item, sort_keys=True) - for item in ingest_data]) + jsonpath_expr = parser.parse(table_info['jsonpath']) + ingest_data = [match.value for match in + jsonpath_expr.find(result)] + self.state[table_name] = set( + [json.dumps(item, sort_keys=True) + for item in ingest_data]) + except BaseException: + LOG.exception('Exception occurred while updating ' + 'table %s.%s from: URL %s', + self.name, table_name, + full_path) self.add_update_method(update_method, table_name) diff --git a/congress/tests/datasources/json_ingester/test_exec_api.py b/congress/tests/datasources/json_ingester/test_exec_api.py index f1c8e2f0a..bf4cf2b65 100644 --- a/congress/tests/datasources/json_ingester/test_exec_api.py +++ b/congress/tests/datasources/json_ingester/test_exec_api.py @@ -38,8 +38,7 @@ class TestExecApiManager(base.TestCase): self.test_configs = [ { - "api_endpoint_host": "test1", - "api_endpoint_path": "url", + "api_endpoint": "test1/url", "tables": { "flavors": { "poll": { @@ -75,8 +74,7 @@ class TestExecApiManager(base.TestCase): }, { "allow_exec_api": True, - "api_endpoint_host": "test2", - "api_endpoint_path": "url", + "api_endpoint": "test2/url", "tables": { "flavors": { "poll": { @@ -112,8 +110,7 @@ class TestExecApiManager(base.TestCase): }, { "allow_exec_api": True, - "api_endpoint_host": "test3", - "api_endpoint_path": "url", + "api_endpoint": "test3/url", "authentication": { "type": "keystone", "config": { diff --git a/devstack/plugin.sh b/devstack/plugin.sh index d1c2af049..266358f5d 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -70,28 +70,60 @@ function configure_congress { iniset $CONGRESS_CONF json_ingester db_connection ${db_connection_mysql/?*:\/\//postgresql:\/\/} iniset $CONGRESS_CONF json_ingester config_path "$CONGRESS_JSON_CONF_DIR" iniset $CONGRESS_CONF json_ingester config_reusables_path "$CONGRESS_JSON_CONF_REUSABLES_PATH" - echo "primary_host: http://$SERVICE_HOST" > "$CONGRESS_JSON_CONF_REUSABLES_PATH" - echo "keystone_admin_auth_config:" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH" + + # setup json ingester config files + if [[ ! -d $CONGRESS_JSON_CONF_DIR ]]; then + mkdir $CONGRESS_JSON_CONF_DIR + fi + echo "keystone_admin_auth_config:" > "$CONGRESS_JSON_CONF_REUSABLES_PATH" echo " type: keystone" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH" echo " config:" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH" echo " project_name: $OS_PROJECT_NAME" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH" echo " username: $OS_USERNAME" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH" echo " password: $OS_PASSWORD" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH" - echo " auth_url: http://$SERVICE_HOST/identity" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH" - local OS_PROJECT_ID="$(openstack project show $OS_PROJECT_NAME -f value -c id)" - echo "cinder_path: volume/v3/$OS_PROJECT_ID/" - echo "heat_path: orchestration/v1/$OS_PROJECT_ID/" - - if [[ ! -d $CONGRESS_JSON_CONF_DIR ]]; then - mkdir $CONGRESS_JSON_CONF_DIR - fi - cp -r $CONGRESS_DIR/etc/sample_json_ingesters/* $CONGRESS_JSON_CONF_DIR + echo " auth_url: `openstack catalog show keystone -f value -c endpoints | sed -n 's/^\s*public: //p' | sed -n '1p'`" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH" fi _congress_setup_keystone $CONGRESS_CONF keystone_authtoken } +function _configure_congress_json_ingester { + local endpoint=`openstack catalog show $1 -f value -c endpoints | sed -n 's/^\s*public:\s*//p' | sed -n '1p'` + if [[ ! -z $endpoint ]]; then + echo "$1_api_endpoint: $endpoint" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH" + _copy_congress_json_ingester_config $1 + fi +} + +function _copy_congress_json_ingester_config { + cp $CONGRESS_DIR/etc/sample_json_ingesters/$1.yaml $CONGRESS_JSON_CONF_DIR/ +} + function configure_congress_datasources { + if [ "$ENABLE_CONGRESS_JSON" == "True" ]; then + _configure_congress_json_ingester cinderv3 + _configure_congress_json_ingester glance + _configure_congress_json_ingester heat + _configure_congress_json_ingester keystone + _configure_congress_json_ingester magnum + _configure_congress_json_ingester masakari + _configure_congress_json_ingester mistral + _configure_congress_json_ingester neutron + _configure_congress_json_ingester nova + _configure_congress_json_ingester tacker + _configure_congress_json_ingester zun + _copy_congress_json_ingester_config monasca + _copy_congress_json_ingester_config cve + if [ "$CONGRESS_MULTIPROCESS_DEPLOYMENT" == "False" ]; then + restart_service devstack@congress.service + echo "Waiting for Congress to restart..." + if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget --no-proxy -q -O- http://$CONGRESS_HOST:$CONGRESS_PORT; do sleep 1; done"; then + die $LINENO "Congress did not restart" + fi + else + restart_service devstack@congress-datasources.service + fi + fi _configure_service neutron neutronv2 _configure_service neutron-qos neutronv2_qos _configure_service nova nova @@ -343,6 +375,13 @@ function start_congress_service_and_check { fi } +function _wait_for_congress { + echo "Waiting for Congress to start..." + if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget --no-proxy -q -O- http://$CONGRESS_HOST:$CONGRESS_PORT; do sleep 1; done"; then + die $LINENO "Congress did not start" + fi +} + # stop_congress() - Stop running processes (non-screen) function stop_congress { diff --git a/etc/sample_json_ingesters/cinder.yaml b/etc/sample_json_ingesters/cinderv3.yaml similarity index 86% rename from etc/sample_json_ingesters/cinder.yaml rename to etc/sample_json_ingesters/cinderv3.yaml index e0dc575a1..85cae685b 100644 --- a/etc/sample_json_ingesters/cinder.yaml +++ b/etc/sample_json_ingesters/cinderv3.yaml @@ -2,8 +2,7 @@ name: _volume poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: !ref cinder_path +api_endpoint: !ref cinderv3_api_endpoint tables: volumes: poll: diff --git a/etc/sample_json_ingesters/cve.yaml b/etc/sample_json_ingesters/cve.yaml index a7140db7c..7ab1a19a7 100644 --- a/etc/sample_json_ingesters/cve.yaml +++ b/etc/sample_json_ingesters/cve.yaml @@ -1,8 +1,7 @@ name: _cve allow_exec_api: true poll_interval: 86400 # one day -api_endpoint_host: https://cve.circl.lu -api_endpoint_path: api/ +api_endpoint: https://cve.circl.lu/api/ tables: linux_kernel: gin_index: false diff --git a/etc/sample_json_ingesters/glance.yaml b/etc/sample_json_ingesters/glance.yaml index ec7745555..04eedd47d 100644 --- a/etc/sample_json_ingesters/glance.yaml +++ b/etc/sample_json_ingesters/glance.yaml @@ -2,8 +2,7 @@ name: _image poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: v2/ +api_endpoint: !ref glance_api_endpoint tables: images: poll: diff --git a/etc/sample_json_ingesters/heat.yaml b/etc/sample_json_ingesters/heat.yaml index c054cf5c2..abf136ea0 100644 --- a/etc/sample_json_ingesters/heat.yaml +++ b/etc/sample_json_ingesters/heat.yaml @@ -2,8 +2,7 @@ name: _heat poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: !ref heat_path +api_endpoint: !ref heat_api_endpoint tables: stacks: poll: diff --git a/etc/sample_json_ingesters/keystone.yaml b/etc/sample_json_ingesters/keystone.yaml index 1251aaa19..a3a9da407 100644 --- a/etc/sample_json_ingesters/keystone.yaml +++ b/etc/sample_json_ingesters/keystone.yaml @@ -2,53 +2,50 @@ name: _identity poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: identity/v3/ -api_default_headers: - X-OpenStack-Nova-API-Version: "2.26" +api_endpoint: !ref keystone_api_endpoint tables: domains: poll: - api_path: domains + api_path: v3/domains api_method: get jsonpath: $.domains[:] groups: poll: - api_path: groups + api_path: v3/groups api_method: get jsonpath: $.groups[:] projects: poll: - api_path: projects + api_path: v3/projects api_method: get jsonpath: $.projects[:] regions: poll: - api_path: regions + api_path: v3/regions api_method: get jsonpath: $.regions[:] roles: poll: - api_path: roles + api_path: v3/roles api_method: get jsonpath: $.roles[:] services: poll: - api_path: services + api_path: v3/services api_method: get jsonpath: $.services[:] endpoints: poll: - api_path: endpoints + api_path: v3/endpoints api_method: get jsonpath: $.endpoints[:] registered_limits: poll: - api_path: registered_limits + api_path: v3/registered_limits api_method: get jsonpath: $.registered_limits[:] users: poll: - api_path: users + api_path: v3/users api_method: get jsonpath: $.users[:] diff --git a/etc/sample_json_ingesters/magnum.yaml b/etc/sample_json_ingesters/magnum.yaml index 55d3bce7c..f7f31b63d 100644 --- a/etc/sample_json_ingesters/magnum.yaml +++ b/etc/sample_json_ingesters/magnum.yaml @@ -2,8 +2,7 @@ name: _magnum poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: magnum/v1/ +api_endpoint: !ref magnum_api_endpoint tables: bays: poll: diff --git a/etc/sample_json_ingesters/masakari.yaml b/etc/sample_json_ingesters/masakari.yaml index 43376aef6..e7c96f5fe 100644 --- a/etc/sample_json_ingesters/masakari.yaml +++ b/etc/sample_json_ingesters/masakari.yaml @@ -2,9 +2,7 @@ name: _masakari poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: v1/ -api_endpoint_port: 15868 +api_endpoint: !ref masakari_api_endpoint tables: segments: poll: diff --git a/etc/sample_json_ingesters/mistral.yaml b/etc/sample_json_ingesters/mistral.yaml index 22f4c11d9..4c9e3d7f1 100644 --- a/etc/sample_json_ingesters/mistral.yaml +++ b/etc/sample_json_ingesters/mistral.yaml @@ -2,9 +2,7 @@ name: _mistral poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: v2/ -api_endpoint_port: 8989 +api_endpoint: !ref mistral_api_endpoint tables: workbooks: poll: @@ -46,8 +44,11 @@ tables: api_path: environments api_method: get jsonpath: $.environments[:] - services: - poll: - api_path: services - api_method: get - jsonpath: $.services[:] +# This API is not supported by default. +# uncomment if available +# see: https://docs.openstack.org/mistral/latest/api/v2.html#services +# services: +# poll: +# api_path: services +# api_method: get +# jsonpath: $.services[:] diff --git a/etc/sample_json_ingesters/neutron-fwaas.yaml b/etc/sample_json_ingesters/neutron-fwaas.yaml new file mode 100644 index 000000000..5e78c2c7e --- /dev/null +++ b/etc/sample_json_ingesters/neutron-fwaas.yaml @@ -0,0 +1,21 @@ +name: _fwaas +poll_interval: 60 +allow_exec_api: true +authentication: !ref keystone_admin_auth_config +api_endpoint: !ref neutron_api_endpoint +tables: + firewall_groups: + poll: + api_path: v2.0/fwaas/firewall_groups + api_method: get + jsonpath: $.firewall_groups[:] + firewall_policies: + poll: + api_path: v2.0/fwaas/firewall_policies + api_method: get + jsonpath: $.firewall_policies[:] + firewall_rules: + poll: + api_path: v2.0/fwaas/firewall_rules + api_method: get + jsonpath: $.firewall_rules[:] diff --git a/etc/sample_json_ingesters/neutron.yaml b/etc/sample_json_ingesters/neutron.yaml index 1b7d0b5fb..3b0950acf 100644 --- a/etc/sample_json_ingesters/neutron.yaml +++ b/etc/sample_json_ingesters/neutron.yaml @@ -2,62 +2,45 @@ name: _networking poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: v2.0/ -api_endpoint_port: 9696 +api_endpoint: !ref neutron_api_endpoint tables: networks: poll: - api_path: networks + api_path: v2.0/networks api_method: get jsonpath: $.networks[:] ports: poll: - api_path: ports + api_path: v2.0/ports api_method: get jsonpath: $.ports[:] floatingips: poll: - api_path: floatingips + api_path: v2.0/floatingips api_method: get jsonpath: $.floatingips[:] routers: poll: - api_path: routers + api_path: v2.0/routers api_method: get jsonpath: $.routers[:] subnets: poll: - api_path: subnets + api_path: v2.0/subnets api_method: get jsonpath: $.subnets[:] - firewall_groups: - poll: - api_path: fwaas/firewall_groups - api_method: get - jsonpath: $.firewall_groups[:] - firewall_policies: - poll: - api_path: fwaas/firewall_policies - api_method: get - jsonpath: $.firewall_policies[:] - firewall_rules: - poll: - api_path: fwaas/firewall_rules - api_method: get - jsonpath: $.firewall_rules[:] security_groups: poll: - api_path: security-groups + api_path: v2.0/security-groups api_method: get jsonpath: $.security_groups[:] rbac_policies: poll: - api_path: rbac-policies + api_path: v2.0/rbac-policies api_method: get jsonpath: $.rbac_policies[:] agents: poll: - api_path: agents + api_path: v2.0/agents api_method: get jsonpath: $.agents[:] diff --git a/etc/sample_json_ingesters/nova.yaml b/etc/sample_json_ingesters/nova.yaml index 4c3304d7f..4d1b5d4b7 100644 --- a/etc/sample_json_ingesters/nova.yaml +++ b/etc/sample_json_ingesters/nova.yaml @@ -2,8 +2,7 @@ name: _compute poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: compute/v2.1/ +api_endpoint: !ref nova_api_endpoint api_default_headers: X-OpenStack-Nova-API-Version: "2.26" tables: diff --git a/etc/sample_json_ingesters/tacker.yaml b/etc/sample_json_ingesters/tacker.yaml index 6765c762b..34fedad5b 100644 --- a/etc/sample_json_ingesters/tacker.yaml +++ b/etc/sample_json_ingesters/tacker.yaml @@ -2,17 +2,15 @@ name: _tacker poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: v1.0/ -api_endpoint_port: 9890 +api_endpoint: !ref tacker_api_endpoint tables: vnfds: poll: - api_path: vnfds + api_path: v1.0/vnfds api_method: get jsonpath: $.vnfds[:] vnfs: poll: - api_path: vnfs + api_path: v1.0/vnfs api_method: get jsonpath: $.vnfs[:] diff --git a/etc/sample_json_ingesters/zun.yaml b/etc/sample_json_ingesters/zun.yaml index 933ed670e..c756ac779 100644 --- a/etc/sample_json_ingesters/zun.yaml +++ b/etc/sample_json_ingesters/zun.yaml @@ -2,8 +2,9 @@ name: _zun poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config -api_endpoint_host: !ref primary_host -api_endpoint_path: zun/v1/ +api_endpoint: !ref zun_api_endpoint +api_default_headers: + OpenStack-API-Version: container 1.4 tables: containers: poll: