Update openstack requests to go through ingress

The requests to Openstack services hang/fail sometimes
due to message loss when connecting to internal service
endpoints. This issue was observed before and fixed in
commit https://review.opendev.org/#/c/683818/ by setting
net.ipv4.tcp_tw_reuse to 0, however, it's still being
seen on recent STX loads.

It has been tested and proved that requests go through
ingress pod do not have the issue. This commit updates
helm charts and manifest to make all requests sending
to openstack services go to ingress pod and then ingress
would forward requests to the corresponding api service.

Changes included:
- update helm-toolkit manifest job-ks-endpoint.yaml
  to provide an ability to conditionally configure
  all types of openstack endpoints with public endpoint
  url when endpoints.identity.force_public_endpoint is
  true. Same update for keystone and keystone-api-proxy.
  With the update, for example, the admin,internal
  and public endpoints for neutron will be
  neutron.openstack.svc.cluster.local:80
- update armada manifest to make neccessary overrides
  in openstack service configuration file to make
  communications between services go through ingress

Change-Id: Icd646dd07d544da420a75f920bd7c3e8fc931327
Closes-Bug: 1880777
Signed-off-by: Angie Wang <angie.wang@windriver.com>
This commit is contained in:
Angie Wang 2020-09-02 19:18:07 -04:00
parent d73c7c494d
commit c14bed0043
6 changed files with 181 additions and 0 deletions

View File

@ -27,6 +27,7 @@ Patch09: 0009-Enable-override-of-mariadb-server-probe-parameters.patch
Patch10: 0010-Mariadb-use-utf8_general_ci-collation-as-default.patch
Patch11: 0011-Add-mariadb-database-config-override-to-support-ipv6.patch
Patch12: 0012-enable-Values.conf.database.config_override-for-mari.patch
Patch13: 0013-Allow-set-public-endpoint-url-for-all-openstack-types.patch
BuildRequires: helm
BuildRequires: chartmuseum
@ -48,6 +49,7 @@ Openstack Helm Infra charts
%patch10 -p1
%patch11 -p1
%patch12 -p1
%patch13 -p1
%build
# Host a server for the charts

View File

@ -0,0 +1,29 @@
From 7fd9d60e124868f277a1f2b0e86296dc50dd07ec Mon Sep 17 00:00:00 2001
From: Angie Wang <angie.wang@windriver.com>
Date: Mon, 31 Aug 2020 23:51:06 -0400
Subject: [PATCH 1/1] Allow set public endpoint url for all openstack
endpoint types
---
helm-toolkit/templates/manifests/_job-ks-endpoints.tpl | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
index 02f2013..39766f4 100644
--- a/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+++ b/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
@@ -84,7 +84,11 @@ spec:
- name: OS_SERVICE_TYPE
value: {{ $osServiceType | quote }}
- name: OS_SERVICE_ENDPOINT
+ {{- if and (hasKey $envAll.Values.endpoints.identity "force_public_endpoint") $envAll.Values.endpoints.identity.force_public_endpoint }}
+ value: {{ tuple $osServiceType "public" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+ {{- else }}
value: {{ tuple $osServiceType $osServiceEndPoint "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+ {{- end }}
{{- end }}
{{- end }}
volumes:
--
1.8.3.1

View File

@ -26,6 +26,7 @@ Patch04: 0004-Nova-chart-Support-ephemeral-pool-creation.patch
Patch05: 0005-Nova-Add-support-for-disabling-Readiness-Liveness-pr.patch
Patch06: 0006-Support-ingress-creation-for-keystone-admin-endpoint.patch
Patch07: 0007-Allow-more-generic-overrides-for-placeme.patch
Patch08: 0008-Allow-set-public-endpoint-url-for-keystone-endpoints.patch
BuildRequires: helm
BuildRequires: openstack-helm-infra
@ -44,6 +45,7 @@ Openstack Helm charts
%patch05 -p1
%patch06 -p1
%patch07 -p1
%patch08 -p1
%build
# Stage helm-toolkit in the local repo

View File

@ -0,0 +1,28 @@
From b272e8ff3a78f38ab82df7995233705611e99f81 Mon Sep 17 00:00:00 2001
From: Angie Wang <angie.wang@windriver.com>
Date: Tue, 1 Sep 2020 00:00:22 -0400
Subject: [PATCH 1/1] Allow set public endpoint url for keystone endpoints
---
keystone/templates/job-db-sync.yaml | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/keystone/templates/job-db-sync.yaml b/keystone/templates/job-db-sync.yaml
index 56a39b8..3d0681e 100644
--- a/keystone/templates/job-db-sync.yaml
+++ b/keystone/templates/job-db-sync.yaml
@@ -20,7 +20,11 @@ env:
- name: OS_BOOTSTRAP_ADMIN_URL
value: {{ tuple "identity" "admin" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
- name: OS_BOOTSTRAP_INTERNAL_URL
+ {{- if and (hasKey $envAll.Values.endpoints.identity "force_public_endpoint") $envAll.Values.endpoints.identity.force_public_endpoint }}
+ value: {{ tuple "identity" "public" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+ {{- else }}
value: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+ {{- end }}
- name: OS_BOOTSTRAP_PUBLIC_URL
value: {{ tuple "identity" "public" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
- name: OPENSTACK_CONFIG_FILE
--
1.8.3.1

View File

@ -61,7 +61,11 @@ spec:
- name: OS_SERVICE_TYPE
value: {{ $osServiceType | quote }}
- name: OS_SERVICE_ENDPOINT
{{- if and (hasKey $envAll.Values.endpoints.identity "force_public_endpoint") $envAll.Values.endpoints.identity.force_public_endpoint }}
value: {{ tuple $osRealServiceType "public" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
{{- else }}
value: {{ tuple $osRealServiceType $osServiceEndPoint "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
{{- end }}
- name: OS_SERVICE_REGION
value: {{ $osServiceRegion | quote }}
{{- end }}

View File

@ -385,6 +385,7 @@ data:
identity:
name: keystone
namespace: openstack
force_public_endpoint: true
labels:
api:
node_selector_key: openstack-control-plane
@ -487,6 +488,9 @@ data:
release_group: osh-openstack-keystone-api-proxy
component: test
values:
endpoints:
identity:
force_public_endpoint: true
images:
tags:
keystone_api_proxy: docker.io/starlingx/stx-keystone-api-proxy:master-centos-stable-latest
@ -495,6 +499,8 @@ data:
keystone_api_proxy:
identity:
bind_host: "::"
remote_host: keystone.openstack.svc.cluster.local
remote_port: 80
source:
type: tar
location: http://172.17.0.1/helm_charts/keystone-api-proxy-0.1.0.tgz
@ -561,10 +567,15 @@ data:
oslo_messaging:
statefulset:
name: osh-openstack-rabbitmq-rabbitmq
identity:
force_public_endpoint: true
conf:
barbican:
barbican_api:
bind_host: "::"
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
source:
type: tar
location: http://172.17.0.1/helm_charts/starlingx/barbican-0.1.0.tgz
@ -720,10 +731,21 @@ data:
oslo_messaging:
statefulset:
name: osh-openstack-rabbitmq-rabbitmq
identity:
force_public_endpoint: true
conf:
glance:
DEFAULT:
bind_host: "::"
registry_host: glance-reg.openstack.svc.cluster.local
registry_port: 80
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
glance_registry:
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
source:
type: tar
location: http://172.17.0.1/helm_charts/starlingx/glance-0.1.0.tgz
@ -808,13 +830,18 @@ data:
oslo_messaging:
statefulset:
name: osh-openstack-rabbitmq-rabbitmq
identity:
force_public_endpoint: true
conf:
cinder:
DEFAULT:
backup_driver: cinder.backup.drivers.ceph.CephBackupDriver
osapi_volume_listen: "::"
glance_api_servers: http://glance.openstack.svc.cluster.local:80/
keystone_authtoken:
interface: internal
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
storage: rbd
source:
type: tar
@ -1081,6 +1108,8 @@ data:
my_ip: "::"
keystone_authtoken:
interface: internal
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
vnc:
novncproxy_host: "::"
vncserver_listen: "::"
@ -1109,8 +1138,17 @@ data:
idle_timeout: 60
max_overflow: 64
max_pool_size: 1
glance:
api_servers: http://glance.openstack.svc.cluster.local:80/
ironic:
api_endpoint: http://ironic.openstack.svc.cluster.local:80/
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
placement:
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
neutron:
default_floating_pool: public
url: http://neutron.openstack.svc.cluster.local:80/
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
notifications:
notification_format: unversioned
filter_scheduler:
@ -1138,6 +1176,7 @@ data:
discover_hosts_in_cells_interval: 30
periodic_task_interval: -1
service_user:
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
send_service_user_token: true
metrics:
required: false
@ -1183,6 +1222,9 @@ data:
labels:
release_group: osh-openstack-placement
values:
endpoints:
identity:
force_public_endpoint: true
labels:
placement:
node_selector_key: openstack-control-plane
@ -1214,6 +1256,9 @@ data:
placement:
DEFAULT:
log_config_append: /etc/placement/logging.conf
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
wsgi_placement: |
Listen :::{{ tuple "placement" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
@ -1284,6 +1329,9 @@ data:
tags:
nova_api_proxy: docker.io/starlingx/stx-nova-api-proxy:master-centos-stable-latest
ks_endpoints: docker.io/starlingx/stx-heat:master-centos-stable-latest
endpoints:
identity:
force_public_endpoint: true
pod:
affinity:
anti:
@ -1296,6 +1344,8 @@ data:
nfvi_compute_listen: "::"
keystone_authtoken:
interface: internal
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
source:
type: tar
location: http://172.17.0.1/helm_charts/starlingx/nova-api-proxy-0.1.0.tgz
@ -1332,6 +1382,8 @@ data:
oslo_messaging:
statefulset:
name: osh-openstack-rabbitmq-rabbitmq
identity:
force_public_endpoint: true
pod:
replicas:
server: 2
@ -1516,6 +1568,11 @@ data:
root_helper: sudo
vhost:
vhost_user_enabled: true
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
nova:
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
dhcp_agent:
DEFAULT:
enable_isolated_metadata: true
@ -1636,6 +1693,18 @@ data:
enabled_raid_interfaces: no-raid
enabled_storage_interfaces: cinder,noop
enabled_vendor_interfaces: ipmitool,no-vendor
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
glance:
glance_host: glance.openstack.svc.cluster.local
glance_port: 80
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
inspector:
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
neutron:
url: http://neutron.openstack.svc.cluster.local:80/
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
api:
port: 6385
pxe:
@ -1654,6 +1723,8 @@ data:
port:
pxe_http:
default: 28080
identity:
force_public_endpoint: true
source:
type: tar
location: http://172.17.0.1/helm_charts/starlingx/ironic-0.1.0.tgz
@ -1697,6 +1768,8 @@ data:
oslo_cache:
hosts:
default: heat-memcached
identity:
force_public_endpoint: true
labels:
api:
node_selector_key: openstack-control-plane
@ -1735,6 +1808,13 @@ data:
stacks:global_index: rule:context_is_admin
software_configs:global_index: rule:context_is_admin
heat:
clients_keystone:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
trustee:
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
heat_api:
bind_host: "::"
heat_api_cloudwatch:
@ -1814,11 +1894,19 @@ data:
oslo_messaging:
statefulset:
name: osh-openstack-rabbitmq-rabbitmq
identity:
force_public_endpoint: true
jobs:
alarms_cleaner:
# daily at the 35 minute mark
cron: "35 */24 * * *"
conf:
aodh:
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
service_credentials:
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
wsgi_aodh: |
Listen :::{{ tuple "alarming" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
@ -1900,6 +1988,8 @@ data:
driver: mariadb
keystone_authtoken:
interface: internal
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
apache: |
Listen :::{{ tuple "metric" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
@ -1996,6 +2086,8 @@ data:
oslo_cache:
hosts:
default: memcached
identity:
force_public_endpoint: true
source:
type: tar
location: http://172.17.0.1/helm_charts/starlingx/gnocchi-0.1.0.tgz
@ -2032,6 +2124,9 @@ data:
release_group: osh-openstack-panko
component: test
values:
endpoints:
identity:
force_public_endpoint: true
pod:
user:
panko:
@ -2054,6 +2149,10 @@ data:
panko_events_cleaner: docker.io/starlingx/stx-panko:master-centos-stable-latest
test: null
conf:
panko:
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
paste:
composite:panko+noauth:
use: egg:Paste#urlmap
@ -2210,6 +2309,8 @@ data:
secret_db: false
secret_mongodb: false
endpoints:
identity:
force_public_endpoint: true
oslo_messaging:
statefulset:
name: osh-openstack-rabbitmq-rabbitmq
@ -2233,6 +2334,15 @@ data:
public: 80
conf:
ceilometer:
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
service_credentials:
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
dispatcher_gnocchi:
url: http://gnocchi.openstack.svc.cluster.local:80/
api:
aodh_url: http://aodh.openstack.svc.cluster.local:80/
cache:
expiration_time: 86400
compute:
@ -2724,6 +2834,9 @@ data:
release_group: osh-openstack-fm-rest-api
component: ks-endpoints
values:
endpoints:
identity:
force_public_endpoint: true
pod:
affinity:
anti:
@ -2749,6 +2862,9 @@ data:
fm:
api:
bind_host: "::"
keystone_authtoken:
auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
auth_url: http://keystone.openstack.svc.cluster.local:80/v3
source:
type: tar
location: http://172.17.0.1/helm_charts/starlingx/fm-rest-api-0.1.0.tgz