diff --git a/contrib/regenerate_pod_resources_api.sh b/contrib/regenerate_pod_resources_api.sh new file mode 100755 index 000000000..ea36a2978 --- /dev/null +++ b/contrib/regenerate_pod_resources_api.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +set -o errexit + +# Exit early if python3 is not available. +python3 --version > /dev/null + +KURYR_DIR=${KURYR_DIR:-./} +KURYR_API_PROTO="kuryr_kubernetes/pod_resources/api.proto" + +# If API_VERSION is not specified assuming v1alpha1. +VERSION=${API_VERSION:-v1alpha1} + +ACTIVATED="no" +ENV_DIR=$(mktemp -d -t kuryr-tmp-env-XXXXXXXXXX) + +function cleanup() { + if [ "${ACTIVATED}" = "yes" ]; then deactivate; fi + rm -rf "${ENV_DIR}" +} +trap cleanup EXIT INT + +if [ -z "${KUBERNETES_API_PROTO}" ]; then + + echo "KUBERNETES_API_PROTO is not specified." \ + "Trying to download api.proto from the k8s github." + + pushd "${ENV_DIR}" + + BASE_URL="https://raw.githubusercontent.com/kubernetes/kubernetes/master" + PROTO_FILE="pkg/kubelet/apis/podresources/${VERSION}/api.proto" + + wget "${BASE_URL}/${PROTO_FILE}" -O api.proto + + KUBERNETES_API_PROTO="$PWD/api.proto" + popd +fi + +if [ ! -f "${KUBERNETES_API_PROTO}" ]; then + echo "Can't find ${KUBERNETES_API_PROTO}" + exit 1 +fi + +KUBERNETES_API_PROTO=$(readlink -e "${KUBERNETES_API_PROTO}") + +pushd "${KURYR_DIR}" + +# Obtaining api version from the proto file. +VERSION=$(grep package "${KUBERNETES_API_PROTO}" \ + | sed 's/^package *\(.*\)\;$/\1/') +echo "\ +// Generated from kubernetes/pkg/kubelet/apis/podresources/${VERSION}/api.proto +// To regenerate api.proto, api_pb2.py and api_pb2_grpc.py follow instructions +// from doc/source/devref/updating_pod_resources_api.rst. +" > ${KURYR_API_PROTO} + +# Stripping unwanted dependencies. +sed '/gogoproto/d;/api.pb.go/d' "${KUBERNETES_API_PROTO}" >> ${KURYR_API_PROTO} +echo '' >> ${KURYR_API_PROTO} +# Stripping redundant empty lines. +sed -i '/^$/N;/^\n$/D' ${KURYR_API_PROTO} + +# Creating new virtual environment. +python3 -m venv "${ENV_DIR}" +source "${ENV_DIR}/bin/activate" +ACTIVATED="yes" + +pip install grpcio-tools==1.19 + +# Checking protobuf version. +protobuf_version=$(grep protobuf lower-constraints.txt \ + | sed 's/^protobuf==\([0-9\.]*\)\.[0-9]*$/\1/') +protoc_version=$(python -m grpc_tools.protoc --version \ + | sed 's/^libprotoc \([0-9\.]*\)\.[0-9]*$/\1/') +if [ "${protobuf_version}" != "${protoc_version}" ]; then + echo "protobuf version in lower-constraints.txt (${protobuf_version})" \ + "!= installed protoc compiler version (${protoc_version})." + echo "Please, update requirements.txt and lower-constraints.txt or" \ + "change version of grpcio-tools used in this script." + # Clearing api.proto to highlight the issue. + echo '' > ${KURYR_API_PROTO} + exit 1 +fi + +# Generating python bindings. +python -m grpc_tools.protoc -I./ \ + --python_out=. --grpc_python_out=. ${KURYR_API_PROTO} +popd diff --git a/doc/source/devref/index.rst b/doc/source/devref/index.rst index fb10af773..03ae06795 100644 --- a/doc/source/devref/index.rst +++ b/doc/source/devref/index.rst @@ -46,6 +46,7 @@ Design documents kuryr_kubernetes_versions port_crd_usage network_policy + updating_pod_resources_api Indices and tables ------------------ diff --git a/doc/source/devref/updating_pod_resources_api.rst b/doc/source/devref/updating_pod_resources_api.rst new file mode 100644 index 000000000..c76435af8 --- /dev/null +++ b/doc/source/devref/updating_pod_resources_api.rst @@ -0,0 +1,119 @@ +.. + This work is licensed under a Creative Commons Attribution 3.0 Unported + License. + + http://creativecommons.org/licenses/by/3.0/legalcode + + Convention for heading levels in Neutron devref: + ======= Heading 0 (reserved for the title in a document) + ------- Heading 1 + ~~~~~~~ Heading 2 + +++++++ Heading 3 + ''''''' Heading 4 + (Avoid deeper levels because they do not render well.) + +================================== +HowTo Update PodResources gRPC API +================================== + + +Purpose +------- +The purpose of this document is to describe how to update gRPC API files in +kuryr-kubernetes repository in case of upgrading to a new version of Kubernetes +PodResources API. These files are ``api_pb2_grpc.py``, ``api_pb2.py`` and +``api.proto`` from ``kuryr_kubernetes/pod_resources/`` directory. + +``api.proto`` is a gRPC API definition file generated from the +``kubernetes/pkg/kubelet/apis/podresources//api.proto`` of the +Kubernetes source tree. + +``api_pb2_grpc.py`` and ``api_pb2.py`` are python bindings for gRPC API. + +.. note:: + + There are only 2 reasons for update: + + #. Kubernetes released new version of PodResources API and the old one is no + longer supported. In this case, without update, we'll not be able to use + PodResources service. + + #. ``protobuf`` version in ``lower-constraints.txt`` changed to lower + version (this is highly unlikely). In this case ``protobuf`` could fail + to use our python bindings. + +Automated update +---------------- +``contrib/regenerate_pod_resources_api.sh`` script could be used to re-generate +PodResources gRPC API files. By default, this script will download ``v1alpha1`` +version of ``api.proto`` file from the Kubernetes GitHub repo and create +required kuryr-kubernetes files from it:: + + [kuryr-kubernetes]$ ./contrib/regenerate_pod_resources_api.sh + +Alternatively, path to ``api.proto`` file could be specified in +``KUBERNETES_API_PROTO`` environment variable:: + + $ export KUBERNETES_API_PROTO=/path/to/api.proto + +Define ``API_VERSION`` environment variable to use specific version of +``api.proto`` from the Kubernetes GitHub:: + + $ export API_VERSION=v1alpha1 + +Manual update steps +------------------- + +Preparing the new api.proto +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Copy the ``api.proto`` from K8s sources to ``kuryr_kubernetes/pod_resources/`` +and remove all the lines that contains ``gogoproto`` since this is unwanted +dependency that is not needed for python bindings:: + + $ sed '/gogoproto/d' \ + ../kubernetes/pkg/kubelet/apis/podresources//api.proto \ + > kuryr_kubernetes/pod_resources/api.proto + +Don't forget to update the file header that should point to the original +``api.proto`` and to this reference document:: + + // Generated from kubernetes/pkg/kubelet/apis/podresources//api.proto + // To regenerate api.proto, api_pb2.py and api_pb2_grpc.py follow instructions + // from doc/source/devref/updating_pod_resources_api.rst. + +Generating the python bindings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* (Optional) Create the python virtual environment:: + + [kuryr-kubernetes]$ python3 -m venv venv + [kuryr-kubernetes]$ . ./venv/bin/activate + +* To generate python bindings we need a ``protoc`` compiler and the + ``gRPC plugin`` for it. The most simple way to get them is to install + ``grpcio-tools``:: + + (venv) [kuryr-kubernetes]$ pip install grpcio-tools==1.19 + + .. note:: + + We're installing specific version of ``grpcio-tools`` to get specific + version of ``protoc`` compiler. The version of ``protoc`` compiler should + be equal to the ``protobuf`` package version in ``lower-constraints.txt``. + This is because older ``protobuf`` might be not able to use files + generated by newer compiler. In case you need to use more recent compiler, + you need update ``requirements.txt`` and ``lower-constraints.txt`` + accordingly. + + To check version of compiler installed with ``grpcio-tools`` use:: + + (venv) [kuryr-kubernetes]$ python -m grpc_tools.protoc --version + libprotoc 3.6.1 + +* Following command will generate ``api_pb2_grpc.py`` and ``api_pb2.py``:: + + (venv) [kuryr-kubernetes]$ python -m grpc_tools.protoc -I./ \ + --python_out=. --grpc_python_out=. \ + kuryr_kubernetes/pod_resources/api.proto + diff --git a/kuryr_kubernetes/pod_resources/api.proto b/kuryr_kubernetes/pod_resources/api.proto index d7ef9c08e..9612453c1 100644 --- a/kuryr_kubernetes/pod_resources/api.proto +++ b/kuryr_kubernetes/pod_resources/api.proto @@ -1,11 +1,6 @@ // Generated from kubernetes/pkg/kubelet/apis/podresources/v1alpha1/api.proto -// by removing the 'gogoproto' dependencies. -// To regenerate api_pb2.py and api_pb2_grpc.py use: -// python -m grpc_tools.protoc -I./ --python_out=. --grpc_python_out=. kuryr_kubernetes/pod_resources/api.proto -// -// Important: Version of 'protoc' compiler should be equal to 'protobuf' -// version specified in lower-constraints.txt. -// Check: 'python -m grpc_tools.protoc --version'. +// To regenerate api.proto, api_pb2.py and api_pb2_grpc.py follow instructions +// from doc/source/devref/updating_pod_resources_api.rst. syntax = 'proto3';