From a6328eee0c82197643716fb85d13946a7e468bbc Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Thu, 20 Dec 2018 14:42:44 -0800 Subject: [PATCH] Add gitea k8s resource definitions and playbook This adds k8s resource definitions for running gitea, and an ansible playbook to create them. It also includes ansible playbooks to create orgs and repos. Change-Id: Ib64597512c6a85d7e1495d18ae42b242f9af5a67 --- kubernetes/gitea/README | 2 + kubernetes/gitea/app.ini.j2 | 74 +++++++++++++++++ kubernetes/gitea/gitea-playbook.yaml | 120 +++++++++++++++++++++++++++ kubernetes/gitea/k8s/deployment.yaml | 69 +++++++++++++++ kubernetes/gitea/k8s/namespace.yaml | 4 + kubernetes/gitea/k8s/secret.yaml | 12 +++ kubernetes/gitea/k8s/service.yaml | 18 ++++ kubernetes/gitea/setup-org.yaml | 56 +++++++++++++ kubernetes/gitea/setup-repo.yaml | 35 ++++++++ kubernetes/gitea/sync-repos.yaml | 36 ++++++++ 10 files changed, 426 insertions(+) create mode 100644 kubernetes/gitea/README create mode 100644 kubernetes/gitea/app.ini.j2 create mode 100644 kubernetes/gitea/gitea-playbook.yaml create mode 100644 kubernetes/gitea/k8s/deployment.yaml create mode 100644 kubernetes/gitea/k8s/namespace.yaml create mode 100644 kubernetes/gitea/k8s/secret.yaml create mode 100644 kubernetes/gitea/k8s/service.yaml create mode 100644 kubernetes/gitea/setup-org.yaml create mode 100644 kubernetes/gitea/setup-repo.yaml create mode 100644 kubernetes/gitea/sync-repos.yaml diff --git a/kubernetes/gitea/README b/kubernetes/gitea/README new file mode 100644 index 0000000000..552997c4d5 --- /dev/null +++ b/kubernetes/gitea/README @@ -0,0 +1,2 @@ +This directory contains Ansible playbooks and Kubernetes resource +definitions for running gitea. diff --git a/kubernetes/gitea/app.ini.j2 b/kubernetes/gitea/app.ini.j2 new file mode 100644 index 0000000000..c1731332da --- /dev/null +++ b/kubernetes/gitea/app.ini.j2 @@ -0,0 +1,74 @@ +[server] +APP_DATA_PATH = /data/gitea +SSH_DOMAIN = localhost +HTTP_PORT = 3000 +ROOT_URL = http://38.108.68.96/ +DISABLE_SSH = false +SSH_PORT = 22 +LFS_CONTENT_PATH = /data/git/lfs +DOMAIN = localhost +LFS_START_SERVER = true +LFS_JWT_SECRET = {{ lfs_jwt_secret }} +OFFLINE_MODE = false + +[database] +DB_TYPE = mysql +HOST = gitea-pxc.gitea-db.svc.cluster.local:3306 +NAME = gitea +USER = {{ db_username }} +PASSWD = {{ db_password }} +SSL_MODE = disable +LOG_SQL = false + +[indexer] +ISSUE_INDEXER_PATH = /data/gitea/indexers/issues.bleve +REPO_INDEXER_ENABLED = true + +[session] +PROVIDER_CONFIG = /data/gitea/sessions +PROVIDER = file + +[picture] +AVATAR_UPLOAD_PATH = /data/gitea/avatars +DISABLE_GRAVATAR = false +ENABLE_FEDERATED_AVATAR = true + +[attachment] +PATH = /data/gitea/attachments + +[log] +ROOT_PATH = /logs +LEVEL = Info + +[security] +INSTALL_LOCK = true +SECRET_KEY = {{ secret_key }} +INTERNAL_TOKEN = {{ internal_token }} + +[service] +DISABLE_REGISTRATION = false +REQUIRE_SIGNIN_VIEW = false +REGISTER_EMAIL_CONFIRM = false +ENABLE_NOTIFY_MAIL = false +ALLOW_ONLY_EXTERNAL_REGISTRATION = false +ENABLE_CAPTCHA = false +DEFAULT_KEEP_EMAIL_PRIVATE = false +DEFAULT_ALLOW_CREATE_ORGANIZATION = true +DEFAULT_ENABLE_TIMETRACKING = true +NO_REPLY_ADDRESS = noreply.example.org + +[mailer] +ENABLED = false + +[openid] +ENABLE_OPENID_SIGNIN = true +ENABLE_OPENID_SIGNUP = true + +[markup.pandoc] +ENABLED = true +; List of file extensions that should be rendered by an external command +FILE_EXTENSIONS = .rst +; External command to render all matching extensions +RENDER_COMMAND = "/usr/bin/pandoc" +; Input is not a standard input but a file +IS_INPUT_FILE = false diff --git a/kubernetes/gitea/gitea-playbook.yaml b/kubernetes/gitea/gitea-playbook.yaml new file mode 100644 index 0000000000..ac07dc7b0d --- /dev/null +++ b/kubernetes/gitea/gitea-playbook.yaml @@ -0,0 +1,120 @@ +- hosts: localhost + tasks: + # Deploy the service + - name: Set up gitea namespace + k8s: + state: present + definition: "{{ lookup('template', 'k8s/namespace.yaml') | from_yaml }}" + - name: Set up gitea secrets + k8s: + state: present + definition: "{{ lookup('template', 'k8s/secret.yaml') | from_yaml }}" + - name: Set up gitea configmap + k8s: + state: present + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: gitea-conf + namespace: gitea + data: + # Note: we are not asking ansible to template this, it + # will be run by jinja-init + app.ini.j2: "{{ lookup('file', 'app.ini.j2') }}" + - name: Set up gitea deployment + k8s: + state: present + definition: "{{ lookup('template', 'k8s/deployment.yaml') | from_yaml }}" + - name: Set up gitea service + k8s: + state: present + definition: "{{ lookup('template', 'k8s/service.yaml') | from_yaml }}" + + # Bootstrap + # TODO: wait until service is up + - name: Get service IP + k8s: + namespace: gitea + kind: Service + name: gitea-service + register: gitea_service + - name: Set service url fact + set_fact: + gitea_url: "http://{{ gitea_service.result.status.loadBalancer.ingress[0].ip }}" + - name: Check if root user exists + uri: + url: "{{ gitea_url }}/api/v1/users/root" + status_code: 200, 404 + register: root_user_check + - name: Create root user + when: root_user_check.status==404 + block: + - name: Find gitea pods + k8s_facts: + namespace: gitea + kind: Pod + label_selectors: + - "app = gitea" + register: gitea_pods + - name: Create root user + command: "kubectl exec {{ gitea_pods.resources[0].metadata.name }} -n gitea -c gitea -- gitea admin create-user --name root --password {{ gitea_root_password }} --email {{ gitea_root_email }} --admin" + no_log: true + - name: Check if gerrit user exists + uri: + url: "{{ gitea_url }}/api/v1/users/gerrit" + status_code: 200, 404 + register: gerrit_user_check + - name: Create gerrit user + when: gerrit_user_check.status==404 + no_log: true + uri: + url: "{{ gitea_url }}/api/v1/admin/users" + method: POST + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + status_code: 201 + body_format: json + body: + email: "gerrit@review.opendev.org" + full_name: Gerrit + login_name: gerrit + password: "{{ gitea_gerrit_password }}" + send_notify: false + source_id: 0 + username: gerrit + - name: Check if gerrit ssh key exists + uri: + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + url: "{{ gitea_url }}/api/v1/users/gerrit/keys" + status_code: 200 + register: gerrit_key_check + no_log: true + - name: Delete old gerrit ssh key + when: gerrit_key_check.json | length > 0 and gerrit_key_check.json[0].key != gitea_gerrit_public_key + no_log: true + uri: + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + url: "{{ gitea_url }}/api/v1/user/keys/{{ gerrit_key_check.json[0].id }}" + method: DELETE + status_code: 204 + - name: Add gerrit ssh key + when: gerrit_key_check.json | length == 0 + no_log: true + uri: + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + url: "{{ gitea_url }}/api/v1/admin/users/gerrit/keys" + method: POST + status_code: 201 + body_format: json + body: + key: "{{ gitea_gerrit_public_key }}" + read_only: false + title: "Gerrit replication key" diff --git a/kubernetes/gitea/k8s/deployment.yaml b/kubernetes/gitea/k8s/deployment.yaml new file mode 100644 index 0000000000..fd982191a6 --- /dev/null +++ b/kubernetes/gitea/k8s/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitea + namespace: gitea + labels: + app: gitea +spec: + replicas: 1 + selector: + matchLabels: + app: gitea + template: + metadata: + labels: + app: gitea + spec: + initContainers: + - name: gitea-config + image: opendevorg/gitea-init + env: + - {name: VERBOSE, value: '1'} + volumeMounts: + - {name: config-template, mountPath: /config_src} + - {name: gitea-conf, mountPath: /conf} + - {name: gitea-data, mountPath: /data} + - {name: secrets, mountPath: /secrets} + containers: + - name: gitea + image: opendevorg/gitea + ports: + - containerPort: 3000 + volumeMounts: + - name: gitea-data + mountPath: /data + - name: gitea-conf + mountPath: /custom/conf + - name: logs + mountPath: /logs + - name: openssh + image: opendevorg/gitea-openssh + ports: + - containerPort: 22 + volumeMounts: + - name: gitea-data + mountPath: /data + - name: gitea-conf + mountPath: /custom/conf + - name: logs + mountPath: /logs + volumes: + - name: gitea-data + flexVolume: + driver: ceph.rook.io/rook + fsType: ceph + options: + fsName: rookfs + clusterNamespace: rook-ceph + clusterName: rook-ceph + - name: config-template + configMap: + name: gitea-conf + - name: gitea-conf + emptyDir: + - name: logs + emptyDir: + - name: secrets + secret: + secretName: gitea-app diff --git a/kubernetes/gitea/k8s/namespace.yaml b/kubernetes/gitea/k8s/namespace.yaml new file mode 100644 index 0000000000..09a988f534 --- /dev/null +++ b/kubernetes/gitea/k8s/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gitea diff --git a/kubernetes/gitea/k8s/secret.yaml b/kubernetes/gitea/k8s/secret.yaml new file mode 100644 index 0000000000..39655f2138 --- /dev/null +++ b/kubernetes/gitea/k8s/secret.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gitea-app + namespace: gitea +type: Opaque +stringData: + secret_key: {{ gitea_secret_key }} + internal_token: {{ gitea_internal_token }} + lfs_jwt_secret: {{ gitea_lfs_jwt_secret }} + db_username: {{ gitea_db_username }} + db_password: {{ gitea_db_password }} diff --git a/kubernetes/gitea/k8s/service.yaml b/kubernetes/gitea/k8s/service.yaml new file mode 100644 index 0000000000..383671cb56 --- /dev/null +++ b/kubernetes/gitea/k8s/service.yaml @@ -0,0 +1,18 @@ +kind: Service +apiVersion: v1 +metadata: + name: gitea-service + namespace: gitea +spec: + selector: + app: gitea + ports: + - protocol: TCP + port: 80 + targetPort: 3000 + name: http + - protocol: TCP + port: 22 + targetPort: 22 + name: ssh + type: LoadBalancer diff --git a/kubernetes/gitea/setup-org.yaml b/kubernetes/gitea/setup-org.yaml new file mode 100644 index 0000000000..f2462ec63d --- /dev/null +++ b/kubernetes/gitea/setup-org.yaml @@ -0,0 +1,56 @@ +- name: Process org + debug: + msg: "Processing org {{ org }}" +- name: Create org + when: org not in gitea_orgs + uri: + url: "{{ gitea_url }}/api/v1/admin/users/root/orgs" + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + status_code: 201 + method: POST + body_format: json + body: + username: "{{ org }}" +- name: Get org team list + uri: + url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/teams" + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + status_code: 200 + register: gitea_org_team_list +- name: Get org owners + uri: + url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members" + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + status_code: 200 + register: gitea_org_members +- name: Add Gerrit user to org + when: "'gerrit' not in gitea_org_members.json | map(attribute='username')" + uri: + url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members/gerrit" + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + status_code: 204 + method: PUT +- name: Get org repo list + uri: + url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/repos" + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + status_code: 200 + register: gitea_org_repo_list +- name: Parse org repo list + set_fact: + gitea_org_repos: "{{ gitea_org_repo_list.json | map(attribute='name') | list }}" +- name: Create repos in org + loop: "{{ (gerrit_projects.keys() | map('regex_search', '^' + org + '/.*') | select | map('regex_replace', '^.*/', '') | list) }}" + loop_control: + loop_var: repo + include_tasks: 'setup-repo.yaml' diff --git a/kubernetes/gitea/setup-repo.yaml b/kubernetes/gitea/setup-repo.yaml new file mode 100644 index 0000000000..697ed35bc0 --- /dev/null +++ b/kubernetes/gitea/setup-repo.yaml @@ -0,0 +1,35 @@ +- name: debug + debug: + msg: "{{ gerrit_projects[org+'/'+repo] }}" +- name: Create repo + when: repo not in gitea_org_repos + uri: + url: "{{ gitea_url }}/api/v1/org/{{ org }}/repos" + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + status_code: 201 + method: POST + body_format: json + body: + auto_init: false + description: "{{ gerrit_projects[org+'/'+repo]['description'] | default('') }}" + name: "{{ repo }}" + private: false + register: create_repo_result +- name: Get created repo id + when: "create_repo_result.json is defined" + set_fact: + repo_id: "{{ create_repo_result.json['id'] }}" +- name: Prepare sql query + when: "repo_id is defined" + set_fact: + sql_statement: | + start transaction; + delete from repo_unit where repo_id = {{ repo_id }} and `type` in (2, 3, 5, 7); + insert into repo_unit (repo_id, `type`, config, created_unix) values ({{ repo_id }}, 7, "{""ExternalTrackerURL"":""https://storyboard.openstack.org/#!/project/{{ org }}/{{ repo }}"",""ExternalTrackerFormat"":""https://storyboard.openstack.org/#!/story/{index}"",""ExternalTrackerStyle"":""""}", unix_timestamp()); + commit; +- name: Adjust repo settings + when: "sql_statement is defined" + command: | + /home/corvus/opendev/kubectl exec gitea-pxc-0 -c database -n gitea-db -- mysql gitea -e '{{ sql_statement }}' diff --git a/kubernetes/gitea/sync-repos.yaml b/kubernetes/gitea/sync-repos.yaml new file mode 100644 index 0000000000..e2b2210b64 --- /dev/null +++ b/kubernetes/gitea/sync-repos.yaml @@ -0,0 +1,36 @@ +- hosts: localhost + vars: + gitea_url: http://38.108.68.66 + tasks: + - name: Get Gerrit project list + uri: + url: "https://review.openstack.org/projects/" + status_code: 200 + return_content: true + register: gerrit_project_list + - name: Parse Gerrit project list + set_fact: + gerrit_projects: "{{ gerrit_project_list.content[4:] | from_json }}" + - name: Parse Gerrit org list + set_fact: + gerrit_orgs: "{{ gerrit_projects.keys() | map('regex_search', '^(.*?)/') | list | unique | select | map('regex_replace', '/', '') | list }}" + - name: debug + debug: + msg: "{{ gerrit_orgs }}" + - name: Get Gitea org list + # We assume that all the orgs we are interested in are owned by root + uri: + url: "{{ gitea_url }}/api/v1/user/orgs" + user: root + password: "{{ gitea_root_password }}" + force_basic_auth: true + status_code: 200 + register: gitea_org_list + - name: Parse Gitea org list + set_fact: + gitea_orgs: "{{ gitea_org_list.json | map(attribute='username') | list }}" + - name: Create orgs + loop: "{{ gerrit_orgs }}" + loop_control: + loop_var: org + include_tasks: 'setup-org.yaml'