From a04f3c986be2b2b8df4090119a3cefcc95c9554b Mon Sep 17 00:00:00 2001 From: Marios Andreou Date: Wed, 8 May 2019 13:55:45 +0300 Subject: [PATCH] Add fresh-box-setup script for zuul-reproducer first run On first run or a fresh setup there are some required steps like creating a user, a clouds.yaml and ssh keys for gerrit. This adds a script into the create reproducer role which can be used for this initial setup Change-Id: I6d70a87006800447a12b4609ccc81400741290db --- .../defaults/main.yml | 1 + .../tasks/main.yml | 6 + ...E-reproducer-zuul-based-quickstart.html.j2 | 218 ++++++++++-------- .../reproducer-fresh-box-setup.sh.j2 | 127 ++++++++++ 4 files changed, 258 insertions(+), 94 deletions(-) create mode 100644 roles/create-zuul-based-reproducer/templates/reproducer-fresh-box-setup.sh.j2 diff --git a/roles/create-zuul-based-reproducer/defaults/main.yml b/roles/create-zuul-based-reproducer/defaults/main.yml index ec5d79002..c890eed23 100644 --- a/roles/create-zuul-based-reproducer/defaults/main.yml +++ b/roles/create-zuul-based-reproducer/defaults/main.yml @@ -1,5 +1,6 @@ --- reproducer_zuul_based_quickstart_script: reproducer-zuul-based-quickstart.sh.j2 +reproducer_fresh_box_setup_script: reproducer-fresh-box-setup.sh.j2 reproducer_zuul_based_quickstart_readme_file: "{{ artcl_collect_dir }}/README-reproducer.html" reproducer_zuul_based_launcher_playbook: launcher-playbook.yaml.j2 launcher_env_setup_playbook: launcher-env-setup-playbook.yaml.j2 diff --git a/roles/create-zuul-based-reproducer/tasks/main.yml b/roles/create-zuul-based-reproducer/tasks/main.yml index 965488819..9447053d8 100644 --- a/roles/create-zuul-based-reproducer/tasks/main.yml +++ b/roles/create-zuul-based-reproducer/tasks/main.yml @@ -76,6 +76,12 @@ dest: "{{ artcl_collect_dir }}/{{ zuul_reproducer_dir }}/reproducer-zuul-based-quickstart.sh" mode: 0755 +- name: Create the reproducer-fresh-box-setup script + template: + src: "{{ reproducer_fresh_box_setup_script }}" + dest: "{{ artcl_collect_dir }}/{{ zuul_reproducer_dir }}/reproducer-fresh-box-setup.sh" + mode: 0755 + - name: Copy the tripleo-quickstart install-deps script copy: src: "/home/{{ undercloud_user }}/src/opendev.org/openstack/tripleo-quickstart/{{ item }}" diff --git a/roles/create-zuul-based-reproducer/templates/README-reproducer-zuul-based-quickstart.html.j2 b/roles/create-zuul-based-reproducer/templates/README-reproducer-zuul-based-quickstart.html.j2 index 949ef4f81..55f378463 100644 --- a/roles/create-zuul-based-reproducer/templates/README-reproducer-zuul-based-quickstart.html.j2 +++ b/roles/create-zuul-based-reproducer/templates/README-reproducer-zuul-based-quickstart.html.j2 @@ -54,125 +54,152 @@ issue.

-Prior to running the Zuul-based reproducer, the following setup steps are needed - -( only need to be done once per local system): +To setup and run the reproducer you must first decide where it will run +- on a pet (your laptop) or on cattle (rdo-cloud vm or other +temporary environment)? For each case see notes below for required setup. + +

+If this is the first time you are running the reproducer you must +ensure you have an externally routable network in your RDO-cloud tenant called +'private'. That is you will need a router attached to the external network that +is also attached to your network called 'private' (which you may need to create). +Assuming you have a clouds.yaml that let's you talk to RDO cloud you can use +the following commands to create the required setup: +

+
+    openstack --os-cloud rdo-cloud network create private
+    openstack --os-cloud rdo-cloud subnet create private --network private --gateway 192.168.0.1 --subnet-range 192.168.0.0/24
+    openstack --os-cloud rdo-cloud router create privaterouter
+    openstack --os-cloud rdo-cloud router set --external-gateway 38.145.32.0/22  privaterouter
+    openstack --os-cloud rdo-cloud router add subnet privaterouter private
+
+
+

+ +Alternatively log in to RDO cloud horizon and create the network and router +that way. + +
+

Cattle

+If you're using a temporary environment like RDO-cloud vm to run the +reproducer you can use the script inside the reproducer-quickstart/ directory +from the logs of the job you are reproducing. The script is acalled +reproducer-fresh-box-setup.sh: + +

+    curl -o reproducer-fresh-box-setup.sh http://logs.openstack.org/79/657779/2/check/tripleo-ci-centos-7-containers-multinode/67d8eb6/logs/reproducer-quickstart/reproducer-fresh-box-setup.sh
+    chmod 775 reproducer-fresh-box-setup.sh
+    ./reproducer-fresh-box-setup.sh -u marios -p password -c reprozuul
+
+ +

note: If you're using an RDO-cloud vm be sure to use at least m1.large2 +flavour as we've hit hard disk limits with m1.large

+

There are two required parameters -u USER and -p PASS +which correspond to the RDO cloud username and password. These are written to +the generated $HOME/.config/openstack/clouds.yaml file expected by the +reproducer. +

+

This script creates the default user 'reprozuul' with passwordless sudo +privileges, creates $HOME/.config/openstack/clouds.yaml with the -u USER and +-p PASS parameters and creates the required ssh keypair. It must run as root, +or as a user with passwordless sudo. +

+

The script finally switches to the newly created user and outputs the newly +created public key. You must upload this key to gerrit (both +opendev.org as well +as rdoproject.org). +

+
+

Pet

+ +If you are running the reproducer on a non-transient machine, that is your +laptop or any other box which isn't temporary (a pet), then you need: + +You only need to do these things once on your pet and then rerun reproducer or +update as needed. + +
    -
  • Test your cloud.yaml by running: -
    -
- openstack --os-cloud rdo-cloud image list
-
-
    -
-
    -
  • Note that you will need a network named 'private' in the tenant connected to an -router that is connected to the external network.
    • -
  • Create an ssh key pair without a pass phrase:
    • +
    +
  • Create an ssh key pair without a pass phrase:
      • -
    • The ssh key pair is configurable - see variables ssh_path
      • -
    • user_pri_key. The default key is:~/.ssh/id_rsa[.pub]
      • -
    • Note: Reference FAQ below when creating ssh keys, some keys fail when used w/ gerrit +
    • The ssh key pair is configurable - see variables ssh_path + user_pri_key. The default key is:~/.ssh/id_rsa[.pub]
      • +
    • Note: Use the following to create keys: ssh-keygen -m PEM -t rsa + See FAQ below for more information about ssh keys requirements.
    -

    +
+

-

    -
  • wget or curl the reproducer-quickstart/reproducer-zuul-based-quickstart.tar file.
    • -
  • Note that the launcher-env-setup-playbook.yaml will set up -clone repos and set up 'roles', 'playbooks', and 'library' directories -from which the tasks will be executed.
    • -
  • Once the files are extracted from the archive, you can run the Bash script -with the various options. Run the script with --help option to -output a list of available run options.
    • -
-Note: the local $USERNAME idealy matches the openstack and rdo gerrit user id's -If not pay extra attention to the -ug and -rg options. -
    - -
    -Options:
-  -w, --workspace dir
-                      directory where the repos and directories
-                      are created. Defaults to creating a directory in /tmp
-  -l, --libvirt
-                      Runs a 2-node multinode job or singlenode job on a
-                      single virthost using libvirt to create the nodes.
-                      If a singlenode reproducer is run, two VMs will still be created.
-  -c, --cloud-name
-                      Host cloud, specified in the clouds.yaml, to target
-                      Defaults to rdo-cloud
-  -cp, --clouds-yaml-path
-                      Full path to the clouds.yaml file
-                      Defaults to /home/$USERNAME/.config/openstack/clouds.yaml
-  -ok, --ovb-key-name
-                      Name of the key to use in the host tenant for OVB deployments
-                      Defaults to tripleo-ci-team
-  -f, --force-post-failure
-                      Force job to fail so that the nodes will be held.
-                      Temporary solution
-  -ug, --upstream-gerrit-user
-                      Set the upstream gerrit user required to clone repos.
-                      Defaults to the current $USERNAME
-  -rg, --rdo-gerrit-user
-                      Set the upstream RDO user required to clone RDO-related repos.
-                      Defaults to the current $USERNAME
-  -k, --ssh-key
-                      private ssh key used to set up an access nodes.
-                      Defaults to id_rsa.
-  -kp, --ssh-key-public
-                      public ssh key used to set up an access nodes.
-                      Defaults to id_rsa.pub
-  -skp, --ssh-key-path
-                      Path to directory where user ssh keys are stored.
-                      Defaults to /home/$USERNAME/.ssh
-  -ugk, --upstream-gerrit-key
-                      Set the upstream gerrit private key.
-                      Defaults to the id_rsa.
-  -rgk, --rdo-gerrit-key
-                      Set the upstream RDO user key.
-                      Defaults to the id_rsa.
-  -e, --extra-params
-                      File or/and parameters used to override default
-                      parameters for playbooks. Multiple files
-                      can be passed [-e @file1.yml -e @file2.yml ...]
-                      and arguments [-e var=value -e var2=value2 ...]
-  -h, --help          print this help and exit
+wget or curl the reproducer-quickstart/reproducer-zuul-based-quickstart.tar file.
+
    
+    wget http://logs.openstack.org/79/657779/2/check/tripleo-ci-centos-7-standalone-upgrade/92fd476/logs/reproducer-quickstart/reproducer-zuul-based-quickstart.tar
+    tar -xvf reproducer-zuul-based-quickstart.tar
+
    
+One of the extracted files is the script called ./reproducer-zuul-based-quickstart.sh
+and you can run it with --help to see available options.
+
    
+
    
+Note: If the local $USERNAME does not match the gerrit user IDs for
+opendev.org or
+ rdoproject.org)
+then you will need to supply the -ug (upstream gerrit) and -rg (rdo gerrit)
+parameters:
 
-    
-
    +
    +
    
+    mkdir WORKSPACE
+    ./reproducer-zuul-based-quickstart.sh --workspace WORKSPACE -ug slim_shady -rg slim_shadier
+

    +Monitor the output and follow any instructions - for example the script will +exit and require re-run once the user is added to the docker group. +
    +
@@ -183,9 +210,12 @@ The playbooks can be run directly using ansible-playbook. The Bash script contains some package installation and Docker user/group setup that you will need to cover prior to running the playbooks directly. Please see the Bash scripts for details, especially if you are running the -reproducer for the first time on teh local environment. +reproducer for the first time on the local environment. The playbooks can be run as follows: diff --git a/roles/create-zuul-based-reproducer/templates/reproducer-fresh-box-setup.sh.j2 b/roles/create-zuul-based-reproducer/templates/reproducer-fresh-box-setup.sh.j2 new file mode 100644 index 000000000..424abbf1a --- /dev/null +++ b/roles/create-zuul-based-reproducer/templates/reproducer-fresh-box-setup.sh.j2 @@ -0,0 +1,127 @@ +#!/bin/bash + +REPRO_USER=${REPRO_USER:=reprozuul} +REPRO_DEPS="tar wget screen" +RDO_USER="" +RDO_PASS="" +CLOUDS_YAML_PATH=".config/openstack/" +usage () { + echo "Usage: $0 [options]" + echo "" + echo "Options:" + echo " -u, --rdo-user" + echo " Username for RDO cloud to be written to" + echo " $CLOUDS_YAML_PATH/clouds.yaml. REQUIRED." + echo " -p, --rdo-pass" + echo " Password for RDO cloud to be written to" + echo " $CLOUDS_YAML_PATH/clouds.yaml. REQUIRED." + echo " -c, --create-user" + echo " Local user to create for running reproducer." + echo " Defaults to $REPRO_USER" + echo " -h, --help print this help and exit" +} + +set -e + +# Input argument assignments +while [ "x$1" != "x" ]; do + + case "$1" in + --rdo-user|-u) + RDO_USER=$2 + shift + ;; + + --rdo-pass|-p) + RDO_PASS=$2 + shift + ;; + + --create-user|-c) + REPRO_USER=$2 + shift + ;; + + --help|-h) + usage + exit + ;; + + --) shift + break + ;; + + -*) echo "ERROR: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + + *) break + ;; + esac + + shift +done + +# check required params - rdo user/pass +for param in "$RDO_USER" "$RDO_PASS" ; do + if [[ $param = "" ]]; then + echo "error --rdo-user and --rdo-pass are required parameters" + exit 1 + fi +done + +# create user +sudo adduser $REPRO_USER +echo "$REPRO_USER ALL=(root) NOPASSWD:ALL" | sudo tee -a /etc/sudoers.d/$REPRO_USER + +# install packages +sudo yum -y install $REPRO_DEPS + +# write clouds.yaml +CLOUDS_YAML_PATH="/home/$REPRO_USER/.config/openstack/" +sudo mkdir -p $CLOUDS_YAML_PATH +sudo /bin/bash -c "cat <>$CLOUDS_YAML_PATH/clouds.yaml +clouds: + rdo-cloud: + identity_api_version: 3 + region_name: regionOne + auth: + auth_url: https://phx2.cloud.rdoproject.org:13000/v3 + password: $RDO_PASS + project_name: $RDO_USER + username: $RDO_USER + user_domain_name: Default + project_domain_name: Default + regions: + - name: regionOne + values: + networks: + - name: 38.145.32.0/22 + routes_externally: true + - name: private +EOF" + +sudo chown -R $REPRO_USER:$REPRO_USER /home/$REPRO_USER/.config + +# create keys +mkdir keystemp +pushd keystemp +ssh-keygen -m PEM -t rsa -N "" -f id_rsa +ssh_path=/home/$REPRO_USER/.ssh +sudo mkdir $ssh_path +sudo mv id_rsa* $ssh_path +sudo chmod 700 $ssh_path +sudo chmod 600 $ssh_path/id_rsa +sudo chmod 644 $ssh_path/id_rsa.pub +sudo chown -R $REPRO_USER:$REPRO_USER $ssh_path +popd +rm -rf keystemp + +echo "Fresh box setup complete - now go upload the public key in" +echo "$CLOUDS_YAML_PATH to Gerrit - both upstream and rdo. Then you are ready " +echo "to run the reproducer. Here is the public key you must upload:" +sudo cat /home/$REPRO_USER/.ssh/id_rsa.pub +echo "Switching to $REPRO_USER" +sudo su - $REPRO_USER +