diff --git a/.zuul.yaml b/.zuul.yaml
index da555fac9..e1994c7b3 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -11,6 +11,7 @@
     check:
       jobs:
         - watcher-tempest-functional
+        - watcher-grenade
         - watcher-tempest-dummy_optim
         - watcher-tempest-actuator
         - watcher-tempest-basic_optim
@@ -164,6 +165,30 @@
       zuul_copy_output:
         /etc/hosts: logs
 
+- job:
+    name: watcher-grenade
+    parent: legacy-dsvm-base
+    timeout: 10800
+    run: playbooks/legacy/grenade-devstack-watcher/run.yaml
+    post-run: playbooks/legacy/grenade-devstack-watcher/post.yaml
+    irrelevant-files:
+      - ^(test-|)requirements.txt$
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^watcher/hacking/.*$
+      - ^watcher/tests/.*$
+      - ^releasenotes/.*$
+      - ^setup.cfg$
+      - ^tools/.*$
+      - ^tox.ini$
+    required-projects:
+      - openstack-dev/grenade
+      - openstack-infra/devstack-gate
+      - openstack/watcher
+      - openstack/python-watcherclient
+      - openstack/watcher-tempest-plugin
+
 - job:
     # This job is used in python-watcherclient repo
     name: watcherclient-tempest-functional
diff --git a/devstack/lib/watcher b/devstack/lib/watcher
index 3bbd83a28..6ce7e6c3b 100644
--- a/devstack/lib/watcher
+++ b/devstack/lib/watcher
@@ -289,6 +289,7 @@ function start_watcher_api {
         service_protocol="http"
     fi
     if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
+        enable_apache_site watcher-api
         restart_apache_server
     else
         run_process watcher-api "$WATCHER_BIN_DIR/watcher-api --config-file $WATCHER_CONF"
@@ -318,7 +319,7 @@ function start_watcher {
 function stop_watcher {
     if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
         disable_apache_site watcher-api
-	restart_apache_server
+        restart_apache_server
     else
         stop_process watcher-api
     fi
diff --git a/devstack/upgrade/from_rocky/upgrade-watcher b/devstack/upgrade/from_rocky/upgrade-watcher
new file mode 100644
index 000000000..9757c3c68
--- /dev/null
+++ b/devstack/upgrade/from_rocky/upgrade-watcher
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# ``upgrade-watcher``
+
+function configure_watcher_upgrade {
+    XTRACE=$(set +o | grep xtrace)
+    set -o xtrace
+
+    # Copy release-specific files
+    sudo cp $TARGET_RELEASE_DIR/watcher/etc/watcher/watcher.conf $WATCHER_CONF_DIR/watcher.conf
+    sudo cp $TARGET_RELEASE_DIR/watcher/etc/watcher/policy.yaml.sample $WATCHER_CONF_DIR/policy.yaml.sample
+
+    # reset to previous state
+    $XTRACE
+}
diff --git a/devstack/upgrade/resources.sh b/devstack/upgrade/resources.sh
new file mode 100755
index 000000000..4da8a4470
--- /dev/null
+++ b/devstack/upgrade/resources.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+
+set -o errexit
+
+source $GRENADE_DIR/grenaderc
+source $GRENADE_DIR/functions
+
+source $TOP_DIR/openrc admin demo
+
+set -o xtrace
+
+function _wait_for_status {
+     while :
+     do
+         state=$("${@:2}" -f value -c State)
+         [[ $state == "SUCCEEDED" ]] && break
+         if [ $state == "ERROR" ]; then
+             die $LINENO "ERROR creating audit"
+         fi
+         sleep 10
+     done
+ }
+
+function create_audit_template {
+    at_id=$(openstack optimize audittemplate create d1 dummy -s dummy -f value -c UUID)
+    resource_save watcher at_id $at_id
+ }
+
+function create_audit {
+    audit_id=$(openstack optimize audit create -s dummy -g dummy -f value -c UUID)
+    resource_save watcher audit_id $audit_id
+}
+
+function create_audit_with_autotrigger {
+    audit_at_id=$(openstack optimize audit create -s dummy -g dummy -f value -c UUID --auto-trigger)
+    resource_save watcher audit_at_id $audit_at_id
+}
+
+function verify_audit_template {
+    local at_id=$(resource_get watcher at_id)
+    openstack optimize audittemplate show $at_id
+}
+
+function verify_audit_with_autotrigger {
+    local audit_at_id=$(resource_get watcher audit_at_id)
+    _wait_for_status "SUCCEEDED" openstack optimize audit show $audit_at_id
+    local actionplan_at_id=$(openstack optimize actionplan list --audit $audit_at_id -c UUID -f value)
+    resource_save watcher actionplan_at $actionplan_at_id
+    actionplan_at_state=$(openstack optimize actionplan show $actionplan_at_id -c State -f value)
+    if [ $actionplan_at_state != "SUCCEEDED" ]; then
+        die $LINENO "ERROR executing actionplan"
+    fi
+}
+
+function verify_audit {
+    local audit_id=$(resource_get watcher audit_id)
+    _wait_for_status "SUCCEEDED" openstack optimize audit show $audit_id
+    local actionplan_id=$(openstack optimize actionplan list --audit $audit_id -c UUID -f value)
+    resource_save watcher actionplan $actionplan_id
+    actionplan_state=$(openstack optimize actionplan show $actionplan_id -c State -f value)
+    if [ $actionplan_state != "RECOMMENDED" ]; then
+        die $LINENO "ERROR creating actionplan"
+    fi
+}
+
+function verify_noapi {
+    # currently no good way
+    :
+}
+
+function delete_audit {
+  local audit_id=$(resource_get watcher audit_id)
+  local actionplan_id=$(resource_get watcher actionplan)
+  watcher actionplan delete $actionplan_id
+  openstack optimize audit delete $audit_id
+}
+
+function delete_audit_with_autotrigger {
+  local audit_at_id=$(resource_get watcher audit_at_id)
+  local actionplan_id=$(resource_get watcher actionplan_at)
+  watcher actionplan delete $actionplan_id
+  openstack optimize audit delete $audit_at_id
+}
+
+function delete_audit_template {
+  local at_id=$(resource_get watcher at_id)
+  openstack optimize audittemplate delete $at_id
+}
+
+function create {
+  create_audit_template
+  create_audit
+  create_audit_with_autotrigger
+}
+
+function verify {
+  verify_audit_template
+  verify_audit
+  verify_audit_with_autotrigger
+}
+
+function destroy {
+  delete_audit_template
+  delete_audit
+  delete_audit_with_autotrigger
+}
+
+# Dispatcher
+case $1 in
+    "create")
+        create
+        ;;
+    "verify_noapi")
+        verify_noapi
+        ;;
+    "verify")
+        verify
+        ;;
+    "destroy")
+        destroy
+        ;;
+    "force_destroy")
+        set +o errexit
+        destroy
+        ;;
+esac
diff --git a/devstack/upgrade/settings b/devstack/upgrade/settings
new file mode 100644
index 000000000..300ff09a0
--- /dev/null
+++ b/devstack/upgrade/settings
@@ -0,0 +1,11 @@
+register_project_for_upgrade watcher
+register_db_to_save watcher
+
+devstack_localrc base enable_plugin watcher https://git.openstack.org/openstack/watcher stable/rocky
+devstack_localrc target enable_plugin watcher https://git.openstack.org/openstack/watcher
+
+devstack_localrc base enable_service watcher-api watcher-decision-engine watcher-applier
+devstack_localrc target enable_service watcher-api watcher-decision-engine watcher-applier
+
+BASE_RUN_SMOKE=False
+TARGET_RUN_SMOKE=False
diff --git a/devstack/upgrade/shutdown.sh b/devstack/upgrade/shutdown.sh
new file mode 100755
index 000000000..baf350dc6
--- /dev/null
+++ b/devstack/upgrade/shutdown.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -o errexit
+
+source $GRENADE_DIR/grenaderc
+source $GRENADE_DIR/functions
+
+# We need base DevStack functions for this
+source $BASE_DEVSTACK_DIR/functions
+source $BASE_DEVSTACK_DIR/stackrc # needed for status directory
+source $BASE_DEVSTACK_DIR/lib/tls
+source $BASE_DEVSTACK_DIR/lib/apache
+
+WATCHER_DEVSTACK_DIR=$(dirname $(dirname $0))
+source $WATCHER_DEVSTACK_DIR/settings
+source $WATCHER_DEVSTACK_DIR/plugin.sh
+source $WATCHER_DEVSTACK_DIR/lib/watcher
+
+set -o xtrace
+
+stop_watcher
+
+# sanity check that service is actually down
+ensure_services_stopped watcher-api watcher-decision-engine watcher-applier
diff --git a/devstack/upgrade/upgrade.sh b/devstack/upgrade/upgrade.sh
new file mode 100755
index 000000000..f4802cf98
--- /dev/null
+++ b/devstack/upgrade/upgrade.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# ``upgrade-watcher``
+
+echo "*********************************************************************"
+echo "Begin $0"
+echo "*********************************************************************"
+
+# Clean up any resources that may be in use
+cleanup() {
+    set +o errexit
+
+    echo "********************************************************************"
+    echo "ERROR: Abort $0"
+    echo "********************************************************************"
+
+    # Kill ourselves to signal any calling process
+    trap 2; kill -2 $$
+}
+
+trap cleanup SIGHUP SIGINT SIGTERM
+
+# Keep track of the grenade directory
+RUN_DIR=$(cd $(dirname "$0") && pwd)
+
+# Source params
+source $GRENADE_DIR/grenaderc
+
+# Import common functions
+source $GRENADE_DIR/functions
+
+# This script exits on an error so that errors don't compound and you see
+# only the first error that occurred.
+set -o errexit
+
+# Upgrade watcher
+# ============
+
+# Get functions from current DevStack
+source $TARGET_DEVSTACK_DIR/stackrc
+source $TARGET_DEVSTACK_DIR/lib/apache
+source $TARGET_DEVSTACK_DIR/lib/tls
+source $(dirname $(dirname $BASH_SOURCE))/settings
+source $(dirname $(dirname $BASH_SOURCE))/plugin.sh
+
+# Print the commands being run so that we can see the command that triggers
+# an error.  It is also useful for following allowing as the install occurs.
+set -o xtrace
+
+# Save current config files for posterity
+[[ -d $SAVE_DIR/etc.watcher ]] || cp -pr $WATCHER_CONF_DIR $SAVE_DIR/etc.watcher
+
+# Install the target watcher
+install_watcher
+
+# calls upgrade-watcher for specific release
+upgrade_project watcher $RUN_DIR $BASE_DEVSTACK_BRANCH $TARGET_DEVSTACK_BRANCH
+
+# Migrate the database
+watcher-db-manage upgrade || die $LINO "DB migration error"
+
+start_watcher
+
+# Don't succeed unless the services come up
+ensure_services_started watcher-api watcher-decision-engine watcher-applier
+
+set +o xtrace
+echo "*********************************************************************"
+echo "SUCCESS: End $0"
+echo "*********************************************************************"
diff --git a/playbooks/legacy/grenade-devstack-watcher/post.yaml b/playbooks/legacy/grenade-devstack-watcher/post.yaml
new file mode 100644
index 000000000..e07f5510a
--- /dev/null
+++ b/playbooks/legacy/grenade-devstack-watcher/post.yaml
@@ -0,0 +1,15 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
diff --git a/playbooks/legacy/grenade-devstack-watcher/run.yaml b/playbooks/legacy/grenade-devstack-watcher/run.yaml
new file mode 100644
index 000000000..a82de932e
--- /dev/null
+++ b/playbooks/legacy/grenade-devstack-watcher/run.yaml
@@ -0,0 +1,60 @@
+- hosts: all
+  name: legacy-grenade-dsvm-watcher
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PYTHONUNBUFFERED=true
+
+          export PROJECTS="openstack-dev/grenade $PROJECTS"
+          export PROJECTS="openstack/watcher $PROJECTS"
+          export PROJECTS="openstack/watcher-tempest-plugin $PROJECTS"
+          export PROJECTS="openstack/python-watcherclient $PROJECTS"
+          export DEVSTACK_PROJECT_FROM_GIT="python-watcherclient $DEVSTACK_PROJECT_FROM_GIT"
+
+          export GRENADE_PLUGINRC="enable_grenade_plugin watcher https://git.openstack.org/openstack/watcher"
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"export TEMPEST_PLUGINS='/opt/stack/new/watcher-tempest-plugin'"
+
+          export DEVSTACK_GATE_TEMPEST_NOTESTS=1
+          export DEVSTACK_GATE_GRENADE=pullup
+
+          export BRANCH_OVERRIDE=default
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+          # Add configuration values for enabling security features in local.conf
+          function pre_test_hook {
+              if [ -f /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh ] ; then
+                  . /opt/stack/old/watcher-tempest-plugin/tools/pre_test_hook.sh
+              fi
+          }
+          export -f pre_test_hook
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'