diff --git a/handlers/main.yml b/handlers/main.yml
index 4473cd0d..44f14ac5 100644
--- a/handlers/main.yml
+++ b/handlers/main.yml
@@ -55,6 +55,16 @@
     - "venv changed"
     - "Restart uWSGI"
 
+# NOTE (noonedeadpunk): Remove this task after Xena release
+- name: Remove obsoleted policy.json file
+  file:
+    path: "/etc/keystone/policy.json"
+    state: absent
+  listen:
+    - "venv changed"
+    - "Restart web server"
+    - "Restart uWSGI"
+
 - name: Start uWSGI
   service:
     name: "{{ item }}"
diff --git a/tasks/keystone_post_install.yml b/tasks/keystone_post_install.yml
index 772029c5..192355fd 100644
--- a/tasks/keystone_post_install.yml
+++ b/tasks/keystone_post_install.yml
@@ -49,12 +49,27 @@
     - Restart uWSGI
     - Restart web server
 
-- name: Implement policy.json if there are overrides configured
-  copy:
-    content: "{{ keystone_policy_overrides | to_nice_json }}"
-    dest: "/etc/keystone/policy.json"
+- name: Implement policy.yaml if there are overrides configured
+  config_template:
+    content: "{{ keystone_policy_overrides }}"
+    dest: "/etc/keystone/policy.yaml"
+    owner: "root"
+    group: "{{ keystone_system_group_name }}"
+    mode: "0640"
+    config_type: yaml
   when:
-    - keystone_policy_overrides != {}
+    - keystone_policy_overrides | length > 0
+  tags:
+    - keystone-policy-override
+
+- name: Remove legacy policy.yaml file
+  file:
+    path: "/etc/keystone/policy.yaml"
+    state: absent
+  when:
+    - keystone_policy_overrides | length == 0
+  tags:
+    - keystone-policy-override
 
 # NOTE(cloudnull): This is using "cp" instead of copy with a remote_source
 #                  because we only want to copy the original files once. and we