Use OpenStack project layout

.gitignore

@@ -1,7 +1,68 @@
-playbook_*.yml
-*.egg-info
+# Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
 *.pyc
+build/
+dist/
+doc/build/
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sql
+*.sqlite
+
+# OS generated files #
+######################
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+.idea
 .tox
-.vagrant
+*.sublime*
+*.egg-info
+Icon?
+ehthumbs.db
+Thumbs.db
+.eggs
+
+# User driven backup files #
+############################
+*.bak
+*.swp
+
+# Generated by pbr while building docs
+######################################
 AUTHORS
 ChangeLog
+
+# Files created by releasenotes build
+releasenotes/build
+
+# Test temp files
+tests/plugins
+
+# Vagrant testing artifacts
+.vagrant
+logs
+playbook_*.yml

Vagrantfile

@@ -0,0 +1,12 @@
+Vagrant.configure(2) do |config|
+  config.vm.box = "ubuntu/trusty64"
+  config.vm.provider "virtualbox" do |v|
+    v.memory = 2048
+    v.cpus = 2
+  end
+  config.vm.provision "shell", inline: <<-SHELL
+    sudo su -
+    cd /vagrant
+    ./provision.sh
+  SHELL
+end

bindep.txt

@@ -0,0 +1,43 @@
+# This file facilitates OpenStack-CI package installation
+# before the execution of any tests.
+#
+# See the following for details:
+#  - http://docs.openstack.org/infra/bindep/
+#  - https://github.com/openstack-infra/bindep
+#
+# Even if the role does not make use of this facility, it
+# is better to have this file empty, otherwise OpenStack-CI
+# will fall back to installing its default packages which
+# will potentially be detrimental to the tests executed.
+
+# Base requirements for Ubuntu
+build-essential   [platform:dpkg]
+git-core          [platform:dpkg]
+libssl-dev        [platform:dpkg]
+libffi-dev        [platform:dpkg]
+python2.7         [platform:dpkg]
+python-dev        [platform:dpkg]
+lxc               [platform:dpkg]
+lxc-dev           [platform:dpkg]
+
+# Base requirements for CentOS
+gcc               [platform:rpm]
+gcc-c++           [platform:rpm]
+git               [platform:rpm]
+python-devel      [platform:rpm]
+libffi-devel      [platform:rpm]
+openssl-devel     [platform:rpm]
+
+# For SELinux
+libselinux-python [platform:rpm]
+
+# For SSL SNI support
+python-pyasn1               [platform:dpkg]
+python-openssl              [platform:dpkg]
+python-ndg-httpsclient      [platform:ubuntu !platform:ubuntu-trusty]
+python2-pyasn1              [platform:rpm]
+pyOpenSSL                   [platform:rpm]
+python-ndg_httpsclient      [platform:rpm]
+
+# Required for compressing collected log files in CI
+gzip

defaults/main.yml

@@ -1,6 +1,35 @@
-almanach_app_dir: /opt/almanach
+---
+# Copyright 2016, Internap Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+almanach_collector_program_name: almanach-collector
+almanach_api_program_name: almanach-api
+
+almanach_service_names:
+  - "almanach-api"
+  - "almanach-collector"
+
+almanach_system_user_name: almanach
+almanach_system_group_name: almanach
+almanach_system_shell: /bin/false
+almanach_system_comment: almanach system user
+
 almanach_log_dir: /var/log/almanach
 almanach_log_file: /var/log/almanach/almanach.log
+
+almanach_app_dir: /opt/almanach
+
 almanach_config_dir: /etc/almanach
 almanach_config_file: /etc/almanach/almanach.cfg
 almanach_logging_config_file: /etc/almanach/logging.cfg
@@ -29,3 +58,6 @@ almanach_keystone_username: my_service_username
 almanach_keystone_password: my_service_password
 almanach_keystone_tenant_name: my_service_tenant_name
 almanach_keystone_auth_url: http://keystone_url:5000/v2.0
+
+almanach_port: 8000
+almanach_listen_ip: 0.0.0.0

example/playbook.yml

@@ -0,0 +1,5 @@
+- name: Install almanach server
+  hosts: almanach_all
+  user: root
+  roles:
+    - { role: "os_almanach" }

examples/api.yml

@@ -1,7 +0,0 @@
-- hosts: api
-  gather_facts: False
-  become: True
-  vars:
-    almanach_mongodb_url: mongodb://almanach:almanach@10.0.0.5:27017/almanach
-  roles:
-    - api

examples/collector.yml

@@ -1,5 +0,0 @@
-- hosts: collector
-  gather_facts: False
-  become: True
-  roles:
-    - collector

examples/db.yml

@@ -1,5 +0,0 @@
-- hosts: database
-  gather_facts: False
-  become: True
-  roles:
-    - database

examples/multi_hosts.yml

@@ -1,21 +0,0 @@
-- hosts: database
-  gather_facts: False
-  become: True
-  roles:
-    - database
-
-- hosts: api
-  gather_facts: False
-  become: True
-  vars:
-    almanach_mongodb_url: mongodb://almanach:almanach@10.0.0.5:27017/almanach
-  roles:
-    - api
-
-- hosts: collector
-  gather_facts: False
-  become: True
-  vars:
-    almanach_mongodb_url: mongodb://almanach:almanach@10.0.0.5:27017/almanach
-  roles:
-    - collector

examples/single_host.yml

@@ -1,7 +0,0 @@
-- hosts: single_host
-  gather_facts: False
-  become: True
-  roles:
-    - database
-    - collector
-    - api

handlers/main.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2016, Internap, Inc.
+# Copyright 2016, Internap Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,14 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Restart almanach collector
+  service:
+    name: "{{ almanach_collector_program_name }}"
+    state: "restarted"
+    pattern: "{{ almanach_collector_program_name }}"
+
 - name: Restart almanach api
   service:
-    name: "almanach-api"
+    name: "{{ almanach_api_program_name }}"
     state: "restarted"
-    pattern: "almanach-api"
+    pattern: "{{ almanach_api_program_name }}"
 
-- name: Restart nginx
+- name: Restart almanach services
   service:
-    name: "nginx"
+    name: "{{ item }}"
     state: "restarted"
-    pattern: "nginx"
+    pattern: "{{ item }}"
+  with_items: "{{ almanach_service_names }}"
+  failed_when: false

meta/main.yml

@@ -14,8 +14,8 @@
 # limitations under the License.
 
 galaxy_info:
-  author: Squeaky Chairs
-  description: Almanach deployment roles
+  author: fguillot
+  description: Installation and setup of almanach
   company: Internap
   license: Apache2
   min_ansible_version: 1.9
@@ -24,7 +24,11 @@ galaxy_info:
     versions:
       - trusty
   galaxy_tags:
+    - cloud
+    - python
     - openstack
     - almanach
     - telemetry
-  dependencies: []
+  dependencies:
+    - pip_install
+    - openstack_openrc

provision.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euov
+
+# Install pip
+if [ ! "$(which pip)" ]; then
+  curl --silent --show-error --retry 5 \
+    https://bootstrap.pypa.io/get-pip.py | sudo python2.7
+fi
+
+# Install bindep and tox
+pip install bindep tox
+
+# CentOS 7 requires two additional packages:
+#   redhat-lsb-core - for bindep profile support
+#   epel-release    - required to install python-ndg_httpsclient/python2-pyasn1
+if [ "$(which yum)" ]; then
+    yum -y install redhat-lsb-core epel-release
+fi
+
+# Install OS packages using bindep
+if apt-get -v >/dev/null 2>&1 ; then
+    apt-get update
+    DEBIAN_FRONTEND=noninteractive \
+      apt-get -q --option "Dpkg::Options::=--force-confold" \
+      --assume-yes install `bindep -b -f bindep.txt test`
+else
+    yum install -y `bindep -b -f bindep.txt test`
+fi
+
+if [ ! -d /openstack/log ]; then
+    mkdir -p /openstack/log
+fi
+
+pip install lxc-python2
+
+tox -e functional

requirements.txt

@@ -1,2 +1 @@
-ansible
-pymongo
+ansible

roles/api/defaults/main.yml

@@ -1,9 +0,0 @@
-almanach_port: 8000
-almanach_listen_ip: 0.0.0.0
-almanach_user: almanach
-almanach_group: almanach
-almanach_service_name: almanach-api
-
-almanach_nginx_hostname: default
-almanach_nginx_port: 80
-almanach_nginx_log_file: /var/log/nginx/default_access.log

roles/collector/defaults/main.yml

@@ -1,3 +0,0 @@
-almanach_user: almanach
-almanach_group: almanach
-almanach_service_name: almanach-collector

roles/collector/handlers/main.yml

@@ -1,20 +0,0 @@
----
-# Copyright 2016, Internap, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Restart almanach collector
-  service:
-    name: "almanach-collector"
-    state: "restarted"
-    pattern: "almanach-collector"

roles/database/defaults/main.yml

@@ -1,3 +0,0 @@
-almanach_username: almanach
-almanach_password: almanach
-almanach_database: almanach

roles/database/handlers/main.yml

@@ -1,20 +0,0 @@
----
-# Copyright 2016, Internap, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Restart mongodb
-  service:
-    name: "mongodb"
-    state: "restarted"
-    pattern: "mongodb"

roles/database/tasks/main.yml

@@ -1,31 +0,0 @@
----
-# Copyright 2016, Internap Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Install MongoDB
-  apt: name={{ item }} update_cache=yes
-  with_items:
-    - mongodb-server
-    - python-pymongo
-
-- name: Listen everywhere
-  replace: dest=/etc/mongodb.conf regexp='bind_ip = 127.0.0.1' replace='bind_ip = 0.0.0.0' backup=yes
-
-- name: Enable remote authentication
-  replace: dest=/etc/mongodb.conf regexp='#auth = true' replace='auth = true' backup=yes
-  notify:
-    - Restart mongodb
-
-- name: Add Almanach user
-  mongodb_user: database={{ almanach_database }} name={{ almanach_username }} password={{ almanach_password }} state=present

tasks/almanach_init.yml

@@ -13,4 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- include: init_upstart.yml
+- include: almanach_init_api.yml
+  when: inventory_hostname in groups['almanach_api']
+
+- include: almanach_init_collector.yml
+  when: inventory_hostname in groups['almanach_collector']

tasks/almanach_init_api.yml

@@ -5,7 +5,7 @@
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,18 +15,25 @@
 
 - name: Place the init script
   template:
-    src: "almanach-collector-upstart.init.j2"
-    dest: "/etc/init/{{ almanach_service_name }}.conf"
+    src: "almanach-api-upstart-init.j2"
+    dest: "/etc/init/{{ almanach_api_program_name }}.conf"
     mode: "0644"
     owner: "root"
     group: "root"
   register: upstart_init
   notify:
-    - Restart almanach collector
+    - Restart almanach services
 
 - name: Reload init scripts
   shell: |
     initctl reload-configuration
   when: upstart_init | changed
   notify:
-    - Restart almanach collector
+    - Restart almanach services
+
+- name: Load service
+  service:
+    name: "{{ almanach_api_program_name }}"
+    enabled: "yes"
+  notify:
+    - Restart almanach services

tasks/almanach_init_collector.yml

@@ -5,7 +5,7 @@
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,18 +15,25 @@
 
 - name: Place the init script
   template:
-    src: "almanach-api-upstart.init.j2"
-    dest: "/etc/init/{{ almanach_service_name }}.conf"
+    src: "almanach-collector-upstart-init.j2"
+    dest: "/etc/init/{{ almanach_collector_program_name }}.conf"
     mode: "0644"
     owner: "root"
     group: "root"
   register: upstart_init
   notify:
-    - Restart almanach api
+    - Restart almanach services
 
 - name: Reload init scripts
   shell: |
     initctl reload-configuration
   when: upstart_init | changed
   notify:
-    - Restart almanach api
+    - Restart almanach services
+
+- name: Load service
+  service:
+    name: "{{ almanach_collector_program_name }}"
+    enabled: "yes"
+  notify:
+    - Restart almanach services

tasks/almanach_install.yml

@@ -0,0 +1,28 @@
+---
+# Copyright 2016, Internap Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Update APT index
+  apt: update_cache=yes
+  become: True
+
+- name: Install Python dependencies
+  apt: name={{ item }} state=present
+  with_items:
+    - python-dev
+    - python-pip
+    - python-virtualenv
+
+- name: Install Almanach
+  pip: name=almanach virtualenv={{ almanach_app_dir }} virtualenv_python=python2.7

tasks/almanach_post_install.yml

@@ -1,11 +1,11 @@
 ---
-# Copyright 2016, Internap Inc.
+# Copyright 2016, Internap, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,37 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Update APT index
-  apt: update_cache=yes
-  become: True
-
-- name: Install Python dependencies
-  apt: name={{ item }} state=present
-  with_items:
-    - python-dev
-    - python-pip
-    - python-virtualenv
-
-- name: Install Almanach
-  pip: name=almanach virtualenv=/opt/almanach virtualenv_python=python2.7
-
 - name: Create configuration folder
   file: path={{ almanach_config_dir }} state=directory
 
-- name: Create Almanach config file
+- name: Create almanach config file
   template: src=almanach.cfg.j2 dest={{ almanach_config_file }}
 
-- name: Create Almanach config logging file
+- name: Create almanach config logging file
   template: src=logging.cfg.j2 dest={{ almanach_logging_config_file }}
 
-- name: Create Almanach group
-  group: name={{ almanach_group }} state=present
-
-- name: Create Almanach user
-  user: name={{ almanach_user }} group={{ almanach_group }} createhome=no state=present
-
 - name: Create log directory
-  file: path={{ almanach_log_dir }} state=directory owner={{ almanach_user }} group={{ almanach_group }}
+  file: path={{ almanach_log_dir }} state=directory owner={{ almanach_system_user_name }} group={{ almanach_system_group_name }}
 
 - name: Configure logrotate
   template: src=logrotate.j2 dest={{ almanach_logrotate_config_file }} mode=0644

tasks/almanach_pre_install.yml

@@ -0,0 +1,31 @@
+---
+# Copyright 2016, Internap, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Create the system group
+  group:
+    name: "{{ almanach_system_group_name }}"
+    state: "present"
+    system: "yes"
+
+- name: Create the almanach system user
+  user:
+    name: "{{ almanach_system_user_name }}"
+    group: "{{ almanach_system_group_name }}"
+    comment: "{{ almanach_system_comment }}"
+    shell: "{{ almanach_system_shell }}"
+    system: "yes"
+
+- name: Create the almanach log dir
+  file: path={{ almanach_log_dir }} state=directory owner={{ almanach_system_user_name }} group={{ almanach_system_group_name }}

tasks/main.yml

@@ -0,0 +1,42 @@
+---
+# Copyright 2016, Internap Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Gather variables for each operating system
+  include_vars: "{{ item }}"
+  with_first_found:
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
+    - "{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
+    - "{{ ansible_distribution | lower }}.yml"
+    - "{{ ansible_os_family | lower }}.yml"
+  tags:
+    - always
+
+- include: almanach_pre_install.yml
+  tags:
+    - almanach-install
+
+- include: almanach_install.yml
+  tags:
+    - almanach-install
+
+- include: almanach_post_install.yml
+  tags:
+    - almanach-install
+    - almanach-config
+
+- include: almanach_init.yml
+  tags:
+    - almanach-install

templates/almanach-api-upstart-init.j2

@@ -1,13 +1,13 @@
 # {{ ansible_managed }}
 
-description "{{ almanach_service_name }}"
+description "{{ almanach_api_program_name }}"
 
 start on runlevel [2345]
 stop on runlevel [!2345]
 
 respawn
-setuid {{ almanach_user }}
-setgid {{ almanach_group }}
+setuid {{ almanach_system_user_name }}
+setgid {{ almanach_system_group_name }}
 
 script
     exec {{ almanach_app_dir }}/bin/almanach api {{ almanach_config_file }} --host {{ almanach_listen_ip }} --port {{ almanach_port }} --logging {{ almanach_logging_config_file }}

templates/almanach-collector-upstart-init.j2

@@ -1,13 +1,13 @@
 # {{ ansible_managed }}
 
-description "{{ almanach_service_name }}"
+description "{{ almanach_api_program_name }}"
 
 start on runlevel [2345]
 stop on runlevel [!2345]
 
 respawn
-setuid {{ almanach_user }}
-setgid {{ almanach_group }}
+setuid {{ almanach_system_user_name }}
+setgid {{ almanach_system_group_name }}
 
 script
     exec {{ almanach_app_dir }}/bin/almanach collector {{ almanach_config_file }} --logging {{ almanach_logging_config_file }}

test-requirements.txt

@@ -0,0 +1,5 @@
+flake8<2.6.0,>=2.5.4 # MIT
+sphinx!=1.3b1,<1.3,>=1.2.1 # BSD
+oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
+doc8 # Apache-2.0
+reno>=1.8.0 # Apache2

tests/ansible-role-requirements.yml

@@ -0,0 +1,44 @@
+- name: apt_package_pinning
+  src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning
+  scm: git
+  version: master
+- name: pip_install
+  src: https://git.openstack.org/openstack/openstack-ansible-pip_install
+  scm: git
+  version: master
+- name: memcached_server
+  src: https://git.openstack.org/openstack/openstack-ansible-memcached_server
+  scm: git
+  version: master
+- name: lxc_hosts
+  src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts
+  scm: git
+  version: master
+- name: lxc_container_create
+  src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create
+  scm: git
+  version: master
+- name: galera_client
+  src: https://git.openstack.org/openstack/openstack-ansible-galera_client
+  scm: git
+  version: master
+- name: galera_server
+  src: https://git.openstack.org/openstack/openstack-ansible-galera_server
+  scm: git
+  version: master
+- name: rabbitmq_server
+  src: https://git.openstack.org/openstack/openstack-ansible-rabbitmq_server
+  scm: git
+  version: master
+- name: os_keystone
+  src: https://git.openstack.org/openstack/openstack-ansible-os_keystone
+  scm: git
+  version: master
+- name: openstack_openrc
+  src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc
+  scm: git
+  version: master
+- name: os_tempest
+  src: https://git.openstack.org/openstack/openstack-ansible-os_tempest
+  scm: git
+  version: master

tests/group_vars/all_containers.yml

@@ -0,0 +1,27 @@
+---
+# Copyright 2016, Internap Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ansible_ssh_host: "{{ ansible_host }}"
+container_name: "{{ inventory_hostname }}"
+container_networks:
+  management_address:
+    address: "{{ ansible_host }}"
+    bridge: "lxcbr0"
+    interface: "eth1"
+    netmask: "255.255.252.0"
+    type: "veth"
+physical_host: localhost
+properties:
+  service_name: "{{ inventory_hostname }}"

tests/inventory

@@ -0,0 +1,27 @@
+[all]
+localhost ansible_connection=local ansible_become=True ansible_user=root ansible_ssh_user=root
+infra1 ansible_ssh_host=10.100.100.2 ansible_host=10.100.100.2 ansible_become=True ansible_user=root
+openstack1 ansible_ssh_host=10.100.100.3 ansible_host=10.100.100.3 ansible_become=True ansible_user=root
+
+[all_containers]
+infra1
+openstack1
+
+[rabbitmq_all]
+infra1
+
+[service_all:children]
+rabbitmq_all
+
+[keystone_all]
+infra1
+
+[almanach_api]
+openstack1
+
+[almanach_collector]
+openstack1
+
+[almanach_all:children]
+almanach_api
+almanach_collector

tests/test-functional-almanach.yml

@@ -0,0 +1,36 @@
+---
+# Copyright 2016, Internap Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Playbook for functional testing almanach
+  hosts: almanach_api
+  user: root
+  gather_facts: false
+  tasks:
+    - name: Install httplib2 so we can use the uri module
+      pip:
+        name: httplib2
+    - name: Check the almanach-api
+      uri:
+        url: "http://localhost:8000/info"
+        status_code: 200
+      register: result
+      until:
+        - result is defined
+        - result.status is defined
+        - result.status == 200
+      retries: 5
+      delay: 10
+  vars_files:
+    - test-vars.yml

tests/test-install-almanach.yml

@@ -13,5 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-dependencies:
-  - common
+- name: Playbook for deploying Almanach
+  hosts: almanach_all
+  user: root
+  gather_facts: true
+  roles:
+    - role: "{{ rolename | basename }}"
+  vars_files:
+    - test-vars.yml

tests/test-install-infra.yml

@@ -0,0 +1,69 @@
+---
+# Copyright 2016, Internap Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Playbook for deploying infra services
+  hosts: service_all
+  user: root
+  gather_facts: true
+  pre_tasks:
+    - name: Install MongoDB packages
+      apt:
+        name: "{{ item }}"
+        state: present
+      with_items: "{{ packages_mongodb }}"
+    - name: Configure the MongoDB bind address
+      lineinfile:
+        dest: /etc/mongodb.conf
+        regexp: "^(#)?bind_ip"
+        line: "bind_ip = 10.100.100.2"
+      register: mongodb_bind
+    - name: Enable the MongoDB smallfiles option
+      lineinfile:
+        dest: /etc/mongodb.conf
+        regexp: "^(#)?smallfiles"
+        line: "smallfiles = true"
+      register: mongodb_smallfiles
+    - name: Restart mongodb
+      service:
+        name: mongodb
+        state: restarted
+      when:
+        - mongodb_bind | changed or mongodb_smallfiles | changed
+      register: mongodb_restart
+    - name: Wait for mongodb to come back online after the restart
+      wait_for:
+        host: "10.100.100.2"
+        port: 27017
+        delay: 5
+        timeout: 30
+      when:
+        - mongodb_restart is defined
+        - mongodb_restart | changed
+    - name: Test mongodb connectivity
+      command: "mongo --host 10.100.100.2 --eval ' '"
+      changed_when: False
+    - name: Add almanach database user
+      mongodb_user:
+        login_host: "10.100.100.2"
+        database: almanach
+        name: almanach
+        password: secrete
+        roles: 'readWrite,dbAdmin'
+        state: present
+  vars:
+    packages_mongodb:
+      - mongodb-clients
+      - mongodb-server
+      - python-pymongo

tests/test-prepare-containers.yml

@@ -0,0 +1,33 @@
+---
+# Copyright 2016, Internap Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Playbook for creating containers
+  hosts: all_containers
+  connection: local
+  gather_facts: false
+  roles:
+    - role: "lxc_container_create"
+      lxc_container_release: trusty
+      lxc_container_backing_store: dir
+      global_environment_variables:
+        PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+  post_tasks:
+    - name: Wait for ssh to be available
+      local_action:
+        module: wait_for
+        port: "{{ ansible_ssh_port | default('22') }}"
+        host: "{{ ansible_ssh_host | default(inventory_hostname) }}"
+        search_regex: OpenSSH
+        delay: 1

tests/test-prepare-host.yml

@@ -0,0 +1,45 @@
+---
+# Copyright 2016, Internap Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Playbook for configuring LXC host
+  hosts: localhost
+  pre_tasks:
+    - name: Update apt cache
+      apt:
+        update_cache: yes
+      when: ansible_os_family == 'Debian'
+    - name: Ensure root's new public ssh key is in authorized_keys
+      authorized_key:
+        user: root
+        key: "{{ hostvars['localhost']['lxc_container_ssh_key'] }}"
+        manage_dir: no
+    - set_fact:
+        lxc_container_ssh_key: "{{ hostvars['localhost']['lxc_container_ssh_key'] }}"
+    - name: Check if this is an OpenStack-CI nodepool instance
+      stat:
+        path: /etc/nodepool/provider
+      register: nodepool
+    - name: Set the files to copy into the container cache for OpenStack-CI instances
+      set_fact:
+        lxc_container_cache_files:
+          - { src: '/etc/pip.conf', dest: '/etc/pip.conf' }
+      when: nodepool.stat.exists | bool
+  roles:
+    - role: "lxc_hosts"
+      lxc_net_address: 10.100.100.1
+      lxc_net_dhcp_range: 10.100.100.8,10.100.100.253
+      lxc_net_bridge: lxcbr0
+      lxc_kernel_options:
+        - { key: 'fs.inotify.max_user_instances', value: 1024 }

tests/test-prepare-keys.yml

@@ -0,0 +1,34 @@
+---
+# Copyright 2016, Internap Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: we use become_user because setting become: no or become: false
+#       doesn't seem to override the ansible_become=true in the
+#       inventory
+
+- name: Playbook for establishing ssh keys
+  hosts: localhost
+  become_user: "{{ ansible_ssh_user }}"
+  pre_tasks:
+    - name: Create ssh key pair for root
+      user:
+        name: "{{ ansible_ssh_user }}"
+        generate_ssh_key: "yes"
+        ssh_key_bits: 2048
+        ssh_key_file: ".ssh/id_rsa"
+    - name: Get the calling user's key
+      command: cat /root/.ssh/id_rsa.pub
+      register: key_get
+    - set_fact:
+        lxc_container_ssh_key: "{{ key_get.stdout }}"

tests/test-vars.yml

@@ -0,0 +1,24 @@
+---
+# Copyright 2016, Internap Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+almanach_mongodb_url: mongodb://almanach:secrete@10.100.100.2:27017/almanach
+almanach_mongodb_database: almanach
+
+internal_lb_vip_address: 10.100.100.2
+external_lb_vip_address: 10.100.100.2
+
+rabbitmq_servers: 10.100.100.2
+rabbitmq_use_ssl: true
+rabbitmq_port: 5671

tests/test.yml

@@ -13,4 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- include: init_upstart.yml
+- include: test-prepare-keys.yml
+
+- include: test-prepare-host.yml
+
+- include: test-prepare-containers.yml
+
+- include: test-install-infra.yml
+
+- include: test-install-almanach.yml
+
+- include: test-functional-almanach.yml

tox.ini

@@ -1,6 +1,131 @@
 [tox]
-envlist = py27
+minversion = 2.0
+skipsdist = True
+envlist = docs,linters,functional
+
 
 [testenv]
+usedevelop = True
+install_command =
+    pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 deps =
-    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+commands =
+    /usr/bin/find . -type f -name "*.pyc" -delete
+passenv =
+    HOME
+    http_proxy
+    HTTP_PROXY
+    https_proxy
+    HTTPS_PROXY
+    no_proxy
+    NO_PROXY
+whitelist_externals =
+    bash
+    git
+    rm
+    wget
+setenv =
+    VIRTUAL_ENV={envdir}
+    PYTHONUNBUFFERED=1
+
+
+# environment used by the -infra templated docs job
+[testenv:venv]
+commands =
+    {posargs}
+
+
+[testenv:pep8]
+commands =
+    # Run hacking/flake8 check for all python files
+    bash -c "grep --recursive --binary-files=without-match \
+        --files-with-match '^.!.*python$' \
+        --exclude-dir .eggs \
+        --exclude-dir .git \
+        --exclude-dir .tox \
+        --exclude-dir *.egg-info \
+        --exclude-dir doc \
+        {toxinidir} | xargs flake8 --verbose"
+
+
+[flake8]
+ignore=F403,H303
+
+
+[testenv:ansible]
+deps =
+    {[testenv]deps}
+    ansible==1.9.4
+    ansible-lint>=2.7.0,<3.0.0
+setenv =
+    {[testenv]setenv}
+    ANSIBLE_HOST_KEY_CHECKING = False
+    ANSIBLE_SSH_CONTROL_PATH = /tmp/%%h-%%r
+    ANSIBLE_ACTION_PLUGINS = {homedir}/.ansible/plugins/action
+    ANSIBLE_CALLBACK_PLUGINS = {homedir}/.ansible/plugins/callback
+    ANSIBLE_FILTER_PLUGINS = {homedir}/.ansible/plugins/filter
+    ANSIBLE_LOOKUP_PLUGINS = {homedir}/.ansible/plugins/lookup
+    ANSIBLE_LIBRARY = {homedir}/.ansible/plugins/library
+    ANSIBLE_ROLES_PATH = {homedir}/.ansible/roles:{toxinidir}/..
+commands =
+    rm -rf {homedir}/.ansible/plugins
+    git clone https://git.openstack.org/openstack/openstack-ansible-plugins \
+              {homedir}/.ansible/plugins
+    rm -rf {homedir}/.ansible/roles
+    ansible-galaxy install \
+                   --role-file={toxinidir}/tests/ansible-role-requirements.yml \
+                   --force
+
+
+[testenv:ansible-syntax]
+deps =
+    {[testenv:ansible]deps}
+setenv =
+    {[testenv:ansible]setenv}
+commands =
+    {[testenv:ansible]commands}
+    ansible-playbook -i {toxinidir}/tests/inventory \
+                     --syntax-check \
+                     --list-tasks \
+                     -e "rolename={toxinidir}" \
+                     {toxinidir}/tests/test.yml
+
+
+[testenv:ansible-lint]
+deps =
+    {[testenv:ansible]deps}
+commands =
+    {[testenv:ansible]commands}
+    ansible-lint {toxinidir}
+
+
+[testenv:functional]
+ignore_errors = True
+install_command =
+    pip install -U --force-reinstall {opts} {packages}
+deps =
+    {[testenv:ansible]deps}
+setenv =
+    {[testenv:ansible]setenv}
+commands =
+    {[testenv:ansible]commands}
+    ansible-playbook -i {toxinidir}/tests/inventory \
+                     -e "rolename={toxinidir}" \
+                     -e "install_test_packages=True" \
+                     {toxinidir}/tests/test.yml -vvvv
+    bash -c 'mkdir -p {toxinidir}/logs'
+    bash -c 'rsync --archive --verbose --ignore-errors /var/log/ /openstack/log/ {toxinidir}/logs/ || true'
+    bash -c 'find "{toxinidir}/logs/" -type f | sed "p;s|$|.txt|" | xargs -n2 mv'
+    bash -c 'command gzip --best --recursive "{toxinidir}/logs/"'
+
+
+[testenv:linters]
+deps =
+    {[testenv:ansible]deps}
+setenv =
+    {[testenv:ansible]setenv}
+commands =
+    {[testenv:pep8]commands}
+    {[testenv:ansible-lint]commands}
+    {[testenv:ansible-syntax]commands}

vars/ubuntu.yml

@@ -13,5 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-dependencies:
-  - common
+## APT Cache options
+cache_timeout: 600