Moves the artifice web API into the artifice package. Updates the build

system to create a working .deb, based on the makefile.
Adds a new script to start up the web daemon.
Adds a new script to test if the database is provisioned
Adds a new script used by Puppet to provision the database
Adds puppet manifests (mirrored in main puppet)
Moves api/ to artifice/api
Alters some of the relative imports
Moves artifice.py to why_is_this_called_artifice.py, as it was causing
import issues.

Change-Id: Id8a909f7ffcc64a5c4e3281c6b5ba83cef73b596
This commit is contained in:
Aurynn Shaw 2014-03-12 16:27:45 +13:00
parent 639ae8af66
commit bf671c0210
33 changed files with 574 additions and 47 deletions

30
.gitignore vendored
View File

@ -5,3 +5,33 @@ work
*.swp
*~
*.swo
*.sw*
lib/
man/
build/
work/
env/
dist/
include/
*.egg*/
bin/activate
bin/activate.csh
bin/activate.fish
bin/activate_this.py
bin/ceilometer
bin/easy_install
bin/easy_install-2.7
bin/keystone
bin/netaddr
bin/nosetests
bin/nosetests-2.7
bin/pip
bin/pip-2.7
bin/pybabel
bin/python
bin/python2
bin/python2.7
bin/waitress-serve
local/
test_vm/
env/

View File

@ -5,7 +5,9 @@ INSTALL_PATH=/opt/stack/artifice
BILLING_PROGRAM=bill.py
BINARY_PATH=/usr/local/bin
CONF_DIR=./work/${INSTALL_PATH}/etc/artifice
WORK_DIR=./work
CONF_DIR=${WORK_DIR}/${INSTALL_PATH}/etc/artifice
clean:
@rm -rf ./work
@ -18,25 +20,35 @@ init:
deb: clean init
@cp -r ./bin ./artifice ./scripts ./README.md ./INVOICES.md \
requirements.txt setup.py ./work/${INSTALL_PATH}
@cp -r ./artifice \
./scripts \
./README.md \
./INVOICES.md \
requirements.txt \
setup.py \
${WORK_DIR}${INSTALL_PATH}
@mkdir ${WORK_DIR}${INSTALL_PATH}/bin
@cp ./bin/collect ./bin/collect.py \
./bin/usage ./bin/usage.py \
./bin/web ./bin/web.py \
${WORK_DIR}${INSTALL_PATH}/bin
@chmod 0755 ${WORK_DIR}${INSTALL_PATH}/bin/web
@cp -r ./packaging/fs/* ${WORK_DIR}/
@mkdir -p ${CONF_DIR}
@cp ./examples/conf.yaml ${CONF_DIR}
@cp ./examples/csv_rates.yaml ${CONF_DIR}
@mkdir -p ${WORK_DIR}/etc/init.d
@mkdir -p ${WORK_DIR}/etc/artifice
@chmod +x ${WORK_DIR}/etc/init.d/artifice
@cp ./examples/conf.yaml ${WORK_DIR}/etc/artifice/conf.yaml
@cp ./examples/csv_rates.yaml ${WORK_DIR}/etc/artifice/csv_rates.yaml
@fpm -s dir -t deb -n ${NAME} -v ${VERSION} \
--pre-install=packaging/scripts/pre_install.sh \
--post-install=packaging/scripts/post_install.sh \
--depends 'postgresql >= 9.3' \
--depends 'postgresql-contrib >= 9.3' \
--depends 'libpq-dev' \
--deb-pre-depends pwgen \
--deb-pre-depends "libmysql++-dev" \
--deb-pre-depends python2.7 \
--deb-pre-depends python-pip \
--deb-pre-depends python-dev \
--deb-pre-depends python-virtualenv \
--template-scripts \
--template-value pg_database=artifice \
--template-value pg_user=artifice \
--template-value pg_port=5432 \
--template-value install_path=${INSTALL_PATH} \
-C ./work \
-C ${WORK_DIR} \
.

View File

@ -48,6 +48,7 @@ def get_app(conf):
global invoicer
module, kls = config["main"]["export_provider"].split(":")
# TODO: Try/except block
invoicer = getattr(importlib.import_module(module), kls)
if config["main"].get("timezone"):

31
artifice/initdb.py Normal file
View File

@ -0,0 +1,31 @@
from models import Base, __VERSION__
from sqlalchemy import create_engine
from sqlalchemy.pool import NullPool
def provision(engine):
Base.metadata.create_all(bind=engine)
if __name__ == '__main__':
import argparse
a = argparse.ArgumentParser()
a.add_argument("--host", "--host")
a.add_argument("-p", "--port")
a.add_argument("-u", "--user")
a.add_argument("-d", "--database")
a.add_argument("-P", "--provider")
a.add_argument("-w", "--password")
args = a.parse_args()
conn_string = "{provider}://{user}:{password}@{host}/{database}".format(
host=args.host,
port=args.port,
provider=args.provider,
user=args.user,
password=args.password,
database=args.database)
engine = create_engine(conn_string, poolclass=NullPool)
provision(engine)

View File

@ -2,7 +2,7 @@ import requests
import json
import auth
from ceilometerclient.v2.client import Client as ceilometer
from .models import resources
from artifice.models import resources
from constants import date_format

View File

@ -1,5 +1,5 @@
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Text, DateTime, DECIMAL, ForeignKey, String
from sqlalchemy import Column, Text, DateTime, Numeric, ForeignKey, String
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method
from sqlalchemy import event, DDL
@ -7,10 +7,21 @@ from sqlalchemy import event, DDL
from sqlalchemy.orm import relationship
from sqlalchemy.schema import ForeignKeyConstraint
# Version digit.
__VERSION__ = 1.0
Base = declarative_base()
class _Version(Base):
"""
A model that knows what version we are, stored in the DB.
"""
__tablename__ = "artifice_database_version"
id = Column(String(10), primary_key=True)
class Resource(Base):
"""Database model for storing metadata associated with a resource."""
__tablename__ = 'resources'
@ -29,7 +40,7 @@ class UsageEntry(Base):
# Service is things like incoming vs. outgoing, as well as instance
# flavour
service = Column(String(100), primary_key=True)
volume = Column(DECIMAL, nullable=False)
volume = Column(Numeric(precision=20, scale=2), nullable=False)
resource_id = Column(String(100), primary_key=True)
tenant_id = Column(String(100), primary_key=True)
start = Column(DateTime, nullable=False)
@ -266,3 +277,14 @@ event.listen(
DDL("DROP FUNCTION %s_exclusion_constraint_trigger()" %
SalesOrder.__tablename__).execute_if(dialect="postgresql")
)
def insert_into_version(target, connection, **kw):
connection.execute("INSERT INTO %s (id) VALUES (%s)" %
(target.name, __VERSION__))
event.listen(
_Version.__table__,
"after_create",
insert_into_version
)

View File

@ -1,5 +1,5 @@
import yaml
from models import Session
from artifice.models import Session
from interface import Artifice
default_config = "/etc/artifice/config.yaml"
@ -23,4 +23,4 @@ def connect(config=None):
# session.configure(bind=engine)
artifice = Artifice(config)
# artifice.artifice = session
return artifice
return artifice

View File

@ -1 +0,0 @@
bill.py

13
bin/web Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
INSTALLED="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
ORIGIN=`pwd`
# Bring up our python environment
# Pass through all our command line opts as expected
# Move ourselves to the code directory
# TODO: Fix this by removing relative imports from Artifice
cd $INSTALLED
cd ../
$INSTALLED/../env/bin/python $INSTALLED/web.py $@

25
bin/web.py Normal file
View File

@ -0,0 +1,25 @@
#!/usr/bin/python
from artifice.api import web
import yaml
import sys
import argparse
a = argparse.ArgumentParser("Web service for Artifice")
a.add_argument("-c", "--config", dest="config", help="Path to config file", default="/etc/artifice/conf.yaml")
a.add_argument("-i", "--interface", dest="ip", help="IP address to serve on.", default="0.0.0.0")
a.add_argument("-p", "--port", help="port to serve on", default="8000")
args = a.parse_args()
conf = None
try:
conf = yaml.load(args.config)
except IOError as e:
print "Couldn't load config file: %s" % e
sys.exit(1)
app = web.get_app(conf)
app.run(host=args.ip, port=args.port)

View File

@ -1,12 +1,6 @@
---
ceilometer:
host: http://localhost:8777/
database:
database: artifice
host: localhost
password_path: /etc/artifice/database
port: '5432'
username: artifice
invoice_object:
delimiter: ','
output_file: '%(tenant)s-%(start)s-%(end)s.csv'
@ -14,7 +8,8 @@ invoice_object:
rates:
file: /etc/artifice/csv_rates.csv
main:
invoice:object: billing.csv_invoice:Csv
export_provider: billing.csv_invoice:Csv
database_uri: postgres://artifice:123456@localhost:5432/artifice
openstack:
authentication_url: http://localhost:35357/v2.0
default_tenant: demo

15
is_provisioned.py Normal file
View File

@ -0,0 +1,15 @@
from artifice.models import Base, __VERSION__, _Version
from sqlalchemy.orm import scoped_session, create_session
from sqlalchemy import create_engine
from sqlalchemy.pool import NullPool
import sys, os
uri = os.environ["DATABASE_URI"]
engine = create_engine(uri, poolclass=NullPool)
session = create_session(bind=engine)
v = session.query(_Version).first()
if v is None:
sys.exit(0)
sys.exit(1)

View File

@ -0,0 +1,63 @@
#!/bin/sh
set -e
### BEGIN INIT INFO
# Provides: artifice
# Required-Start: $local_fs $remote_fs $network $time
# Required-Stop: $local_fs $remote_fs $network $time
# Should-Start: $syslog
# Should-Stop: $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Artifice-Openstack billing server
### END INIT INFO
ARTIFICE_PATH=/opt/stack/artifice
DAEMON="$ARTIFICE_PATH/bin/web"
NAME=artifice
PIDFILE="/var/run/artifice/${NAME}.pid"
test -x $DAEMON || exit 0
. /lib/lsb/init-functions
# versions can be specified explicitly
case "$1" in
start)
start-stop-daemon --start --quiet --pidfile $PIDFILE\
--startas $DAEMON
;;
stop)
start-stop-daemon --stop --quiet --pidfile $PIDFILE\
--oknodo
;;
restart)
start-stop-daemon --stop --quiet --pidfile $PIDFILE --retry TERM/10/KILL/5 --quiet --oknodo
start-stop-daemon --start --quiet --pidfile $PIDFILE \
--startas $DAEMON -- $NAME
;;
status)
if [ -f $PIDFILE ]; then
PID=`cat $PIDFILE`
RUNNING=`ps aux | awk '{print $2}' | grep $PIDFILE`
if [ $RUNNING = $PID ]; then
log_success_msg "$NAME is running"
else
log_failure_msg "$NAME is not running"
fi
else
log_failure_msg "$PIDFILE not found."
fi
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
exit 1
;;
esac
exit 0

View File

@ -1,27 +1,32 @@
#!/bin/sh
PASSWORD=`cat <%= install_path %>/etc/artifice/database`
# PASSWORD=`cat <%= install_path %>/etc/artifice/database`
# pip install virtualenv
export DATABASE_URL="postgresql://<%= pg_user %>:$PASSWORD@localhost:<%=pg_port%>/<%=pg_database%>"
pip install virtualenv
mkdir /var/run/artifice
# Set up a virtualenv for ourselves in this directory
virtualenv <%= install_path %>/env
# First, install an up-to-date pip into the virtualenv, since this is ridiculously ancient
<%=install_path%>/env/bin/pip install --upgrade pip
# Now iterate our requirements
# this should now be limited to only this space
<%=install_path%>/env/bin/pip install -r <%= install_path %>/requirements.txt
<%=install_path%>/env/bin/python <%= install_path %>/scripts/initdb.py
# And this. Woo.
<%=install_path%>/env/bin/python <%= install_path%>/setup.py install # register with python!
# Set up the /usr/local/artifice-bill script
cd <%=install_path%>
sudo ./env/bin/python ./setup.py install # register with python!
# Set up the artifice control scripts
cat > /usr/local/bin/artifice-bill <<EOF
#!/bin/bash
<%=install_path%>/env/bin/python <%=install_path%>/bin/bill.py \$@
<%=install_path%>/env/bin/python <%=install_path%>/bin/collect.py \$@
EOF
@ -33,7 +38,7 @@ EOF
chmod 0755 /usr/local/bin/artifice-usage
chmod 0755 /usr/local/bin/artifice-bill
cp <%=install_path%>/etc/artifice/conf.yaml /etc/artifice/conf.yaml
cp <%=install_path%>/etc/artifice/database /etc/artifice/database
chown 0644 /etc/artifice/database
# cp <%=install_path%>/etc/artifice/conf.yaml /etc/artifice/conf.yaml
# cp <%=install_path%>/etc/artifice/database /etc/artifice/database
# chown 0644 /etc/artifice/database

5
puppet/Modulefile Normal file
View File

@ -0,0 +1,5 @@
name 'catalyst-artifice'
version '0.0.1'
dependency 'reidmv/yamlfile', '0.2.0'
description "This module handles the installation and configuration of an
Openstack-Billing server, associated plugins, and storage backend (currently mysql and postgres.). Depends upon yamlfile to handle construction of a configuration file.

View File

@ -0,0 +1,68 @@
class artifice::config (
$keystone_uri,
$keystone_tenant,
$keystone_password,
$keystone_username,
$database_uri,
$ceilometer_uri,
$region
) {
# target => '/tmp/example1.yaml',
# key => 'value/subkey/final',
# value => ['one', 'two', 'three'],
#
$artifice_config_file = "/etc/artifice/conf.yaml"
# OPENSTACK SETTINGS
#
yaml_setting {"artifice.config.ceilometer.uri":
target => $artifice_config_file,
key => "ceilometer/host",
value => $ceilometer_uri
}
yaml_setting {"artifice.config.keystone.uri":
target => $artifice_config_file,
key => "openstack/authentication_url",
value => $keystone_uri
}
yaml_setting {"artifice.config.keystone.username":
target => $artifice_config_file,
key => "openstack/username",
value => $keystone_user
}
yaml_setting {"artifice.config.keystone.tenant":
target => $artifice_config_file,
key => "openstack/default_tenant",
value => $keystone_tenant
}
yaml_setting {"artifice.config.keystone.password":
target => $artifice_config_file,
key => "openstack/password",
value => $keystone_password
}
# DATABASE SETTINGS
yaml_setting {"artifice.config.database.uri":
target => $artifice_config_file,
key => "database/uri",
value => $database_uri
}
# Config settings for plugins are stored in the plugins directory
# file {"/etc/artifice/conf.yaml":
# ensure => present,
# content => template("artifice/etc/artifice/conf.yaml")
# }
# Region
#
yaml_setting {"artifice.config.region":
target => $artifice_config_file,
key => "region",
value => $region
}
}

View File

@ -0,0 +1,27 @@
class artifice::database (
$provider,
$host,
$port,
$user,
$password,
$database_name
) {
# I think the install path should
#
if $provider != "postgres" and $provider != "mysql" {
fail("Provider must be postgres or mysql")
}
$install_path = "/opt/stack/artifice"
# Create is handled by the Galera setup earlier.
# exec {"create.database":
# command => $create_command,
# cwd => $pwd,
# onlyif => $unless_command
# }
exec {"sqlalchemy.create":
command => "/usr/bin/python $install_path/initdb.py",
environment => "DATABASE_URI=$provider://$user:$password@$host/$database_name",
onlyif => "/usr/bin/python $install_path/is_provisioned.py",
}
}

61
puppet/manifests/init.pp Normal file
View File

@ -0,0 +1,61 @@
class artifice (
$keystone_password,
$region,
$version,
$database_password,
$database_provider = "postgres",
$database_host = "localhost",
$database_port = 5432,
$database_name = "artifice",
$database_user = "artifice",
$csv_output_directory = '/var/lib/artifice/csv',
$ceilometer_uri = "http://localhost:8777",
$keystone_uri = "http://localhost:35357/v2.0",
$keystone_tenant = "demo",
$keystone_username = "admin"
) {
# Materialises the class
# I think.. this is better served as part of the package install
$config_file = "/etc/artifice/conf.yaml"
$install_path = "/opt/stack/artifice"
class {"artifice::server":
# region => $region,
version => $version,
# require => Class["artifice::dependencies"]
}
$database_uri = "$database_provider://${database_user}:${database_password}@${database_host}:${database_port}/${database_name}"
class {"artifice::config":
keystone_uri => $keystone_uri,
keystone_tenant => $keystone_tenant,
keystone_username => $keystone_username,
keystone_password => $keystone_password,
database_uri => $database_uri,
ceilometer_uri => $ceilometer_uri,
require => Class["artifice::server"],
notify => Service["artifice"],
region => $region
}
class {"artifice::database":
provider => $database_provider,
host => $database_host,
port => $database_port,
user => $database_user,
password => $database_password,
database_name => $database_name,
require => Class["artifice::server"]
}
service {"artifice":
ensure => running,
require => [
Class["artifice::server"],
Class["artifice::config"]
]
}
}

View File

@ -0,0 +1,29 @@
class artifice::plugins::csv (
$delimiter,
$output_path,
$output_pattern
) {
# This should cause a runtime error if another plugin is loaded
yaml_setting {"artifice.billing.plugin":
target => $artifice::config_file,
key => "main/invoice:object",
value => "billing.csv_plugin:Csv"
}
yaml_setting {"artifice.csv.config.delimiter":
target => $artifice::config_file,
key => "invoice_object/delimiter",
value => $delimiter
}
yaml_setting {"artifice.csv.config.output_path":
target => $artifice::config_file,
key => "invoice_object/output_path",
value => $output_path
}
yaml_setting {"artifice.csv.config.output_file":
target => $artifice::config_file,
key => "invoice_object/output_file",
value => $output_pattern
}
# Rates information is pulled from the rates-file plugin
}

View File

@ -0,0 +1,11 @@
class artifice::plugins::csv::rates_file (
$path
) {
# Sets the path to the rates information, if any
yaml_setting {"artifice.plugins.rates":
target => $artifice::config_file,
key => "invoice_object/rates/file",
value => $path,
require => File[$path]
}
}

View File

@ -0,0 +1,27 @@
class artifice::server(
$version
) {
# $path_to_package = $::package_path + "/artifice" + $version + ".deb"
# package {"python2.7":
# ensure => present
# }
package {"artifice":
name => "openstack-artifice",
ensure => present,
require => Package["python2.7"]
}
package {"libpq-dev":
ensure => present
}
package {"python2.7": ensure => present}
package {"python-pip": ensure => present}
package {"python-dev": ensure => present}
package {"python-virtualenv": ensure => present}
Package["python-virtualenv"] -> Package["artifice"]
# We don't try to ensure running here.
#
}

52
puppet_generate.py Normal file
View File

@ -0,0 +1,52 @@
import sys
# pip install requirements-parser
import requirements
class Requirements(object):
def __init__(self):
self.reqs = []
def parse(self, stream):
self.reqs = requirements.parse(stream)
def package_list(self):
final = """"""
for req in self.reqs:
final += """
package {"%(package)s":
ensure => "%(version)s",
provider => pip
}
""" % {"package": req.name, "version": req.specs[0][1] }
return final
def requirement_list(self):
return ",\n".join( [ """Package[%(package)s]""" %
{"package": req.name } for req in self.reqs ] )
if __name__ == '__main__':
import argparse
a = argparse.ArgumentParser()
a.add_argument("-f", dest="filename")
a.add_argument("-l", dest="list_", action="store_true")
args = a.parse_args()
if args.filename == "-":
# We're following standardized posix thing
fh = sys.stdin
else:
try:
fh = open(args.filename)
except IOError as e:
print "Couldn't open %s" % args.filename
sys.exit(1)
r = Requirements()
r.parse(fh)
if args.list_:
print r.requirement_list()
sys.exit(0)
print r.package_list()

View File

@ -1,13 +1,37 @@
sqlalchemy>=0.8
psycopg2>=2.5.1
requests==1.1.0
pyaml==13.07
python-ceilometerclient==1.0.3
python-novaclient>=2.17
python-keystoneclient==0.3.2
urllib3==1.5
Babel==1.3
Flask==0.10.1
Jinja2==2.7.2
MarkupSafe==0.18
MySQL-python==1.2.5
PyMySQL==0.6.1
PyYAML==3.10
SQLAlchemy==0.8.0
WebOb==1.3.1
WebTest==2.0.14
Werkzeug==0.9.4
argparse==1.2.1
beautifulsoup4==4.3.2
decorator==3.4.0
httplib2==0.8
iso8601==0.1.8
itsdangerous==0.23
mock==1.0.1
netaddr==0.7.10
nose==1.3.0
oslo.config==1.2.1
pbr==0.6
prettytable==0.7.2
psycopg2==2.5.2
pyaml==13.07.0
python-ceilometerclient==1.0.3
python-keystoneclient==0.3.2
python-novaclient==2.17.0
pytz==2013.9
requests==1.1.0
requirements-parser==0.0.6
simplejson==3.3.3
six==1.5.2
urllib3==1.5
waitress==0.8.8
wsgiref==0.1.2

View File

@ -0,0 +1,12 @@
from setuptools import setup
setup(name='openstack-artifice',
version='0.1',
description='Artifice, a set of APIs for creating billable items from Openstack-Ceilometer',
author='Aurynn Shaw',
author_email='aurynn@catalyst.net.nz',
contributors=["Chris Forbes", "Adrian Turjak"],
contributor_emails=["chris.forbes@catalyst.net.nz", "adriant@catalyst.net.nz"],
url='https://github.com/catalyst/artifice',
packages=["artifice", "artifice.api", "artifice.models"]
)

View File

@ -1,6 +1,6 @@
from webtest import TestApp
from . import test_interface, helpers, constants
from api.web import get_app
from artifice.api.web import get_app
from artifice import models
from artifice import interface
from datetime import datetime

View File

@ -68,7 +68,7 @@ class db(unittest.TestCase):
t = self.db.query(Tenant).get("asfd")
r = self.db.query(Resource).filter(Resource.id == "1234")[0]
u = UsageEntry(service="cheese",
volume=1.234,
volume=1.23,
resource=r,
tenant=r,
start=datetime.datetime.now() - datetime.timedelta(minutes=5),