summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlexandre Levine <alevine@cloudscaling.com>2014-07-11 03:46:52 +0400
committeralevine <alevine@cloudscaling.com>2014-07-18 19:33:55 -0700
commit66826e9e5b36d60f66b6c3539c9d468c7312db3f (patch)
treeb40d75503049fadd75dd3fa16ba931f6d4e4ea36
parent1592a74ea34ce2af27d52908308a048b5b7f7c96 (diff)
Initial EC2-API service commit.
This code introduces standalone service which proxies its calls to existing nova EC2-API. All the code here except for the ec2api/api/proxy.py, ec2api/api/ec2client.py and some util functions is taken from current nova and unused functionality is cut of it. The proxy.py and ec2client.py files implement the new code which proxies incoming request (on port 8788) to original EC2 API in nova on port 8773. The result is transparently translated back to user. Change-Id: I4cb84f833d7d4f0e379672710ed39562811d43e0
Notes
Notes (review): Verified+2: Jenkins Code-Review+2: Alexandre Levine <alevine@cloudscaling.com> Workflow+1: Alexandre Levine <alevine@cloudscaling.com> Code-Review+1: Feodor Tersin <ftersin@cloudscaling.com> Code-Review+1: Andrey Pavlov <apavlov@cloudscaling.com> Submitted-by: Jenkins Submitted-at: Sat, 19 Jul 2014 02:49:25 +0000 Reviewed-on: https://review.openstack.org/106209 Project: stackforge/ec2-api Branch: refs/heads/master
-rw-r--r--.testr.conf4
-rw-r--r--HACKING.rst43
-rw-r--r--LICENSE176
-rw-r--r--MANIFEST.in20
-rw-r--r--README.rst68
-rw-r--r--babel.cfg1
-rwxr-xr-xbin/ec2api-db-setup292
-rw-r--r--ec2api/.project18
-rw-r--r--ec2api/.pydevproject5
-rw-r--r--ec2api/__init__.py27
-rw-r--r--ec2api/api/__init__.py400
-rw-r--r--ec2api/api/apirequest.py143
-rw-r--r--ec2api/api/auth.py54
-rw-r--r--ec2api/api/clients.py141
-rw-r--r--ec2api/api/cloud.py41
-rw-r--r--ec2api/api/ec2client.py222
-rw-r--r--ec2api/api/ec2utils.py186
-rw-r--r--ec2api/api/faults.py96
-rw-r--r--ec2api/api/proxy.py27
-rw-r--r--ec2api/api/validator.py132
-rw-r--r--ec2api/cmd/__init__.py16
-rw-r--r--ec2api/cmd/api.py42
-rw-r--r--ec2api/cmd/manage.py75
-rw-r--r--ec2api/config.py30
-rw-r--r--ec2api/context.py150
-rw-r--r--ec2api/exception.py279
-rw-r--r--ec2api/openstack/__init__.py0
-rw-r--r--ec2api/openstack/common/__init__.py17
-rw-r--r--ec2api/openstack/common/context.py126
-rw-r--r--ec2api/openstack/common/db/__init__.py0
-rw-r--r--ec2api/openstack/common/db/api.py162
-rw-r--r--ec2api/openstack/common/db/exception.py56
-rw-r--r--ec2api/openstack/common/db/options.py171
-rw-r--r--ec2api/openstack/common/db/sqlalchemy/__init__.py0
-rw-r--r--ec2api/openstack/common/db/sqlalchemy/migration.py278
-rw-r--r--ec2api/openstack/common/db/sqlalchemy/models.py119
-rw-r--r--ec2api/openstack/common/db/sqlalchemy/provision.py157
-rw-r--r--ec2api/openstack/common/db/sqlalchemy/session.py905
-rw-r--r--ec2api/openstack/common/db/sqlalchemy/test_base.py167
-rw-r--r--ec2api/openstack/common/db/sqlalchemy/test_migrations.py270
-rw-r--r--ec2api/openstack/common/db/sqlalchemy/utils.py655
-rw-r--r--ec2api/openstack/common/eventlet_backdoor.py145
-rw-r--r--ec2api/openstack/common/excutils.py113
-rw-r--r--ec2api/openstack/common/gettextutils.py479
-rw-r--r--ec2api/openstack/common/importutils.py73
-rw-r--r--ec2api/openstack/common/jsonutils.py190
-rw-r--r--ec2api/openstack/common/local.py45
-rw-r--r--ec2api/openstack/common/log.py689
-rw-r--r--ec2api/openstack/common/loopingcall.py147
-rw-r--r--ec2api/openstack/common/service.py512
-rw-r--r--ec2api/openstack/common/strutils.py295
-rw-r--r--ec2api/openstack/common/systemd.py106
-rw-r--r--ec2api/openstack/common/threadgroup.py147
-rw-r--r--ec2api/openstack/common/timeutils.py210
-rw-r--r--ec2api/openstack/common/uuidutils.py37
-rw-r--r--ec2api/paths.py64
-rw-r--r--ec2api/service.py163
-rw-r--r--ec2api/tests/__init__.py26
-rw-r--r--ec2api/tests/fakes_request_response.py312
-rw-r--r--ec2api/tests/matchers.py451
-rw-r--r--ec2api/tests/test_api_init.py129
-rw-r--r--ec2api/tests/test_tools.py45
-rw-r--r--ec2api/tests/tools.py38
-rw-r--r--ec2api/utils.py49
-rw-r--r--ec2api/version.py17
-rw-r--r--ec2api/wsgi.py501
-rw-r--r--etc/ec2api/api-paste.ini39
-rw-r--r--etc/ec2api/ec2api.conf.sample717
-rwxr-xr-xinstall.sh247
-rw-r--r--openstack-common.conf7
-rw-r--r--requirements.txt26
-rwxr-xr-xrun_tests.sh123
-rw-r--r--setup.cfg69
-rw-r--r--setup.py30
-rw-r--r--test-requirements.txt13
-rw-r--r--tools/config/README20
-rwxr-xr-xtools/config/analyze_opts.py81
-rwxr-xr-xtools/config/check_uptodate.sh25
-rwxr-xr-xtools/config/generate_sample.sh119
-rw-r--r--tools/config/oslo.config.generator.rc2
-rwxr-xr-xtools/db/schema_diff.py270
-rwxr-xr-xtools/enable-pre-commit-hook.sh42
-rw-r--r--tools/install_venv.py74
-rw-r--r--tools/install_venv_common.py213
-rwxr-xr-xtools/lintstack.py199
-rwxr-xr-xtools/lintstack.sh59
-rw-r--r--tools/patch_tox_venv.py50
-rwxr-xr-xtools/regression_tester.py109
-rwxr-xr-xtools/with_venv.sh7
-rw-r--r--tox.ini56
90 files changed, 13351 insertions, 0 deletions
diff --git a/.testr.conf b/.testr.conf
new file mode 100644
index 0000000..d43ba19
--- /dev/null
+++ b/.testr.conf
@@ -0,0 +1,4 @@
1[DEFAULT]
2test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_LOG_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ec2api/tests $LISTOPT $IDOPTION
3test_id_option=--load-list $IDFILE
4test_list_option=--list
diff --git a/HACKING.rst b/HACKING.rst
new file mode 100644
index 0000000..146bc49
--- /dev/null
+++ b/HACKING.rst
@@ -0,0 +1,43 @@
1Ec2api Style Commandments
2=========================
3
4- Step 1: Read the OpenStack Style Commandments
5 https://github.com/openstack-dev/hacking/blob/master/doc/source/index.rst
6- Step 2: Read on
7
8Ec2api Specific Commandments
9----------------------------
10
11General
12-------
13- Do not use locals(). Example::
14
15 LOG.debug(_("volume %(vol_name)s: creating size %(vol_size)sG") %
16 locals()) # BAD
17
18 LOG.debug(_("volume %(vol_name)s: creating size %(vol_size)sG") %
19 {'vol_name': vol_name,
20 'vol_size': vol_size}) # OKAY
21
22- Use 'raise' instead of 'raise e' to preserve original traceback or exception being reraised::
23
24 except Exception as e:
25 ...
26 raise e # BAD
27
28 except Exception:
29 ...
30 raise # OKAY
31
32
33
34Creating Unit Tests
35-------------------
36For every new feature, unit tests should be created that both test and
37(implicitly) document the usage of said feature. If submitting a patch for a
38bug that had no unit test, a new passing unit test should be added. If a
39submitted bug fix does have a unit test, be sure to add a new one that fails
40without the patch and passes with the patch.
41
42For more information on creating unit tests and utilizing the testing
43infrastructure in OpenStack Ec2api, please read ec2api/testing/README.rst.
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..68c771a
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,176 @@
1
2 Apache License
3 Version 2.0, January 2004
4 http://www.apache.org/licenses/
5
6 TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
8 1. Definitions.
9
10 "License" shall mean the terms and conditions for use, reproduction,
11 and distribution as defined by Sections 1 through 9 of this document.
12
13 "Licensor" shall mean the copyright owner or entity authorized by
14 the copyright owner that is granting the License.
15
16 "Legal Entity" shall mean the union of the acting entity and all
17 other entities that control, are controlled by, or are under common
18 control with that entity. For the purposes of this definition,
19 "control" means (i) the power, direct or indirect, to cause the
20 direction or management of such entity, whether by contract or
21 otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 outstanding shares, or (iii) beneficial ownership of such entity.
23
24 "You" (or "Your") shall mean an individual or Legal Entity
25 exercising permissions granted by this License.
26
27 "Source" form shall mean the preferred form for making modifications,
28 including but not limited to software source code, documentation
29 source, and configuration files.
30
31 "Object" form shall mean any form resulting from mechanical
32 transformation or translation of a Source form, including but
33 not limited to compiled object code, generated documentation,
34 and conversions to other media types.
35
36 "Work" shall mean the work of authorship, whether in Source or
37 Object form, made available under the License, as indicated by a
38 copyright notice that is included in or attached to the work
39 (an example is provided in the Appendix below).
40
41 "Derivative Works" shall mean any work, whether in Source or Object
42 form, that is based on (or derived from) the Work and for which the
43 editorial revisions, annotations, elaborations, or other modifications
44 represent, as a whole, an original work of authorship. For the purposes
45 of this License, Derivative Works shall not include works that remain
46 separable from, or merely link (or bind by name) to the interfaces of,
47 the Work and Derivative Works thereof.
48
49 "Contribution" shall mean any work of authorship, including
50 the original version of the Work and any modifications or additions
51 to that Work or Derivative Works thereof, that is intentionally
52 submitted to Licensor for inclusion in the Work by the copyright owner
53 or by an individual or Legal Entity authorized to submit on behalf of
54 the copyright owner. For the purposes of this definition, "submitted"
55 means any form of electronic, verbal, or written communication sent
56 to the Licensor or its representatives, including but not limited to
57 communication on electronic mailing lists, source code control systems,
58 and issue tracking systems that are managed by, or on behalf of, the
59 Licensor for the purpose of discussing and improving the Work, but
60 excluding communication that is conspicuously marked or otherwise
61 designated in writing by the copyright owner as "Not a Contribution."
62
63 "Contributor" shall mean Licensor and any individual or Legal Entity
64 on behalf of whom a Contribution has been received by Licensor and
65 subsequently incorporated within the Work.
66
67 2. Grant of Copyright License. Subject to the terms and conditions of
68 this License, each Contributor hereby grants to You a perpetual,
69 worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 copyright license to reproduce, prepare Derivative Works of,
71 publicly display, publicly perform, sublicense, and distribute the
72 Work and such Derivative Works in Source or Object form.
73
74 3. Grant of Patent License. Subject to the terms and conditions of
75 this License, each Contributor hereby grants to You a perpetual,
76 worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 (except as stated in this section) patent license to make, have made,
78 use, offer to sell, sell, import, and otherwise transfer the Work,
79 where such license applies only to those patent claims licensable
80 by such Contributor that are necessarily infringed by their
81 Contribution(s) alone or by combination of their Contribution(s)
82 with the Work to which such Contribution(s) was submitted. If You
83 institute patent litigation against any entity (including a
84 cross-claim or counterclaim in a lawsuit) alleging that the Work
85 or a Contribution incorporated within the Work constitutes direct
86 or contributory patent infringement, then any patent licenses
87 granted to You under this License for that Work shall terminate
88 as of the date such litigation is filed.
89
90 4. Redistribution. You may reproduce and distribute copies of the
91 Work or Derivative Works thereof in any medium, with or without
92 modifications, and in Source or Object form, provided that You
93 meet the following conditions:
94
95 (a) You must give any other recipients of the Work or
96 Derivative Works a copy of this License; and
97
98 (b) You must cause any modified files to carry prominent notices
99 stating that You changed the files; and
100
101 (c) You must retain, in the Source form of any Derivative Works
102 that You distribute, all copyright, patent, trademark, and
103 attribution notices from the Source form of the Work,
104 excluding those notices that do not pertain to any part of
105 the Derivative Works; and
106
107 (d) If the Work includes a "NOTICE" text file as part of its
108 distribution, then any Derivative Works that You distribute must
109 include a readable copy of the attribution notices contained
110 within such NOTICE file, excluding those notices that do not
111 pertain to any part of the Derivative Works, in at least one
112 of the following places: within a NOTICE text file distributed
113 as part of the Derivative Works; within the Source form or
114 documentation, if provided along with the Derivative Works; or,
115 within a display generated by the Derivative Works, if and
116 wherever such third-party notices normally appear. The contents
117 of the NOTICE file are for informational purposes only and
118 do not modify the License. You may add Your own attribution
119 notices within Derivative Works that You distribute, alongside
120 or as an addendum to the NOTICE text from the Work, provided
121 that such additional attribution notices cannot be construed
122 as modifying the License.
123
124 You may add Your own copyright statement to Your modifications and
125 may provide additional or different license terms and conditions
126 for use, reproduction, or distribution of Your modifications, or
127 for any such Derivative Works as a whole, provided Your use,
128 reproduction, and distribution of the Work otherwise complies with
129 the conditions stated in this License.
130
131 5. Submission of Contributions. Unless You explicitly state otherwise,
132 any Contribution intentionally submitted for inclusion in the Work
133 by You to the Licensor shall be under the terms and conditions of
134 this License, without any additional terms or conditions.
135 Notwithstanding the above, nothing herein shall supersede or modify
136 the terms of any separate license agreement you may have executed
137 with Licensor regarding such Contributions.
138
139 6. Trademarks. This License does not grant permission to use the trade
140 names, trademarks, service marks, or product names of the Licensor,
141 except as required for reasonable and customary use in describing the
142 origin of the Work and reproducing the content of the NOTICE file.
143
144 7. Disclaimer of Warranty. Unless required by applicable law or
145 agreed to in writing, Licensor provides the Work (and each
146 Contributor provides its Contributions) on an "AS IS" BASIS,
147 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 implied, including, without limitation, any warranties or conditions
149 of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 PARTICULAR PURPOSE. You are solely responsible for determining the
151 appropriateness of using or redistributing the Work and assume any
152 risks associated with Your exercise of permissions under this License.
153
154 8. Limitation of Liability. In no event and under no legal theory,
155 whether in tort (including negligence), contract, or otherwise,
156 unless required by applicable law (such as deliberate and grossly
157 negligent acts) or agreed to in writing, shall any Contributor be
158 liable to You for damages, including any direct, indirect, special,
159 incidental, or consequential damages of any character arising as a
160 result of this License or out of the use or inability to use the
161 Work (including but not limited to damages for loss of goodwill,
162 work stoppage, computer failure or malfunction, or any and all
163 other commercial damages or losses), even if such Contributor
164 has been advised of the possibility of such damages.
165
166 9. Accepting Warranty or Additional Liability. While redistributing
167 the Work or Derivative Works thereof, You may choose to offer,
168 and charge a fee for, acceptance of support, warranty, indemnity,
169 or other liability obligations and/or rights consistent with this
170 License. However, in accepting such obligations, You may act only
171 on Your own behalf and on Your sole responsibility, not on behalf
172 of any other Contributor, and only if You agree to indemnify,
173 defend, and hold each Contributor harmless for any liability
174 incurred by, or claims asserted against, such Contributor by reason
175 of your accepting any such warranty or additional liability.
176
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..7f0f568
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,20 @@
1include run_tests.sh ChangeLog
2include README.rst builddeb.sh
3include MANIFEST.in pylintrc
4include AUTHORS
5include run_tests.py
6include HACKING.rst
7include LICENSE
8include ChangeLog
9include babel.cfg tox.ini
10include openstack-common.conf
11include ec2api/openstack/common/README
12include ec2api/db/sqlalchemy/migrate_repo/README
13include ec2api/db/sqlalchemy/migrate_repo/migrate.cfg
14include ec2api/db/sqlalchemy/migrate_repo/versions/*.sql
15graft doc
16graft etc
17graft ec2api/locale
18graft ec2api/tests
19graft tools
20global-exclude *.pyc
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..ce18148
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,68 @@
1OpenStack EC2 API README
2-----------------------------
3
4Support of EC2 API for OpenStack.
5This project provides a standalone EC2 API service which pursues two goals:
61. Implement VPC API which now absent in nova's EC2 API
72. Create a standalone service for EC2 API support which later can accommodate
8not only the VPC API but the rest of the EC2 API currently present in nova as
9well.
10
11This service implements VPC API related commands only. For the rest of the
12EC2 API functionality it redirects request to original EC2 API in nova.
13
14It doesn't replace existing nova EC2 API service in deployment it gets
15installed to a different port (8788 by default).
16
17Installation
18=====
19
20Run install.sh
21
22#TODO: The following should be automated later.
23
24Change /etc/ec2api/ec2api.conf:
25[database]
26connection_nova = <connection to nova> #should be taken from nova.conf
27[DEFAULT]
28external_network = <public network name> #obtained by neutron net-external-list
29
30The service gets installed on port 8788 by default. It can be changed before the
31installation in install.sh script.
32
33Usage
34=====
35
36Download aws cli from Amazon.
37Create configuration file for aws cli in your home directory ~/.aws/config:
38
39[default]
40aws_access_key_id = 1b013f18d5ed47ae8ed0fbb8debc036b
41aws_secret_access_key = 9bbc6f270ffd4dfdbe0e896947f41df3
42region = us-east-1
43
44Change the aws_access_key_id and aws_secret_acces_key above to the values
45appropriate for your cloud (can be obtained by "keystone ec2-credentials-list"
46command).
47
48Run aws cli commands using new EC2 API endpoint URL (can be obtained from
49keystone with the new port 8788) like this:
50
51aws --endpoint-url http://10.0.2.15:8788/services/Cloud ec2 describe-instances
52
53
54Limitations
55===========
56
57This is an alpha-version, Tempest tests are not run yet.
58VPN-related functionality is not supported yet.
59Route-tables functionality is limited.
60Filtering in describe functions can be done by IDs only.
61Security groups are attached to network interfaces only, not to instances yet.
62Rollbacks in case of failure during object creation are not supported yet.
63Some other not-listed here limitations exist also.
64
65Supported Features
66==================
67
68VPC API except for the Limitations above is supported.
diff --git a/babel.cfg b/babel.cfg
new file mode 100644
index 0000000..efceab8
--- /dev/null
+++ b/babel.cfg
@@ -0,0 +1 @@
[python: **.py]
diff --git a/bin/ec2api-db-setup b/bin/ec2api-db-setup
new file mode 100755
index 0000000..1b7cbcd
--- /dev/null
+++ b/bin/ec2api-db-setup
@@ -0,0 +1,292 @@
1#!/bin/bash -e
2#
3# Copyright 2014 Cloudscaling Group, Inc
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may
6# not use this file except in compliance with the License. You may obtain
7# a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations
15# under the License.
16#
17
18#
19# Print --help output and exit.
20#
21usage() {
22
23cat << EOF
24Set up a local MySQL database for use with ec2api.
25This script will create a 'ec2api' database that is accessible
26only on localhost by user 'ec2api' with password 'ec2api'.
27
28Usage: ec2api-db-setup <rpm|deb> [options]
29Options:
30 select a distro type (rpm or debian)
31
32 --help | -h
33 Print usage information.
34 --password <pw> | -p <pw>
35 Specify the password for the 'ec2api' MySQL user that will
36 use to connect to the 'ec2api' MySQL database. By default,
37 the password 'ec2api' will be used.
38 --rootpw <pw> | -r <pw>
39 Specify the root MySQL password. If the script installs
40 the MySQL server, it will set the root password to this value
41 instead of prompting for a password. If the MySQL server is
42 already installed, this password will be used to connect to the
43 database instead of having to prompt for it.
44 --yes | -y
45 In cases where the script would normally ask for confirmation
46 before doing something, such as installing mysql-server,
47 just assume yes. This is useful if you want to run the script
48 non-interactively.
49EOF
50
51 exit 0
52}
53
54install_mysql_server() {
55 if [ -z "${ASSUME_YES}" ] ; then
56 $PACKAGE_INSTALL mysql-server
57 else
58 $PACKAGE_INSTALL -y mysql-server
59 fi
60}
61
62start_mysql_server() {
63 $SERVICE_START
64}
65
66MYSQL_EC2API_PW_DEFAULT="ec2api"
67MYSQL_EC2API_PW=${MYSQL_EC2API_PW_DEFAULT}
68EC2API_CONFIG="/etc/ec2api/ec2api.conf"
69ASSUME_YES=""
70ELEVATE=""
71
72# Check for root privileges
73if [[ $EUID -ne 0 ]] ; then
74 echo "This operation requires superuser privileges, using sudo:"
75 if sudo -l > /dev/null ; then
76 ELEVATE="sudo"
77 else
78 exit 1
79 fi
80fi
81
82case "$1" in
83 rpm)
84 echo "Installing on an RPM system."
85 PACKAGE_INSTALL="$ELEVATE yum install"
86 PACKAGE_STATUS="rpm -q"
87 SERVICE_MYSQLD="mysqld"
88 SERVICE_START="$ELEVATE service $SERVICE_MYSQLD start"
89 SERVICE_STATUS="service $SERVICE_MYSQLD status"
90 SERVICE_ENABLE="$ELEVATE chkconfig"
91 ;;
92 deb)
93 echo "Installing on a Debian system."
94 PACKAGE_INSTALL="$ELEVATE apt-get install"
95 PACKAGE_STATUS="dpkg-query -s"
96 SERVICE_MYSQLD="mysql"
97 SERVICE_START="$ELEVATE service $SERVICE_MYSQLD start"
98 SERVICE_STATUS="$ELEVATE service $SERVICE_MYSQLD status"
99 SERVICE_ENABLE=""
100 ;;
101 *)
102 usage
103 ;;
104esac
105
106while [ $# -gt 0 ]
107do
108 case "$1" in
109 -h|--help)
110 usage
111 ;;
112 -p|--password)
113 shift
114 MYSQL_EC2API_PW=${1}
115 ;;
116 -r|--rootpw)
117 shift
118 MYSQL_ROOT_PW=${1}
119 ;;
120 -y|--yes)
121 ASSUME_YES="yes"
122 ;;
123 *)
124 # ignore
125 ;;
126 esac
127 shift
128done
129
130
131# Make sure MySQL is installed.
132
133NEW_MYSQL_INSTALL=0
134if ! $PACKAGE_STATUS mysql-server && ! $PACKAGE_STATUS mariadb-server && ! $PACKAGE_STATUS mariadb-galera-server > /dev/null
135then
136 if [ -z "${ASSUME_YES}" ] ; then
137 printf "mysql-server is not installed. Would you like to install it now? (y/n): "
138 read response
139 case "$response" in
140 y|Y)
141 ;;
142 n|N)
143 echo "mysql-server must be installed. Please install it before proceeding."
144 exit 0
145 ;;
146 *)
147 echo "Invalid response."
148 exit 1
149 esac
150 fi
151
152 NEW_MYSQL_INSTALL=1
153 install_mysql_server
154fi
155
156
157# Make sure mysqld is running.
158
159if ! $SERVICE_STATUS > /dev/null
160then
161 if [ -z "${ASSUME_YES}" ] ; then
162 printf "$SERVICE_MYSQLD is not running. Would you like to start it now? (y/n): "
163 read response
164 case "$response" in
165 y|Y)
166 ;;
167 n|N)
168 echo "$SERVICE_MYSQLD must be running. Please start it before proceeding."
169 exit 0
170 ;;
171 *)
172 echo "Invalid response."
173 exit 1
174 esac
175 fi
176
177 start_mysql_server
178
179 # If we both installed and started, ensure it starts at boot
180 [ $NEW_MYSQL_INSTALL -eq 1 ] && $SERVICE_ENABLE $SERVICE_MYSQLD on
181fi
182
183
184# Get MySQL root access.
185
186if [ $NEW_MYSQL_INSTALL -eq 1 ]
187then
188 if [ ! "${MYSQL_ROOT_PW+defined}" ] ; then
189 echo "Since this is a fresh installation of MySQL, please set a password for the 'root' mysql user."
190
191 PW_MATCH=0
192 while [ $PW_MATCH -eq 0 ]
193 do
194 printf "Enter new password for 'root' mysql user: "
195 read -s MYSQL_ROOT_PW
196 echo
197 printf "Enter new password again: "
198 read -s PW2
199 echo
200 if [ "${MYSQL_ROOT_PW}" = "${PW2}" ] ; then
201 PW_MATCH=1
202 else
203 echo "Passwords did not match."
204 fi
205 done
206 fi
207
208 echo "UPDATE mysql.user SET password = password('${MYSQL_ROOT_PW}') WHERE user = 'root'; DELETE FROM mysql.user WHERE user = ''; flush privileges;" | mysql -u root
209 if ! [ $? -eq 0 ] ; then
210 echo "Failed to set password for 'root' MySQL user."
211 exit 1
212 fi
213elif [ ! "${MYSQL_ROOT_PW+defined}" ] ; then
214 printf "Please enter the password for the 'root' MySQL user: "
215 read -s MYSQL_ROOT_PW
216 echo
217fi
218
219
220# Sanity check MySQL credentials.
221
222MYSQL_ROOT_PW_ARG=""
223if [ "${MYSQL_ROOT_PW+defined}" ]
224then
225 MYSQL_ROOT_PW_ARG="--password=${MYSQL_ROOT_PW}"
226fi
227echo "SELECT 1;" | mysql -u root ${MYSQL_ROOT_PW_ARG} > /dev/null
228if ! [ $? -eq 0 ]
229then
230 echo "Failed to connect to the MySQL server. Please check your root user credentials."
231 exit 1
232fi
233echo "Verified connectivity to MySQL."
234
235
236# Now create the db.
237
238echo "Creating 'ec2api' database."
239cat << EOF | mysql -u root ${MYSQL_ROOT_PW_ARG}
240DROP DATABASE IF EXISTS ec2api;
241CREATE DATABASE IF NOT EXISTS ec2api DEFAULT CHARACTER SET utf8;
242GRANT ALL ON ec2api.* TO 'ec2api'@'localhost' IDENTIFIED BY '${MYSQL_EC2API_PW}';
243GRANT ALL ON ec2api.* TO 'ec2api'@'%' IDENTIFIED BY '${MYSQL_EC2API_PW}';
244flush privileges;
245EOF
246
247
248# Make sure ec2api configuration has the right MySQL password.
249
250if [ "${MYSQL_EC2API_PW}" != "${MYSQL_EC2API_PW_DEFAULT}" ] ; then
251 echo "Updating 'ec2api' database password in ${EC2API_CONFIG}"
252 sed -i -e "s/mysql:\/\/ec2api:\(.*\)@/mysql:\/\/ec2api:${MYSQL_EC2API_PW}@/" ${EC2API_CONFIG}
253fi
254
255# override the logging config in ec2api.conf
256log_conf=$(mktemp /tmp/ec2api-logging.XXXXXXXXXX.conf)
257cat <<EOF > $log_conf
258[loggers]
259keys=root
260
261[handlers]
262keys=consoleHandler
263
264[formatters]
265keys=simpleFormatter
266
267[logger_root]
268level=INFO
269handlers=consoleHandler
270
271[handler_consoleHandler]
272class=StreamHandler
273formatter=simpleFormatter
274args=(sys.stdout,)
275
276[formatter_simpleFormatter]
277format=%(name)s - %(levelname)s - %(message)s
278EOF
279
280ec2-api-manage --log-config=$log_conf db_sync
281rm $log_conf
282
283# Do a final sanity check on the database.
284
285echo "SELECT * FROM migrate_version;" | mysql -u ec2api --password=${MYSQL_EC2API_PW} ec2api > /dev/null
286if ! [ $? -eq 0 ]
287then
288 echo "Final sanity check failed."
289 exit 1
290fi
291
292echo "Complete!"
diff --git a/ec2api/.project b/ec2api/.project
new file mode 100644
index 0000000..27c66e7
--- /dev/null
+++ b/ec2api/.project
@@ -0,0 +1,18 @@
1<?xml version="1.0" encoding="UTF-8"?>
2<projectDescription>
3 <name>ec2api</name>
4 <comment></comment>
5 <projects>
6 </projects>
7 <buildSpec>
8 <buildCommand>
9 <name>org.python.pydev.PyDevBuilder</name>
10 <arguments>
11 </arguments>
12 </buildCommand>
13 </buildSpec>
14 <natures>
15 <nature>com.aptana.projects.webnature</nature>
16 <nature>org.python.pydev.pythonNature</nature>
17 </natures>
18</projectDescription>
diff --git a/ec2api/.pydevproject b/ec2api/.pydevproject
new file mode 100644
index 0000000..40e9f40
--- /dev/null
+++ b/ec2api/.pydevproject
@@ -0,0 +1,5 @@
1<?xml version="1.0" encoding="UTF-8" standalone="no"?>
2<?eclipse-pydev version="1.0"?><pydev_project>
3<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
4<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
5</pydev_project>
diff --git a/ec2api/__init__.py b/ec2api/__init__.py
new file mode 100644
index 0000000..62f43c5
--- /dev/null
+++ b/ec2api/__init__.py
@@ -0,0 +1,27 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15"""
16:mod:`ec2api` -- Cloud IaaS Platform
17===================================
18
19.. automodule:: ec2api
20 :platform: Unix
21 :synopsis: Infrastructure-as-a-Service Cloud platform.
22"""
23
24import gettext
25
26
27gettext.install('ec2api', unicode=1)
diff --git a/ec2api/api/__init__.py b/ec2api/api/__init__.py
new file mode 100644
index 0000000..f3aceff
--- /dev/null
+++ b/ec2api/api/__init__.py
@@ -0,0 +1,400 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15"""
16Starting point for routing EC2 requests.
17"""
18
19from eventlet.green import httplib
20import netaddr
21from oslo.config import cfg
22import six
23import six.moves.urllib.parse as urlparse
24import webob
25import webob.dec
26import webob.exc
27
28from ec2api.api import apirequest
29from ec2api.api import ec2utils
30from ec2api.api import faults
31from ec2api.api import validator
32from ec2api import context
33from ec2api import exception
34from ec2api.openstack.common.gettextutils import _
35from ec2api.openstack.common import jsonutils
36from ec2api.openstack.common import log as logging
37from ec2api.openstack.common import timeutils
38from ec2api import wsgi
39
40
41LOG = logging.getLogger(__name__)
42
43ec2_opts = [
44 cfg.StrOpt('keystone_url',
45 default='http://localhost:5000/v2.0',
46 help='URL to get token from ec2 request.'),
47 cfg.IntOpt('ec2_timestamp_expiry',
48 default=300,
49 help='Time in seconds before ec2 timestamp expires'),
50]
51
52CONF = cfg.CONF
53CONF.register_opts(ec2_opts)
54CONF.import_opt('use_forwarded_for', 'ec2api.api.auth')
55
56
57# Fault Wrapper around all EC2 requests #
58class FaultWrapper(wsgi.Middleware):
59 """Calls the middleware stack, captures any exceptions into faults."""
60
61 @webob.dec.wsgify(RequestClass=wsgi.Request)
62 def __call__(self, req):
63 try:
64 return req.get_response(self.application)
65 except Exception as ex:
66 LOG.exception(_("FaultWrapper: %s"), unicode(ex))
67 return faults.Fault(webob.exc.HTTPInternalServerError())
68
69
70class RequestLogging(wsgi.Middleware):
71 """Access-Log akin logging for all EC2 API requests."""
72
73 @webob.dec.wsgify(RequestClass=wsgi.Request)
74 def __call__(self, req):
75 start = timeutils.utcnow()
76 rv = req.get_response(self.application)
77 self.log_request_completion(rv, req, start)
78 return rv
79
80 def log_request_completion(self, response, request, start):
81 apireq = request.environ.get('ec2.request', None)
82 if apireq:
83 action = apireq.action
84 else:
85 action = None
86 ctxt = request.environ.get('ec2api.context', None)
87 delta = timeutils.utcnow() - start
88 seconds = delta.seconds
89 microseconds = delta.microseconds
90 LOG.info(
91 "%s.%ss %s %s %s %s %s [%s] %s %s",
92 seconds,
93 microseconds,
94 request.remote_addr,
95 request.method,
96 "%s%s" % (request.script_name, request.path_info),
97 action,
98 response.status_int,
99 request.user_agent,
100 request.content_type,
101 response.content_type,
102 context=ctxt)
103
104
105class EC2KeystoneAuth(wsgi.Middleware):
106 """Authenticate an EC2 request with keystone and convert to context."""
107
108 @webob.dec.wsgify(RequestClass=wsgi.Request)
109 def __call__(self, req):
110 request_id = context.generate_request_id()
111 signature = req.params.get('Signature')
112 if not signature:
113 msg = _("Signature not provided")
114 return faults.ec2_error_response(request_id, "AuthFailure", msg,
115 status=400)
116 access = req.params.get('AWSAccessKeyId')
117 if not access:
118 msg = _("Access key not provided")
119 return faults.ec2_error_response(request_id, "AuthFailure", msg,
120 status=400)
121
122 # Make a copy of args for authentication and signature verification.
123 auth_params = dict(req.params)
124 # Not part of authentication args
125 auth_params.pop('Signature')
126
127 cred_dict = {
128 'access': access,
129 'signature': signature,
130 'host': req.host,
131 'verb': req.method,
132 'path': req.path,
133 'params': auth_params,
134 }
135 token_url = CONF.keystone_url + "/ec2tokens"
136 if "ec2" in token_url:
137 creds = {'ec2Credentials': cred_dict}
138 else:
139 creds = {'auth': {'OS-KSEC2:ec2Credentials': cred_dict}}
140 creds_json = jsonutils.dumps(creds)
141 headers = {'Content-Type': 'application/json'}
142
143 o = urlparse.urlparse(token_url)
144 if o.scheme == "http":
145 conn = httplib.HTTPConnection(o.netloc)
146 else:
147 conn = httplib.HTTPSConnection(o.netloc)
148 conn.request('POST', o.path, body=creds_json, headers=headers)
149 response = conn.getresponse()
150 data = response.read()
151 if response.status != 200:
152 if response.status == 401:
153 msg = response.reason
154 else:
155 msg = _("Failure communicating with keystone")
156 return faults.ec2_error_response(request_id, "AuthFailure", msg,
157 status=response.status)
158 result = jsonutils.loads(data)
159 conn.close()
160
161 try:
162 token_id = result['access']['token']['id']
163 user_id = result['access']['user']['id']
164 project_id = result['access']['token']['tenant']['id']
165 user_name = result['access']['user'].get('name')
166 project_name = result['access']['token']['tenant'].get('name')
167 roles = [role['name'] for role
168 in result['access']['user']['roles']]
169 except (AttributeError, KeyError) as e:
170 LOG.exception(_("Keystone failure: %s") % e)
171 msg = _("Failure communicating with keystone")
172 return faults.ec2_error_response(request_id, "AuthFailure", msg,
173 status=400)
174
175 remote_address = req.remote_addr
176 if CONF.use_forwarded_for:
177 remote_address = req.headers.get('X-Forwarded-For',
178 remote_address)
179
180 headers["X-Auth-Token"] = token_id
181 o = urlparse.urlparse(CONF.keystone_url
182 + ("/users/%s/credentials/OS-EC2/%s" % (user_id, access)))
183 if o.scheme == "http":
184 conn = httplib.HTTPConnection(o.netloc)
185 else:
186 conn = httplib.HTTPSConnection(o.netloc)
187 conn.request('GET', o.path, headers=headers)
188 response = conn.getresponse()
189 data = response.read()
190 if response.status != 200:
191 if response.status == 401:
192 msg = response.reason
193 else:
194 msg = _("Failure communicating with keystone")
195 return faults.ec2_error_response(request_id, "AuthFailure", msg,
196 status=response.status)
197 ec2_creds = jsonutils.loads(data)
198 conn.close()
199
200 catalog = result['access']['serviceCatalog']
201 ctxt = context.RequestContext(user_id,
202 project_id,
203 ec2_creds["credential"]["access"],
204 ec2_creds["credential"]["secret"],
205 user_name=user_name,
206 project_name=project_name,
207 roles=roles,
208 auth_token=token_id,
209 remote_address=remote_address,
210 service_catalog=catalog,
211 api_version=req.params.get('Version'))
212
213 req.environ['ec2api.context'] = ctxt
214
215 return self.application
216
217
218class Requestify(wsgi.Middleware):
219
220 def __init__(self, app):
221 super(Requestify, self).__init__(app)
222
223 @webob.dec.wsgify(RequestClass=wsgi.Request)
224 def __call__(self, req):
225 non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod',
226 'SignatureVersion', 'Version', 'Timestamp']
227 args = dict(req.params)
228 try:
229 expired = ec2utils.is_ec2_timestamp_expired(req.params,
230 expires=CONF.ec2_timestamp_expiry)
231 if expired:
232 msg = _("Timestamp failed validation.")
233 LOG.exception(msg)
234 raise webob.exc.HTTPForbidden(explanation=msg)
235
236 # Raise KeyError if omitted
237 action = req.params['Action']
238 # Fix bug lp:720157 for older (version 1) clients
239 version = req.params['SignatureVersion']
240 if int(version) == 1:
241 non_args.remove('SignatureMethod')
242 if 'SignatureMethod' in args:
243 args.pop('SignatureMethod')
244 for non_arg in non_args:
245 # Remove, but raise KeyError if omitted
246 args.pop(non_arg)
247 except KeyError:
248 raise webob.exc.HTTPBadRequest()
249 except exception.InvalidRequest as err:
250 raise webob.exc.HTTPBadRequest(explanation=unicode(err))
251
252 LOG.debug('action: %s', action)
253 for key, value in args.items():
254 LOG.debug('arg: %(key)s\t\tval: %(value)s',
255 {'key': key, 'value': value})
256
257 # Success!
258 api_request = apirequest.APIRequest(
259 action, req.params['Version'], args)
260 req.environ['ec2.request'] = api_request
261 return self.application
262
263
264def validate_ec2_id(val):
265 if not validator.validate_str()(val):
266 return False
267 try:
268 ec2utils.ec2_id_to_id(val)
269 except exception.InvalidEc2Id:
270 return False
271 return True
272
273
274def is_valid_ipv4(address):
275 """Verify that address represents a valid IPv4 address."""
276 try:
277 return netaddr.valid_ipv4(address)
278 except Exception:
279 return False
280
281
282class Validator(wsgi.Middleware):
283
284 validator.validate_ec2_id = validate_ec2_id
285
286 validator.DEFAULT_VALIDATOR = {
287 'instance_id': validate_ec2_id,
288 'volume_id': validate_ec2_id,
289 'image_id': validate_ec2_id,
290 'attribute': validator.validate_str(),
291 'image_location': validator.validate_image_path,
292 'public_ip': is_valid_ipv4,
293 'region_name': validator.validate_str(),
294 'group_name': validator.validate_str(max_length=255),
295 'group_description': validator.validate_str(max_length=255),
296 'size': validator.validate_int(),
297 'user_data': validator.validate_user_data
298 }
299
300 def __init__(self, application):
301 super(Validator, self).__init__(application)
302
303 @webob.dec.wsgify(RequestClass=wsgi.Request)
304 def __call__(self, req):
305 if validator.validate(req.environ['ec2.request'].args,
306 validator.DEFAULT_VALIDATOR):
307 return self.application
308 else:
309 raise webob.exc.HTTPBadRequest()
310
311
312def exception_to_ec2code(ex):
313 """Helper to extract EC2 error code from exception.
314
315 For other than EC2 exceptions (those without ec2_code attribute),
316 use exception name.
317 """
318 if hasattr(ex, 'ec2_code'):
319 code = ex.ec2_code
320 else:
321 code = type(ex).__name__
322 return code
323
324
325def ec2_error_ex(ex, req, code=None, message=None):
326 """Return an EC2 error response based on passed exception and log it."""
327 if not code:
328 code = exception_to_ec2code(ex)
329 status = getattr(ex, 'code', None)
330 if not status:
331 status = 500
332
333 log_fun = LOG.error
334 log_msg = _("Unexpected %(ex_name)s raised: %(ex_str)s")
335
336 context = req.environ['ec2api.context']
337 request_id = context.request_id
338 log_msg_args = {
339 'ex_name': type(ex).__name__,
340 'ex_str': unicode(ex)
341 }
342 log_fun(log_msg % log_msg_args, context=context)
343
344 if ex.args and not message and status < 500:
345 message = unicode(ex.args[0])
346 # Log filtered environment for unexpected errors.
347 env = req.environ.copy()
348 for k in env.keys():
349 if not isinstance(env[k], six.string_types):
350 env.pop(k)
351 log_fun(_('Environment: %s') % jsonutils.dumps(env))
352 if not message:
353 message = _('Unknown error occurred.')
354 return faults.ec2_error_response(request_id, code, message, status=status)
355
356
357class Executor(wsgi.Application):
358
359 """Execute an EC2 API request.
360
361 Executes 'ec2.action', passing 'ec2api.context' and
362 'ec2.action_args' (all variables in WSGI environ.) Returns an XML
363 response, or a 400 upon failure.
364 """
365
366 @webob.dec.wsgify(RequestClass=wsgi.Request)
367 def __call__(self, req):
368 context = req.environ['ec2api.context']
369 api_request = req.environ['ec2.request']
370 try:
371 result = api_request.invoke(context)
372 except exception.InstanceNotFound as ex:
373 ec2_id = ec2utils.id_to_ec2_inst_id(ex.kwargs['instance_id'])
374 message = ex.msg_fmt % {'instance_id': ec2_id}
375 return ec2_error_ex(ex, req, message=message)
376 except exception.MethodNotFound:
377 try:
378 http, response = api_request.proxy(req)
379 resp = webob.Response()
380 resp.status = http["status"]
381 resp.headers["content-type"] = http["content-type"]
382 resp.body = str(response)
383 return resp
384 except Exception as ex:
385 return ec2_error_ex(ex, req)
386 except exception.EC2ServerError as ex:
387 resp = webob.Response()
388 resp.status = ex.response['status']
389 resp.headers['Content-Type'] = ex.response['content-type']
390 resp.body = ex.content
391 return resp
392 except Exception as ex:
393 return ec2_error_ex(ex, req)
394 else:
395 resp = webob.Response()
396 resp.status = 200
397 resp.headers['Content-Type'] = 'text/xml'
398 resp.body = str(result)
399
400 return resp
diff --git a/ec2api/api/apirequest.py b/ec2api/api/apirequest.py
new file mode 100644
index 0000000..75ffd02
--- /dev/null
+++ b/ec2api/api/apirequest.py
@@ -0,0 +1,143 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15"""
16APIRequest class
17"""
18
19import datetime
20# TODO(termie): replace minidom with etree
21from xml.dom import minidom
22
23from lxml import etree
24
25from ec2api.api import cloud
26from ec2api.api import ec2utils
27from ec2api.api import proxy
28from ec2api import exception
29from ec2api.openstack.common import log as logging
30
31
32LOG = logging.getLogger(__name__)
33
34
35def _underscore_to_camelcase(st):
36 return ''.join([x[:1].upper() + x[1:] for x in st.split('_')])
37
38
39def _underscore_to_xmlcase(st):
40 res = _underscore_to_camelcase(st)
41 return res[:1].lower() + res[1:]
42
43
44def _database_to_isoformat(datetimeobj):
45 """Return a xs:dateTime parsable string from datatime."""
46 return datetimeobj.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z'
47
48
49class APIRequest(object):
50 def __init__(self, action, version, args):
51 self.action = action
52 self.version = version
53 self.args = args
54 self.controller = cloud.CloudController()
55 self.proxyController = proxy.ProxyController()
56
57 def invoke(self, context):
58 try:
59 method = getattr(self.controller,
60 ec2utils.camelcase_to_underscore(self.action))
61 except AttributeError:
62 raise exception.MethodNotFound(name=self.action)
63
64 args = ec2utils.dict_from_dotted_str(self.args.items())
65
66 for key in args.keys():
67 # NOTE(vish): Turn numeric dict keys into lists
68 # NOTE(Alex): Turn "value"-only dict keys into values
69 if isinstance(args[key], dict):
70 if args[key] == {}:
71 continue
72 if args[key].keys()[0].isdigit():
73 s = args[key].items()
74 s.sort()
75 args[key] = [v for k, v in s]
76 elif args[key].keys()[0] == 'value' and len(args[key]) == 1:
77 args[key] = args[key]['value']
78
79 result = method(context, **args)
80 return self._render_response(result, context.request_id)
81
82 def proxy(self, req):
83 return self.proxyController.proxy(req, self.args)
84
85 def _render_response(self, response_data, request_id):
86 xml = minidom.Document()
87
88 response_el = xml.createElement(self.action + 'Response')
89 response_el.setAttribute('xmlns',
90 'http://ec2.amazonaws.com/doc/%s/' % self.version)
91 request_id_el = xml.createElement('requestId')
92 request_id_el.appendChild(xml.createTextNode(request_id))
93 response_el.appendChild(request_id_el)
94 if response_data is True:
95 self._render_dict(xml, response_el, {'return': 'true'})
96 else:
97 self._render_dict(xml, response_el, response_data)
98
99 xml.appendChild(response_el)
100
101 response = xml.toxml()
102 root = etree.fromstring(response)
103 response = etree.tostring(root, pretty_print=True)
104
105 xml.unlink()
106
107 # Don't write private key to log
108 if self.action != "CreateKeyPair":
109 LOG.debug(response)
110 else:
111 LOG.debug("CreateKeyPair: Return Private Key")
112
113 return response
114
115 def _render_dict(self, xml, el, data):
116 try:
117 for key in data.keys():
118 val = data[key]
119 el.appendChild(self._render_data(xml, key, val))
120 except Exception:
121 LOG.debug(data)
122 raise
123
124 def _render_data(self, xml, el_name, data):
125 el_name = _underscore_to_xmlcase(el_name)
126 data_el = xml.createElement(el_name)
127
128 if isinstance(data, list):
129 for item in data:
130 data_el.appendChild(self._render_data(xml, 'item', item))
131 elif isinstance(data, dict):
132 self._render_dict(xml, data_el, data)
133 elif hasattr(data, '__dict__'):
134 self._render_dict(xml, data_el, data.__dict__)
135 elif isinstance(data, bool):
136 data_el.appendChild(xml.createTextNode(str(data).lower()))
137 elif isinstance(data, datetime.datetime):
138 data_el.appendChild(
139 xml.createTextNode(_database_to_isoformat(data)))
140 elif data is not None:
141 data_el.appendChild(xml.createTextNode(str(data)))
142
143 return data_el
diff --git a/ec2api/api/auth.py b/ec2api/api/auth.py
new file mode 100644
index 0000000..4978dd5
--- /dev/null
+++ b/ec2api/api/auth.py
@@ -0,0 +1,54 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15"""
16Common Auth Middleware.
17
18"""
19
20from oslo.config import cfg
21
22from ec2api.openstack.common import log as logging
23
24
25auth_opts = [
26 cfg.BoolOpt('api_rate_limit',
27 default=False,
28 help='whether to use per-user rate limiting for the api.'),
29 cfg.BoolOpt('use_forwarded_for',
30 default=False,
31 help='Treat X-Forwarded-For as the canonical remote address. '
32 'Only enable this if you have a sanitizing proxy.'),
33]
34
35CONF = cfg.CONF
36CONF.register_opts(auth_opts)
37
38LOG = logging.getLogger(__name__)
39
40
41def pipeline_factory(loader, global_conf, **local_conf):
42 """A paste pipeline replica that keys off of auth_strategy."""
43 auth_strategy = "keystone"
44 pipeline = local_conf[auth_strategy]
45 if not CONF.api_rate_limit:
46 limit_name = auth_strategy + '_nolimit'
47 pipeline = local_conf.get(limit_name, pipeline)
48 pipeline = pipeline.split()
49 filters = [loader.get_filter(n) for n in pipeline[:-1]]
50 app = loader.get_app(pipeline[-1])
51 filters.reverse()
52 for fltr in filters:
53 app = fltr(app)
54 return app
diff --git a/ec2api/api/clients.py b/ec2api/api/clients.py
new file mode 100644
index 0000000..b907852
--- /dev/null
+++ b/ec2api/api/clients.py
@@ -0,0 +1,141 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15
16from keystoneclient.v2_0 import client as kc
17from novaclient import client as novaclient
18from novaclient import shell as novashell
19from oslo.config import cfg
20
21from ec2api.openstack.common.gettextutils import _
22from ec2api.openstack.common import log as logging
23
24logger = logging.getLogger(__name__)
25
26CONF = cfg.CONF
27
28
29try:
30 from neutronclient.v2_0 import client as neutronclient
31except ImportError:
32 neutronclient = None
33 logger.info(_('neutronclient not available'))
34try:
35 from cinderclient import client as cinderclient
36except ImportError:
37 cinderclient = None
38 logger.info(_('cinderclient not available'))
39try:
40 from glanceclient import client as glanceclient
41except ImportError:
42 glanceclient = None
43 logger.info(_('glanceclient not available'))
44
45
46def nova(context, service_type='compute'):
47 computeshell = novashell.OpenStackComputeShell()
48 extensions = computeshell._discover_extensions("1.1")
49
50 args = {
51 'project_id': context.project_id,
52 'auth_url': CONF.keystone_url,
53 'service_type': service_type,
54 'username': None,
55 'api_key': None,
56 'extensions': extensions,
57 }
58
59 client = novaclient.Client(1.1, **args)
60
61 management_url = _url_for(context, service_type=service_type)
62 client.client.auth_token = context.auth_token
63 client.client.management_url = management_url
64
65 return client
66
67
68def neutron(context):
69 if neutronclient is None:
70 return None
71
72 args = {
73 'auth_url': CONF.keystone_url,
74 'service_type': 'network',
75 'token': context.auth_token,
76 'endpoint_url': _url_for(context, service_type='network'),
77 }
78
79 return neutronclient.Client(**args)
80
81
82def glance(context):
83 if glanceclient is None:
84 return None
85
86 args = {
87 'auth_url': CONF.keystone_url,
88 'service_type': 'image',
89 'token': context.auth_token,
90 }
91
92 return glanceclient.Client(
93 "1", endpoint=_url_for(context, service_type='image'), **args)
94
95
96def cinder(context):
97 if cinderclient is None:
98 return nova(context, 'volume')
99
100 args = {
101 'service_type': 'volume',
102 'auth_url': CONF.keystone_url,
103 'username': None,
104 'api_key': None,
105 }
106
107 _cinder = cinderclient.Client('1', **args)
108 management_url = _url_for(context, service_type='volume')
109 _cinder.client.auth_token = context.auth_token
110 _cinder.client.management_url = management_url
111
112 return _cinder
113
114
115def keystone(context):
116 _keystone = kc.Client(
117 token=context.auth_token,
118 tenant_id=context.project_id,
119 auth_url=CONF.keystone_url)
120
121 return _keystone
122
123
124def _url_for(context, **kwargs):
125 service_catalog = context.service_catalog
126 if not service_catalog:
127 catalog = keystone(context).service_catalog.catalog
128 service_catalog = catalog["serviceCatalog"]
129 context.service_catalog = service_catalog
130
131 service_type = kwargs["service_type"]
132 for service in service_catalog:
133 if service["type"] != service_type:
134 continue
135 for endpoint in service["endpoints"]:
136 if "publicURL" in endpoint:
137 return endpoint["publicURL"]
138 else:
139 return None
140
141 return None
diff --git a/ec2api/api/cloud.py b/ec2api/api/cloud.py
new file mode 100644
index 0000000..f43e49c
--- /dev/null
+++ b/ec2api/api/cloud.py
@@ -0,0 +1,41 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15
16"""
17Cloud Controller: Implementation of EC2 REST API calls, which are
18dispatched to other nodes via AMQP RPC. State is via distributed
19datastore.
20"""
21
22from oslo.config import cfg
23
24from ec2api.openstack.common import log as logging
25
26CONF = cfg.CONF
27LOG = logging.getLogger(__name__)
28
29
30class CloudController(object):
31 """Cloud Controller
32
33 Provides the critical dispatch between
34 inbound API calls through the endpoint and messages
35 sent to the other nodes.
36 """
37 def __init__(self):
38 pass
39
40 def __str__(self):
41 return 'CloudController' \ No newline at end of file
diff --git a/ec2api/api/ec2client.py b/ec2api/api/ec2client.py
new file mode 100644
index 0000000..d57d12e
--- /dev/null
+++ b/ec2api/api/ec2client.py
@@ -0,0 +1,222 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import base64
16import hashlib
17import hmac
18import re
19import time
20import types
21import urllib
22import urlparse
23
24import httplib2
25from lxml import etree
26from oslo.config import cfg
27
28from ec2api.api import ec2utils
29from ec2api import exception
30from ec2api.openstack.common import log as logging
31
32
33ec2_opts = [
34 cfg.StrOpt('base_ec2_host',
35 default="localhost",
36 help='The IP address of the EC2 API server'),
37 cfg.IntOpt('base_ec2_port',
38 default=8773,
39 help='The port of the EC2 API server'),
40 cfg.StrOpt('base_ec2_scheme',
41 default='http',
42 help='The protocol to use when connecting to the EC2 API '
43 'server (http, https)'),
44 cfg.StrOpt('base_ec2_path',
45 default='/services/Cloud',
46 help='The path prefix used to call the ec2 API server'),
47]
48
49CONF = cfg.CONF
50CONF.register_opts(ec2_opts)
51LOG = logging.getLogger(__name__)
52
53ISO8601 = '%Y-%m-%dT%H:%M:%SZ'
54
55
56def ec2client(context):
57 return EC2Client(context)
58
59
60class EC2Requester(object):
61
62 def __init__(self, version, http_method):
63 self.http_obj = httplib2.Http(
64 disable_ssl_certificate_validation=True)
65 self.version = version
66 self.method = http_method
67
68 def request(self, context, action, args):
69 headers = {
70 'content-type': 'application/x-www-form-urlencoded',
71 'connection': 'close',
72 }
73 params = args
74 params['Action'] = action
75 params['Version'] = self.version
76 self._add_auth(context, params)
77 params = self._get_query_string(params)
78
79 if self.method == 'POST':
80 url = self._ec2_url
81 body = params
82 else:
83 url = '?'.join((self._ec2_url, params,))
84 body = None
85
86 response, content = self.http_obj.request(url, self.method,
87 body=body, headers=headers)
88 return response, content
89
90 _ec2_url = '%s://%s:%s%s' % (CONF.base_ec2_scheme,
91 CONF.base_ec2_host,
92 CONF.base_ec2_port,
93 CONF.base_ec2_path)
94
95 @staticmethod
96 def _get_query_string(params):
97 pairs = []
98 for key in sorted(params):
99 value = params[key]
100 pairs.append(urllib.quote(key.encode('utf-8'), safe='') + '=' +
101 urllib.quote(value.encode('utf-8'), safe='-_~'))
102 return '&'.join(pairs)
103
104 def _calc_signature(self, context, params):
105 LOG.debug('Calculating signature using v2 auth.')
106 split = urlparse.urlsplit(self._ec2_url)
107 path = split.path
108 if len(path) == 0:
109 path = '/'
110 string_to_sign = '%s\n%s\n%s\n' % (self.method,
111 split.netloc,
112 path)
113 secret = context.secret_key
114 lhmac = hmac.new(secret.encode('utf-8'), digestmod=hashlib.sha256)
115 string_to_sign += self._get_query_string(params)
116 LOG.debug('String to sign: %s', string_to_sign)
117 lhmac.update(string_to_sign.encode('utf-8'))
118 b64 = base64.b64encode(lhmac.digest()).strip().decode('utf-8')
119 return b64
120
121 def _add_auth(self, context, params):
122 params['AWSAccessKeyId'] = context.access_key
123 params['SignatureVersion'] = '2'
124 params['SignatureMethod'] = 'HmacSHA256'
125 params['Timestamp'] = time.strftime(ISO8601, time.gmtime())
126 signature = self._calc_signature(context, params)
127 params['Signature'] = signature
128
129
130class EC2Client(object):
131
132 def __init__(self, context):
133 self.context = context
134 self.requester = EC2Requester(context.api_version, 'POST')
135
136 def __getattr__(self, name):
137 ec2_name = self._underscore_to_camelcase(name)
138
139 def func(self, **kwargs):
140 params = self._build_params(**kwargs)
141 response, content = self.requester.request(self.context, ec2_name,
142 params)
143 return self._process_response(response, content)
144
145 func.__name__ = name
146 setattr(self, name, types.MethodType(func, self, self.__class__))
147 setattr(self.__class__, name,
148 types.MethodType(func, None, self.__class__))
149 return getattr(self, name)
150
151 @staticmethod
152 def _process_response(response, content):
153 if response.status > 200:
154 raise exception.EC2ServerError(response, content)
155
156 res = EC2Client._parse_xml(content)
157
158 res = next(res.itervalues())
159 if 'return' in res:
160 return res['return']
161 else:
162 res.pop('requestId')
163 return res
164
165 @staticmethod
166 def _build_params(**kwargs):
167 def add_list_param(params, items, label):
168 for i in range(1, len(items) + 1):
169 item = items[i - 1]
170 item_label = '%s.%d' % (label, i)
171 if isinstance(item, dict):
172 add_dict_param(params, item, item_label)
173 else:
174 params[item_label] = str(item)
175
176 def add_dict_param(params, items, label=None):
177 for key, value in items.iteritems():
178 ec2_key = EC2Client._underscore_to_camelcase(key)
179 item_label = '%s.%s' % (label, ec2_key) if label else ec2_key
180 if isinstance(value, dict):
181 add_dict_param(params, value, item_label)
182 elif isinstance(value, list):
183 add_list_param(params, value, item_label)
184 else:
185 params[item_label] = str(value)
186
187 params = {}
188 add_dict_param(params, kwargs)
189 return params
190
191 _xml_scheme = re.compile('\sxmlns=".*"')
192
193 @staticmethod
194 # NOTE(ft): this function is used in unit tests until it be moved to one
195 # of utils module
196 def _parse_xml(xml_string):
197 xml_string = EC2Client._xml_scheme.sub('', xml_string)
198 xml = etree.fromstring(xml_string)
199
200 def convert_node(node):
201 children = list(node)
202 if len(children):
203 if children[0].tag == 'item':
204 val = list(convert_node(child)[1] for child in children)
205 else:
206 val = dict(convert_node(child) for child in children)
207 elif node.tag.endswith('Set'):
208 val = []
209 else:
210 # TODO(ft): do not use private function
211 val = (ec2utils._try_convert(node.text)
212 if node.text
213 else node.text)
214 return node.tag, val
215
216 return dict([convert_node(xml)])
217
218 @staticmethod
219 # NOTE(ft): this function is copied from apirequest to avoid circular
220 # module reference. It should be moved to one of utils module
221 def _underscore_to_camelcase(st):
222 return ''.join([x[:1].upper() + x[1:] for x in st.split('_')])
diff --git a/ec2api/api/ec2utils.py b/ec2api/api/ec2utils.py
new file mode 100644
index 0000000..07340ca
--- /dev/null
+++ b/ec2api/api/ec2utils.py
@@ -0,0 +1,186 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import re
16
17from ec2api import exception
18from ec2api.openstack.common.gettextutils import _
19from ec2api.openstack.common import log as logging
20from ec2api.openstack.common import timeutils
21
22LOG = logging.getLogger(__name__)
23
24
25def resource_type_from_id(context, resource_id):
26 """Get resource type by ID
27
28 Returns a string representation of the Amazon resource type, if known.
29 Returns None on failure.
30
31 :param context: context under which the method is called
32 :param resource_id: resource_id to evaluate
33 """
34
35 known_types = {
36 'i': 'instance',
37 'r': 'reservation',
38 'vol': 'volume',
39 'snap': 'snapshot',
40 'ami': 'image',
41 'aki': 'image',
42 'ari': 'image'
43 }
44
45 type_marker = resource_id.split('-')[0]
46
47 return known_types.get(type_marker)
48
49
50_c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))')
51
52
53def camelcase_to_underscore(str):
54 return _c2u.sub(r'_\1', str).lower().strip('_')
55
56
57def _try_convert(value):
58 """Return a non-string from a string or unicode, if possible.
59
60 ============= =====================================================
61 When value is returns
62 ============= =====================================================
63 zero-length ''
64 'None' None
65 'True' True case insensitive
66 'False' False case insensitive
67 '0', '-0' 0
68 0xN, -0xN int from hex (positive) (N is any number)
69 0bN, -0bN int from binary (positive) (N is any number)
70 * try conversion to int, float, complex, fallback value
71
72 """
73 def _negative_zero(value):
74 epsilon = 1e-7
75 return 0 if abs(value) < epsilon else value
76
77 if len(value) == 0:
78 return ''
79 if value == 'None':
80 return None
81 lowered_value = value.lower()
82 if lowered_value == 'true':
83 return True
84 if lowered_value == 'false':
85 return False
86 for prefix, base in [('0x', 16), ('0b', 2), ('0', 8), ('', 10)]:
87 try:
88 if lowered_value.startswith((prefix, "-" + prefix)):
89 return int(lowered_value, base)
90 except ValueError:
91 pass
92 try:
93 return _negative_zero(float(value))
94 except ValueError:
95 return value
96
97
98def dict_from_dotted_str(items):
99 """parse multi dot-separated argument into dict.
100
101 EBS boot uses multi dot-separated arguments like
102 BlockDeviceMapping.1.DeviceName=snap-id
103 Convert the above into
104 {'block_device_mapping': {'1': {'device_name': snap-id}}}
105 """
106 args = {}
107 for key, value in items:
108 parts = key.split(".")
109 key = str(camelcase_to_underscore(parts[0]))
110 if isinstance(value, str) or isinstance(value, unicode):
111 # NOTE(vish): Automatically convert strings back
112 # into their respective values
113 value = _try_convert(value)
114
115 if len(parts) > 1:
116 d = args.get(key, {})
117 args[key] = d
118 for k in parts[1:-1]:
119 k = camelcase_to_underscore(k)
120 v = d.get(k, {})
121 d[k] = v
122 d = v
123 d[camelcase_to_underscore(parts[-1])] = value
124 else:
125 args[key] = value
126
127 return args
128
129
130_ms_time_regex = re.compile('^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3,6}Z$')
131
132
133def is_ec2_timestamp_expired(request, expires=None):
134 """Checks the timestamp or expiry time included in an EC2 request
135
136 and returns true if the request is expired
137 """
138 query_time = None
139 timestamp = request.get('Timestamp')
140 expiry_time = request.get('Expires')
141
142 def parse_strtime(strtime):
143 if _ms_time_regex.match(strtime):
144 # NOTE(MotoKen): time format for aws-sdk-java contains millisecond
145 time_format = "%Y-%m-%dT%H:%M:%S.%fZ"
146 else:
147 time_format = "%Y-%m-%dT%H:%M:%SZ"
148 return timeutils.parse_strtime(strtime, time_format)
149
150 try:
151 if timestamp and expiry_time:
152 msg = _("Request must include either Timestamp or Expires,"
153 " but cannot contain both")
154 LOG.error(msg)
155 raise exception.InvalidRequest(msg)
156 elif expiry_time:
157 query_time = parse_strtime(expiry_time)
158 return timeutils.is_older_than(query_time, -1)
159 elif timestamp:
160 query_time = parse_strtime(timestamp)
161
162 # Check if the difference between the timestamp in the request
163 # and the time on our servers is larger than 5 minutes, the
164 # request is too old (or too new).
165 if query_time and expires:
166 return (timeutils.is_older_than(query_time, expires) or
167 timeutils.is_newer_than(query_time, expires))
168 return False
169 except ValueError:
170 LOG.audit(_("Timestamp is invalid."))
171 return True
172
173
174# TODO(Alex) This function is copied as is from original cloud.py. It doesn't
175# check for the prefix which allows any prefix used for any object.
176def ec2_id_to_id(ec2_id):
177 """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)."""
178 try:
179 return int(ec2_id.split('-')[-1], 16)
180 except ValueError:
181 raise exception.InvalidEc2Id(ec2_id=ec2_id)
182
183
184def id_to_ec2_id(instance_id, template='i-%08x'):
185 """Convert an instance ID (int) to an ec2 ID (i-[base 16 number])."""
186 return template % int(instance_id)
diff --git a/ec2api/api/faults.py b/ec2api/api/faults.py
new file mode 100644
index 0000000..4fb9b67
--- /dev/null
+++ b/ec2api/api/faults.py
@@ -0,0 +1,96 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15from xml.sax import saxutils
16
17from oslo.config import cfg
18import webob.dec
19import webob.exc
20
21import ec2api.api
22from ec2api import context
23from ec2api.openstack.common import gettextutils
24from ec2api.openstack.common import log as logging
25
26CONF = cfg.CONF
27LOG = logging.getLogger(__name__)
28
29
30def xhtml_escape(value):
31 """Escapes a string so it is valid within XML or XHTML.
32
33 """
34 return saxutils.escape(value, {'"': '&quot;', "'": '&apos;'})
35
36
37def utf8(value):
38 """Try to turn a string into utf-8 if possible.
39
40 Code is directly from the utf8 function in
41 http://github.com/facebook/tornado/blob/master/tornado/escape.py
42
43 """
44 if isinstance(value, unicode):
45 return value.encode('utf-8')
46 elif isinstance(value, gettextutils.Message):
47 return unicode(value).encode('utf-8')
48 assert isinstance(value, str)
49 return value
50
51
52def ec2_error_response(request_id, code, message, status=500):
53 """Helper to construct an EC2 compatible error response."""
54 LOG.debug('EC2 error response: %(code)s: %(message)s',
55 {'code': code, 'message': message})
56 resp = webob.Response()
57 resp.status = status
58 resp.headers['Content-Type'] = 'text/xml'
59 resp.body = str('<?xml version="1.0"?>\n'
60 '<Response><Errors><Error><Code>%s</Code>'
61 '<Message>%s</Message></Error></Errors>'
62 '<RequestID>%s</RequestID></Response>' %
63 (xhtml_escape(utf8(code)),
64 xhtml_escape(utf8(message)),
65 xhtml_escape(utf8(request_id))))
66 return resp
67
68
69class Fault(webob.exc.HTTPException):
70 """Captures exception and return REST Response."""
71
72 def __init__(self, exception):
73 """Create a response for the given webob.exc.exception."""
74 self.wrapped_exc = exception
75
76 @webob.dec.wsgify
77 def __call__(self, req):
78 """Generate a WSGI response based on the exception passed to ctor."""
79 code = ec2api.api.exception_to_ec2code(self.wrapped_exc)
80 status = self.wrapped_exc.status_int
81 message = self.wrapped_exc.explanation
82
83 if status == 501:
84 message = "The requested function is not supported"
85
86 if 'AWSAccessKeyId' not in req.params:
87 raise webob.exc.HTTPBadRequest()
88 user_id, _sep, project_id = req.params['AWSAccessKeyId'].partition(':')
89 project_id = project_id or user_id
90 remote_address = getattr(req, 'remote_address', '127.0.0.1')
91 if CONF.use_forwarded_for:
92 remote_address = req.headers.get('X-Forwarded-For', remote_address)
93
94 resp = ec2_error_response(context.generate_request_id(), code,
95 message=message, status=status)
96 return resp
diff --git a/ec2api/api/proxy.py b/ec2api/api/proxy.py
new file mode 100644
index 0000000..87645e1
--- /dev/null
+++ b/ec2api/api/proxy.py
@@ -0,0 +1,27 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15from ec2api.api import ec2client
16
17
18class ProxyController(object):
19
20 def __str__(self):
21 return 'ProxyController'
22
23 def proxy(self, req, args):
24 requester = ec2client.EC2Requester(req.params["Version"],
25 req.environ["REQUEST_METHOD"])
26 return requester.request(req.environ['ec2api.context'],
27 req.params["Action"], args)
diff --git a/ec2api/api/validator.py b/ec2api/api/validator.py
new file mode 100644
index 0000000..01745fa
--- /dev/null
+++ b/ec2api/api/validator.py
@@ -0,0 +1,132 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import base64
16import re
17
18from ec2api.openstack.common.gettextutils import _
19from ec2api.openstack.common import log as logging
20
21
22LOG = logging.getLogger(__name__)
23
24
25def _get_path_validator_regex():
26 # rfc3986 path validator regex from
27 # http://jmrware.com/articles/2009/uri_regexp/URI_regex.html
28 pchar = "([A-Za-z0-9\-._~!$&'()*+,;=:@]|%[0-9A-Fa-f]{2})"
29 path = "((/{pchar}*)*|"
30 path += "/({pchar}+(/{pchar}*)*)?|"
31 path += "{pchar}+(/{pchar}*)*|"
32 path += "{pchar}+(/{pchar}*)*|)"
33 path = path.format(pchar=pchar)
34 return re.compile(path)
35
36
37VALIDATE_PATH_RE = _get_path_validator_regex()
38
39
40def validate_str(max_length=None):
41
42 def _do(val):
43 if not isinstance(val, basestring):
44 return False
45 if max_length and len(val) > max_length:
46 return False
47 return True
48
49 return _do
50
51
52def validate_int(max_value=None):
53
54 def _do(val):
55 if not isinstance(val, int):
56 return False
57 if max_value and val > max_value:
58 return False
59 return True
60
61 return _do
62
63
64def validate_url_path(val):
65 """True if val is matched by the path component grammar in rfc3986."""
66
67 if not validate_str()(val):
68 return False
69
70 return VALIDATE_PATH_RE.match(val).end() == len(val)
71
72
73def validate_image_path(val):
74 if not validate_str()(val):
75 return False
76
77 bucket_name = val.split('/')[0]
78 manifest_path = val[len(bucket_name) + 1:]
79 if not len(bucket_name) or not len(manifest_path):
80 return False
81
82 if val[0] == '/':
83 return False
84
85 # make sure the image path if rfc3986 compliant
86 # prepend '/' to make input validate
87 if not validate_url_path('/' + val):
88 return False
89
90 return True
91
92
93def validate_user_data(user_data):
94 """Check if the user_data is encoded properly."""
95 try:
96 user_data = base64.b64decode(user_data)
97 except TypeError:
98 return False
99 return True
100
101
102def validate(args, validator):
103 """Validate values of args against validators in validator.
104
105 :param args: Dict of values to be validated.
106 :param validator: A dict where the keys map to keys in args
107 and the values are validators.
108 Applies each validator to ``args[key]``
109 :returns: True if validation succeeds. Otherwise False.
110
111 A validator should be a callable which accepts 1 argument and which
112 returns True if the argument passes validation. False otherwise.
113 A validator should not raise an exception to indicate validity of the
114 argument.
115
116 Only validates keys which show up in both args and validator.
117
118 """
119
120 for key in validator:
121 if key not in args:
122 continue
123
124 f = validator[key]
125 assert callable(f)
126
127 if not f(args[key]):
128 LOG.debug(_("%(key)s with value %(value)s failed"
129 " validator %(name)s"),
130 {'key': key, 'value': args[key], 'name': f.__name__})
131 return False
132 return True
diff --git a/ec2api/cmd/__init__.py b/ec2api/cmd/__init__.py
new file mode 100644
index 0000000..e7cf12c
--- /dev/null
+++ b/ec2api/cmd/__init__.py
@@ -0,0 +1,16 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15from ec2api.openstack.common import gettextutils
16gettextutils.install('ec2api')
diff --git a/ec2api/cmd/api.py b/ec2api/cmd/api.py
new file mode 100644
index 0000000..42212e1
--- /dev/null
+++ b/ec2api/cmd/api.py
@@ -0,0 +1,42 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15"""
16EC2api API Server
17"""
18
19import sys
20
21from oslo.config import cfg
22
23from ec2api import config
24from ec2api.openstack.common import log as logging
25from ec2api import service
26
27CONF = cfg.CONF
28CONF.import_opt('use_ssl', 'ec2api.service')
29
30
31def main():
32 config.parse_args(sys.argv)
33 logging.setup('ec2api')
34
35 server = service.WSGIService(
36 'ec2api', use_ssl=CONF.use_ssl, max_url_len=16384)
37 service.serve(server)
38 service.wait()
39
40
41if __name__ == '__main__':
42 main()
diff --git a/ec2api/cmd/manage.py b/ec2api/cmd/manage.py
new file mode 100644
index 0000000..6cded38
--- /dev/null
+++ b/ec2api/cmd/manage.py
@@ -0,0 +1,75 @@
1# Copyright 2013 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15
16"""
17 CLI interface for EC2 API management.
18"""
19
20import sys
21
22from oslo.config import cfg
23
24from ec2api.db import migration
25from ec2api.openstack.common import log
26from ec2api import version
27
28
29CONF = cfg.CONF
30
31
32def do_db_version():
33 """Print database's current migration level."""
34 print(migration.db_version())
35
36
37def do_db_sync():
38 """Place a database under migration control and upgrade,
39
40 creating if necessary.
41 """
42 migration.db_sync(CONF.command.version)
43
44
45def add_command_parsers(subparsers):
46 parser = subparsers.add_parser('db_version')
47 parser.set_defaults(func=do_db_version)
48
49 parser = subparsers.add_parser('db_sync')
50 parser.set_defaults(func=do_db_sync)
51 parser.add_argument('version', nargs='?')
52 parser.add_argument('current_version', nargs='?')
53
54
55command_opt = cfg.SubCommandOpt('command',
56 title='Commands',
57 help='Available commands',
58 handler=add_command_parsers)
59
60
61def main():
62 CONF.register_cli_opt(command_opt)
63 try:
64 default_config_files = cfg.find_config_files('ec2api')
65 CONF(sys.argv[1:], project='ec2api', prog='ec2-api-manage',
66 version=version.version_info.version_string(),
67 default_config_files=default_config_files)
68 log.setup("ec2api")
69 except RuntimeError as e:
70 sys.exit("ERROR: %s" % e)
71
72 try:
73 CONF.command.func()
74 except Exception as e:
75 sys.exit("ERROR: %s" % e)
diff --git a/ec2api/config.py b/ec2api/config.py
new file mode 100644
index 0000000..a50c1ca
--- /dev/null
+++ b/ec2api/config.py
@@ -0,0 +1,30 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15from oslo.config import cfg
16
17from ec2api.openstack.common.db import options
18from ec2api import paths
19from ec2api import version
20
21_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('nova.sqlite')
22
23
24def parse_args(argv, default_config_files=None):
25 options.set_defaults(sql_connection=_DEFAULT_SQL_CONNECTION,
26 sqlite_db='nova.sqlite')
27 cfg.CONF(argv[1:],
28 project='ec2api',
29 version=version.version_info.version_string(),
30 default_config_files=default_config_files)
diff --git a/ec2api/context.py b/ec2api/context.py
new file mode 100644
index 0000000..a0953dd
--- /dev/null
+++ b/ec2api/context.py
@@ -0,0 +1,150 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15"""RequestContext: context for requests that persist through all of ec2."""
16
17import uuid
18
19import six
20
21from ec2api import exception
22from ec2api.openstack.common.gettextutils import _
23from ec2api.openstack.common import local
24from ec2api.openstack.common import log as logging
25from ec2api.openstack.common import timeutils
26
27
28LOG = logging.getLogger(__name__)
29
30
31def generate_request_id():
32 return 'req-' + str(uuid.uuid4())
33
34
35class RequestContext(object):
36 """Security context and request information.
37
38 Represents the user taking a given action within the system.
39
40 """
41
42 def __init__(self, user_id, project_id, access_key, secret_key,
43 is_admin=None, roles=None, remote_address=None,
44 auth_token=None, user_name=None, project_name=None,
45 overwrite=True, service_catalog=None, api_version=None,
46 **kwargs):
47 """Parameters
48
49 :param overwrite: Set to False to ensure that the greenthread local
50 copy of the index is not overwritten.
51
52
53 :param kwargs: Extra arguments that might be present, but we ignore
54 because they possibly came in from older rpc messages.
55 """
56 if kwargs:
57 LOG.warn(_('Arguments dropped when creating context: %s') %
58 str(kwargs))
59
60 self.user_id = user_id
61 self.project_id = project_id
62 self.access_key = access_key
63 self.secret_key = secret_key
64 self.roles = roles or []
65 self.remote_address = remote_address
66 timestamp = timeutils.utcnow()
67 if isinstance(timestamp, six.string_types):
68 timestamp = timeutils.parse_strtime(timestamp)
69 self.timestamp = timestamp
70 self.request_id = generate_request_id()
71 self.auth_token = auth_token
72
73 self.service_catalog = service_catalog
74 if self.service_catalog is None:
75 # if list is empty or none
76 self.service_catalog = []
77
78 self.user_name = user_name
79 self.project_name = project_name
80 self.is_admin = is_admin
81 self.api_version = api_version
82 if overwrite or not hasattr(local.store, 'context'):
83 self.update_store()
84
85 def update_store(self):
86 local.store.context = self
87
88 def to_dict(self):
89 return {'user_id': self.user_id,
90 'project_id': self.project_id,
91 'is_admin': self.is_admin,
92 'roles': self.roles,
93 'remote_address': self.remote_address,
94 'timestamp': timeutils.strtime(self.timestamp),
95 'request_id': self.request_id,
96 'auth_token': self.auth_token,
97 'user_name': self.user_name,
98 'service_catalog': self.service_catalog,
99 'project_name': self.project_name,
100 'tenant': self.tenant,
101 'user': self.user}
102
103 @classmethod
104 def from_dict(cls, values):
105 values.pop('user', None)
106 values.pop('tenant', None)
107 return cls(**values)
108
109 # NOTE(sirp): the openstack/common version of RequestContext uses
110 # tenant/user whereas the ec2 version uses project_id/user_id. We need
111 # this shim in order to use context-aware code from openstack/common, like
112 # logging, until we make the switch to using openstack/common's version of
113 # RequestContext.
114 @property
115 def tenant(self):
116 return self.project_id
117
118 @property
119 def user(self):
120 return self.user_id
121
122
123def get_admin_context(read_deleted="no"):
124 return RequestContext(user_id=None,
125 project_id=None,
126 access_key=None,
127 secret_key=None,
128 is_admin=True,
129 read_deleted=read_deleted,
130 overwrite=False)
131
132
133def is_user_context(context):
134 """Indicates if the request context is a normal user."""
135 if not context:
136 return False
137 if context.is_admin:
138 return False
139 if not context.user_id or not context.project_id:
140 return False
141 return True
142
143
144def require_context(ctxt):
145 """Raise exception.Forbidden()
146
147 if context is not a user or an admin context.
148 """
149 if not ctxt.is_admin and not is_user_context(ctxt):
150 raise exception.Forbidden()
diff --git a/ec2api/exception.py b/ec2api/exception.py
new file mode 100644
index 0000000..0db93df
--- /dev/null
+++ b/ec2api/exception.py
@@ -0,0 +1,279 @@
1# Copyright 2014 Cloudscaling Group, Inc
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15"""ec2api base exception handling.
16
17Includes decorator for re-raising ec2api-type exceptions.
18
19SHOULD include dedicated exception logging.
20
21"""
22
23import sys
24
25from oslo.config import cfg
26
27from ec2api.openstack.common.gettextutils import _
28from ec2api.openstack.common import log as logging
29
30LOG = logging.getLogger(__name__)
31
32exc_log_opts = [
33 cfg.BoolOpt('fatal_exception_format_errors',
34 default=False,
35 help='Make exception message format errors fatal'),
36]
37
38CONF = cfg.CONF
39CONF.register_opts(exc_log_opts)
40
41
42class EC2ServerError(Exception):
43
44 def __init__(self, response, content):
45 self.response = response
46 self.content = content
47
48
49class EC2Exception(Exception):
50
51 """Base EC2 Exception
52
53 To correctly use this class, inherit from it and define
54 a 'msg_fmt' property. That msg_fmt will get printf'd
55 with the keyword arguments provided to the constructor.
56
57 """
58 msg_fmt = _("An unknown exception occurred.")
59 code = 500
60 headers = {}
61 safe = False
62
63 def __init__(self, message=None, **kwargs):
64 self.kwargs = kwargs
65
66 if 'code' not in self.kwargs:
67 try:
68 self.kwargs['code'] = self.code
69 except AttributeError:
70 pass
71
72 if not message:
73 try:
74 message = self.msg_fmt % kwargs
75
76 except Exception:
77 exc_info = sys.exc_info()
78 # kwargs doesn't match a variable in the message
79 # log the issue and the kwargs
80 LOG.exception(_('Exception in string format operation'))
81 for name, value in kwargs.iteritems():
82 LOG.error("%s: %s" % (name, value))
83
84 if CONF.fatal_exception_format_errors:
85 raise exc_info[0], exc_info[1], exc_info[2]
86 else:
87 # at least get the core message out if something happened
88 message = self.msg_fmt
89
90 super(EC2Exception, self).__init__(message)
91
92 def format_message(self):
93 # NOTE(mrodden): use the first argument to the python Exception object
94 # which should be our full EC2Exception message, (see __init__)
95 return self.args[0]
96
97
98class Invalid(EC2Exception):
99 msg_fmt = _("Unacceptable parameters.")
100 code = 400
101
102
103class InvalidRequest(Invalid):
104 msg_fmt = _("The request is invalid.")
105
106
107class InvalidEc2Id(Invalid):
108 msg_fmt = _("Ec2 id %(ec2_id)s is unacceptable.")
109
110
111class InvalidInput(Invalid):
112 msg_fmt = _("Invalid input received: %(reason)s")
113
114
115class ConfigNotFound(EC2Exception):
116 msg_fmt = _("Could not find config at %(path)s")
117
118
119class PasteAppNotFound(EC2Exception):
120 msg_fmt = _("Could not load paste app '%(name)s' from %(path)s")
121
122
123class MethodNotFound(EC2Exception):
124 msg_fmt = _("Could not find method '%(name)s'")
125
126
127class Forbidden(EC2Exception):
128 ec2_code = 'AuthFailure'
129 msg_fmt = _("Not authorized.")
130 code = 403
131
132
133class AuthFailure(Invalid):
134 pass
135
136
137class NotFound(EC2Exception):
138 msg_fmt = _("Resource could not be found.")
139 code = 404
140
141
142class EC2NotFound(NotFound):
143 code = 400
144
145
146class InstanceNotFound(EC2NotFound):
147 ec2_code = 'InvalidInstanceID.NotFound'
148 msg_fmt = _("Instance %(instance_id)s could not be found.")
149
150
151class InvalidVpcIDNotFound(EC2NotFound):
152 ec2_code = 'InvalidVpcID.NotFound'
153 msg_fmt = _("The vpc ID '%(vpc_id)s' does not exist")
154
155
156class InvalidInternetGatewayIDNotFound(EC2NotFound):
157 ec2_code = 'InvalidInternetGatewayID.NotFound'
158 msg_fmt = _("The internetGateway ID '%(igw_id)s' does not exist")
159
160
161class InvalidSubnetIDNotFound(EC2NotFound):
162 ec2_code = 'InvalidSubnetID.NotFound'
163 msg_fmt = _("The subnet ID '%(subnet_id)s' does not exist")
164
165
166class InvalidNetworkInterfaceIDNotFound(EC2NotFound):
167 ec2_code = 'InvalidNetworkInterfaceID.NotFound'
168 msg_fmt = _("Network interface %(eni_id)s could not "
169 "be found.")
170
171
172class InvalidAttachmentIDNotFound(EC2NotFound):
173 ec2_code = 'InvalidAttachmentID.NotFound'
174 msg_fmt = _("Attachment %(eni-attach_id)s could not "
175 "be found.")
176
177
178class InvalidDhcpOptionsIDNotFound(EC2NotFound):
179 ec2_code = 'InvalidDhcpOptionsID.NotFound'
180 msg_fmt = _("The dhcp options ID '%(dopt_id)s' does not exist")
181
182
183class InvalidAllocationIDNotFound(EC2NotFound):
184 ec2_code = 'InvalidAllocationID.NotFound'
185 msg_fmt = _("The allocation ID '%(eipalloc_id)s' does not exist")
186
187
188class InvalidAssociationIDNotFound(EC2NotFound):
189 ec2_code = 'InvalidAssociationID.NotFound'
190 msg_fmt = _("The association ID '%(assoc_id)s' does not exist")
191
192
193class InvalidRouteTableIDNotFound(EC2NotFound):
194 ec2_code = 'InvalidRouteTableID.NotFound'
195 msg_fmt = _("The routeTable ID '%(rtb_id)s' does not exist")
196
197
198class InvalidRouteNotFound(EC2NotFound):
199 ec2_code = 'InvalidRoute.NotFound'
200 msg_fmt = _('No route with destination-cidr-block '
201 '%(destination_cidr_block)s in route table %(route_table_id)s')
202
203
204class InvalidGroupNotFound(EC2NotFound):
205 ec2_code = 'InvalidGroup.NotFound'
206 msg_fmg = _("The security group ID '%(sg_id)s' does not exist")
207
208
209class InvalidPermissionNotFound(EC2NotFound):
210 ec2_code = 'InvalidPermission.NotFound'
211 msg_fmg = _("The specified permission does not exist")
212
213
214class IncorrectState(EC2Exception):
215 ec2_code = 'IncorrectState'
216 code = 400
217 msg_fmt = _("The resource is in incorrect state for the request - reason: "
218 "'%(reason)s'")
219
220
221class InvalidVpcRange(Invalid):
222 ec2_code = 'InvalidVpc.Range'
223 msg_fmt = _("The CIDR '%(cidr_block)s' is invalid.")
224
225
226class InvalidSubnetRange(Invalid):
227 ec2_code = 'InvalidSubnet.Range'
228 msg_fmt = _("The CIDR '%(cidr_block)s' is invalid.")
229
230
231class InvalidSubnetConflict(Invalid):
232 ec2_code = 'InvalidSubnet.Conflict'
233 msg_fmt = _("The CIDR '%(cidr_block)s' conflicts with another subnet")
234
235
236class MissingParameter(Invalid):
237 pass
238
239
240class InvalidParameterValue(Invalid):
241 msg_fmt = _("Value (%(value)s) for parameter %(parameter)s is invalid. "
242 "%(reason)s")
243
244
245class InvalidParameterCombination(Invalid):
246 pass
247
248
249class ResourceAlreadyAssociated(Invalid):
250 ec2_code = 'Resource.AlreadyAssociated'
251
252
253class GatewayNotAttached(Invalid):
254 ec2_code = 'Gateway.NotAttached'
255 msg_fmt = _("resource %(igw_id)s is not attached to network %(vpc_id)s")
256
257
258class DependencyViolation(Invalid):
259 ec2_code = 'DependencyViolation'
260 msg_fmt = _('Object %(obj1_id)s has dependent resource %(obj2_id)s')
261
262
263class InvalidNetworkInterfaceInUse(Invalid):
264 ec2_code = 'InvalidNetworkInterface.InUse'
265 msg_fmt = _('Interface: %(interface_ids)s in use.')
266
267
268class InvalidInstanceId(Invalid):
269 ec2_code = 'InvalidInstanceID'
270
271
272class InvalidIPAddressInUse(Invalid):
273 ec2_code = 'InvalidIPAddress.InUse'
274 msg_fmt = _('Address %(ip_address)s is in use.')
275
276
277class RouteAlreadyExists(Invalid):
278 msg_fmt = _('The route identified by %(destination_cidr_block)s '
279 'already exists.')
diff --git a/ec2api/openstack/__init__.py b/ec2api/openstack/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/ec2api/openstack/__init__.py
diff --git a/ec2api/openstack/common/__init__.py b/ec2api/openstack/common/__init__.py
new file mode 100644
index 0000000..d1223ea
--- /dev/null
+++ b/ec2api/openstack/common/__init__.py
@@ -0,0 +1,17 @@
1#
2# Licensed under the Apache License, Version 2.0 (the "License"); you may
3# not use this file except in compliance with the License. You may obtain
4# a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7#
8# Unless required by applicable law or agreed to in writing, software
9# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
10# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
11# License for the specific language governing permissions and limitations
12# under the License.
13
14import six
15
16
17six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))
diff --git a/ec2api/openstack/common/context.py b/ec2api/openstack/common/context.py
new file mode 100644
index 0000000..b612db7
--- /dev/null
+++ b/ec2api/openstack/common/context.py
@@ -0,0 +1,126 @@
1# Copyright 2011 OpenStack Foundation.
2# All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16"""
17Simple class that stores security context information in the web request.
18
19Projects should subclass this class if they wish to enhance the request
20context or provide additional information in their specific WSGI pipeline.
21"""
22
23import itertools
24import uuid
25
26
27def generate_request_id():
28 return b'req-' + str(uuid.uuid4()).encode('ascii')
29
30
31class RequestContext(object):
32
33 """Helper class to represent useful information about a request context.
34
35 Stores information about the security context under which the user
36 accesses the system, as well as additional request information.
37 """
38
39 user_idt_format = '{user} {tenant} {domain} {user_domain} {p_domain}'
40
41 def __init__(self, auth_token=None, user=None, tenant=None, domain=None,
42 user_domain=None, project_domain=None, is_admin=False,
43 read_only=False, show_deleted=False, request_id=None,
44 instance_uuid=None):
45 self.auth_token = auth_token
46 self.user = user
47 self.tenant = tenant
48 self.domain = domain
49 self.user_domain = user_domain
50 self.project_domain = project_domain
51 self.is_admin = is_admin
52 self.read_only = read_only
53 self.show_deleted = show_deleted
54 self.instance_uuid = instance_uuid
55 if not request_id:
56 request_id = generate_request_id()
57 self.request_id = request_id
58
59 def to_dict(self):
60 user_idt = (
61 self.user_idt_format.format(user=self.user or '-',
62 tenant=self.tenant or '-',
63 domain=self.domain or '-',
64 user_domain=self.user_domain or '-',
65 p_domain=self.project_domain or '-'))
66
67 return {'user': self.user,
68 'tenant': self.tenant,
69 'domain': self.domain,
70 'user_domain': self.user_domain,
71 'project_domain': self.project_domain,
72 'is_admin': self.is_admin,
73 'read_only': self.read_only,
74 'show_deleted': self.show_deleted,
75 'auth_token': self.auth_token,
76 'request_id': self.request_id,
77 'instance_uuid': self.instance_uuid,
78 'user_identity': user_idt}
79
80 @classmethod
81 def from_dict(cls, ctx):
82 return cls(
83 auth_token=ctx.get("auth_token"),
84 user=ctx.get("user"),
85 tenant=ctx.get("tenant"),
86 domain=ctx.get("domain"),
87 user_domain=ctx.get("user_domain"),
88 project_domain=ctx.get("project_domain"),
89 is_admin=ctx.get("is_admin", False),
90 read_only=ctx.get("read_only", False),
91 show_deleted=ctx.get("show_deleted", False),
92 request_id=ctx.get("request_id"),
93 instance_uuid=ctx.get("instance_uuid"))
94
95
96def get_admin_context(show_deleted=False):
97 context = RequestContext(None,
98 tenant=None,
99 is_admin=True,
100 show_deleted=show_deleted)
101 return context
102
103
104def get_context_from_function_and_args(function, args, kwargs):
105 """Find an arg of type RequestContext and return it.
106
107 This is useful in a couple of decorators where we don't
108 know much about the function we're wrapping.
109 """
110
111 for arg in itertools.chain(kwargs.values(), args):
112 if isinstance(arg, RequestContext):
113 return arg
114
115 return None
116
117
118def is_user_context(context):
119 """Indicates if the request context is a normal user."""
120 if not context:
121 return False
122 if context.is_admin:
123 return False
124 if not context.user_id or not context.project_id:
125 return False
126 return True
diff --git a/ec2api/openstack/common/db/__init__.py b/ec2api/openstack/common/db/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/ec2api/openstack/common/db/__init__.py
diff --git a/ec2api/openstack/common/db/api.py b/ec2api/openstack/common/db/api.py
new file mode 100644
index 0000000..7c56ccc
--- /dev/null
+++ b/ec2api/openstack/common/db/api.py
@@ -0,0 +1,162 @@
1# Copyright (c) 2013 Rackspace Hosting
2# All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16"""Multiple DB API backend support.
17
18A DB backend module should implement a method named 'get_backend' which
19takes no arguments. The method can return any object that implements DB
20API methods.
21"""
22
23import functools
24import logging
25import threading
26import time
27
28from ec2api.openstack.common.db import exception
29from ec2api.openstack.common.gettextutils import _LE
30from ec2api.openstack.common import importutils
31
32
33LOG = logging.getLogger(__name__)
34
35
36def safe_for_db_retry(f):
37 """Enable db-retry for decorated function, if config option enabled."""
38 f.__dict__['enable_retry'] = True
39 return f
40
41
42class wrap_db_retry(object):
43 """Retry db.api methods, if DBConnectionError() raised
44
45 Retry decorated db.api methods. If we enabled `use_db_reconnect`
46 in config, this decorator will be applied to all db.api functions,
47 marked with @safe_for_db_retry decorator.
48 Decorator catchs DBConnectionError() and retries function in a
49 loop until it succeeds, or until maximum retries count will be reached.
50 """
51
52 def __init__(self, retry_interval, max_retries, inc_retry_interval,
53 max_retry_interval):
54 super(wrap_db_retry, self).__init__()
55
56 self.retry_interval = retry_interval
57 self.max_retries = max_retries
58 self.inc_retry_interval = inc_retry_interval
59 self.max_retry_interval = max_retry_interval
60
61 def __call__(self, f):
62 @functools.wraps(f)
63 def wrapper(*args, **kwargs):
64 next_interval = self.retry_interval
65 remaining = self.max_retries
66
67 while True:
68 try:
69 return f(*args, **kwargs)
70 except exception.DBConnectionError as e:
71 if remaining == 0:
72 LOG.exception(_LE('DB exceeded retry limit.'))
73 raise exception.DBError(e)
74 if remaining != -1:
75 remaining -= 1
76 LOG.exception(_LE('DB connection error.'))
77 # NOTE(vsergeyev): We are using patched time module, so
78 # this effectively yields the execution
79 # context to another green thread.
80 time.sleep(next_interval)
81 if self.inc_retry_interval:
82 next_interval = min(
83 next_interval * 2,
84 self.max_retry_interval
85 )
86 return wrapper
87
88
89class DBAPI(object):
90 def __init__(self, backend_name, backend_mapping=None, lazy=False,
91 **kwargs):
92 """Initialize the chosen DB API backend.
93
94 :param backend_name: name of the backend to load
95 :type backend_name: str
96
97 :param backend_mapping: backend name -> module/class to load mapping
98 :type backend_mapping: dict
99
100 :param lazy: load the DB backend lazily on the first DB API method call
101 :type lazy: bool
102
103 Keyword arguments:
104
105 :keyword use_db_reconnect: retry DB transactions on disconnect or not
106 :type use_db_reconnect: bool
107
108 :keyword retry_interval: seconds between transaction retries
109 :type retry_interval: int
110
111 :keyword inc_retry_interval: increase retry interval or not
112 :type inc_retry_interval: bool
113
114 :keyword max_retry_interval: max interval value between retries
115 :type max_retry_interval: int
116
117 :keyword max_retries: max number of retries before an error is raised
118 :type max_retries: int
119
120 """
121
122 self._backend = None
123 self._backend_name = backend_name
124 self._backend_mapping = backend_mapping or {}
125 self._lock = threading.Lock()
126
127 if not lazy:
128 self._load_backend()
129
130 self.use_db_reconnect = kwargs.get('use_db_reconnect', False)
131 self.retry_interval = kwargs.get('retry_interval', 1)
132 self.inc_retry_interval = kwargs.get('inc_retry_interval', True)
133 self.max_retry_interval = kwargs.get('max_retry_interval', 10)
134 self.max_retries = kwargs.get('max_retries', 20)
135
136 def _load_backend(self):
137 with self._lock:
138 if not self._backend:
139 # Import the untranslated name if we don't have a mapping
140 backend_path = self._backend_mapping.get(self._backend_name,
141 self._backend_name)
142 backend_mod = importutils.import_module(backend_path)
143 self._backend = backend_mod.get_backend()
144
145 def __getattr__(self, key):
146 if not self._backend:
147 self._load_backend()
148
149 attr = getattr(self._backend, key)
150 if not hasattr(attr, '__call__'):
151 return attr
152 # NOTE(vsergeyev): If `use_db_reconnect` option is set to True, retry
153 # DB API methods, decorated with @safe_for_db_retry
154 # on disconnect.
155 if self.use_db_reconnect and hasattr(attr, 'enable_retry'):
156 attr = wrap_db_retry(
157 retry_interval=self.retry_interval,
158 max_retries=self.max_retries,
159 inc_retry_interval=self.inc_retry_interval,
160 max_retry_interval=self.max_retry_interval)(attr)
161
162 return attr
diff --git a/ec2api/openstack/common/db/exception.py b/ec2api/openstack/common/db/exception.py
new file mode 100644
index 0000000..28fdb38
--- /dev/null
+++ b/ec2api/openstack/common/db/exception.py
@@ -0,0 +1,56 @@
1# Copyright 2010 United States Government as represented by the
2# Administrator of the National Aeronautics and Space Administration.
3# All Rights Reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may
6# not use this file except in compliance with the License. You may obtain
7# a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations
15# under the License.
16
17"""DB related custom exceptions."""
18
19import six
20
21from ec2api.openstack.common.gettextutils import _
22
23
24class DBError(Exception):
25 """Wraps an implementation specific exception."""
26 def __init__(self, inner_exception=None):
27 self.inner_exception = inner_exception
28 super(DBError, self).__init__(six.text_type(inner_exception))
29
30
31class DBDuplicateEntry(DBError):
32 """Wraps an implementation specific exception."""
33 def __init__(self, columns=[], inner_exception=None):
34 self.columns = columns
35 super(DBDuplicateEntry, self).__init__(inner_exception)
36
37
38class DBDeadlock(DBError):
39 def __init__(self, inner_exception=None):
40 super(DBDeadlock, self).__init__(inner_exception)
41
42
43class DBInvalidUnicodeParameter(Exception):
44 message = _("Invalid Parameter: "
45 "Unicode is not supported by the current database.")
46
47
48class DbMigrationError(DBError):
49 """Wraps migration specific exception."""
50 def __init__(self, message=None):
51 super(DbMigrationError, self).__init__(message)
52
53
54class DBConnectionError(DBError):
55 """Wraps connection specific exception."""
56 pass
diff --git a/ec2api/openstack/common/db/options.py b/ec2api/openstack/common/db/options.py
new file mode 100644
index 0000000..df5daa0
--- /dev/null
+++ b/ec2api/openstack/common/db/options.py
@@ -0,0 +1,171 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13import copy
14
15from oslo.config import cfg
16
17
18database_opts = [
19 cfg.StrOpt('sqlite_db',
20 deprecated_group='DEFAULT',
21 default='ec2api.sqlite',
22 help='The file name to use with SQLite'),
23 cfg.BoolOpt('sqlite_synchronous',
24 deprecated_group='DEFAULT',
25 default=True,
26 help='If True, SQLite uses synchronous mode'),
27 cfg.StrOpt('backend',
28 default='sqlalchemy',
29 deprecated_name='db_backend',
30 deprecated_group='DEFAULT',
31 help='The backend to use for db'),
32 cfg.StrOpt('connection',
33 help='The SQLAlchemy connection string used to connect to the '
34 'database',
35 secret=True,
36 deprecated_opts=[cfg.DeprecatedOpt('sql_connection',
37 group='DEFAULT'),
38 cfg.DeprecatedOpt('sql_connection',
39 group='DATABASE'),
40 cfg.DeprecatedOpt('connection',
41 group='sql'), ]),
42 cfg.StrOpt('mysql_sql_mode',
43 default='TRADITIONAL',
44 help='The SQL mode to be used for MySQL sessions. '
45 'This option, including the default, overrides any '
46 'server-set SQL mode. To use whatever SQL mode '
47 'is set by the server configuration, '
48 'set this to no value. Example: mysql_sql_mode='),
49 cfg.IntOpt('idle_timeout',
50 default=3600,
51 deprecated_opts=[cfg.DeprecatedOpt('sql_idle_timeout',
52 group='DEFAULT'),
53 cfg.DeprecatedOpt('sql_idle_timeout',
54 group='DATABASE'),
55 cfg.DeprecatedOpt('idle_timeout',
56 group='sql')],
57 help='Timeout before idle sql connections are reaped'),
58 cfg.IntOpt('min_pool_size',
59 default=1,
60 deprecated_opts=[cfg.DeprecatedOpt('sql_min_pool_size',
61 group='DEFAULT'),
62 cfg.DeprecatedOpt('sql_min_pool_size',
63 group='DATABASE')],
64 help='Minimum number of SQL connections to keep open in a '
65 'pool'),
66 cfg.IntOpt('max_pool_size',
67 default=None,
68 deprecated_opts=[cfg.DeprecatedOpt('sql_max_pool_size',
69 group='DEFAULT'),
70 cfg.DeprecatedOpt('sql_max_pool_size',
71 group='DATABASE')],
72 help='Maximum number of SQL connections to keep open in a '
73 'pool'),
74 cfg.IntOpt('max_retries',
75 default=10,
76 deprecated_opts=[cfg.DeprecatedOpt('sql_max_retries',
77 group='DEFAULT'),
78 cfg.DeprecatedOpt('sql_max_retries',
79 group='DATABASE')],
80 help='Maximum db connection retries during startup. '
81 '(setting -1 implies an infinite retry count)'),
82 cfg.IntOpt('retry_interval',
83 default=10,
84 deprecated_opts=[cfg.DeprecatedOpt('sql_retry_interval',
85 group='DEFAULT'),
86 cfg.DeprecatedOpt('reconnect_interval',
87 group='DATABASE')],
88 help='Interval between retries of opening a sql connection'),
89 cfg.IntOpt('max_overflow',
90 default=None,
91 deprecated_opts=[cfg.DeprecatedOpt('sql_max_overflow',
92 group='DEFAULT'),
93 cfg.DeprecatedOpt('sqlalchemy_max_overflow',
94 group='DATABASE')],
95 help='If set, use this value for max_overflow with sqlalchemy'),
96 cfg.IntOpt('connection_debug',
97 default=0,
98 deprecated_opts=[cfg.DeprecatedOpt('sql_connection_debug',
99 group='DEFAULT')],
100 help='Verbosity of SQL debugging information. 0=None, '
101 '100=Everything'),
102 cfg.BoolOpt('connection_trace',
103 default=False,
104 deprecated_opts=[cfg.DeprecatedOpt('sql_connection_trace',
105 group='DEFAULT')],
106 help='Add python stack traces to SQL as comment strings'),
107 cfg.IntOpt('pool_timeout',
108 default=None,
109 deprecated_opts=[cfg.DeprecatedOpt('sqlalchemy_pool_timeout',
110 group='DATABASE')],
111 help='If set, use this value for pool_timeout with sqlalchemy'),
112 cfg.BoolOpt('use_db_reconnect',
113 default=False,
114 help='Enable the experimental use of database reconnect '
115 'on connection lost'),
116 cfg.IntOpt('db_retry_interval',
117 default=1,
118 help='seconds between db connection retries'),
119 cfg.BoolOpt('db_inc_retry_interval',
120 default=True,
121 help='Whether to increase interval between db connection '
122 'retries, up to db_max_retry_interval'),
123 cfg.IntOpt('db_max_retry_interval',
124 default=10,
125 help='max seconds between db connection retries, if '
126 'db_inc_retry_interval is enabled'),
127 cfg.IntOpt('db_max_retries',
128 default=20,
129 help='maximum db connection retries before error is raised. '
130 '(setting -1 implies an infinite retry count)'),
131]
132
133CONF = cfg.CONF
134CONF.register_opts(database_opts, 'database')
135
136
137def set_defaults(sql_connection, sqlite_db, max_pool_size=None,
138 max_overflow=None, pool_timeout=None):
139 """Set defaults for configuration variables."""
140 cfg.set_defaults(database_opts,
141 connection=sql_connection,
142 sqlite_db=sqlite_db)
143 # Update the QueuePool defaults
144 if max_pool_size is not None:
145 cfg.set_defaults(database_opts,
146 max_pool_size=max_pool_size)
147 if max_overflow is not None:
148 cfg.set_defaults(database_opts,
149 max_overflow=max_overflow)
150 if pool_timeout is not None:
151 cfg.set_defaults(database_opts,
152 pool_timeout=pool_timeout)
153
154
155def list_opts():
156 """Returns a list of oslo.config options available in the library.
157
158 The returned list includes all oslo.config options which may be registered
159 at runtime by the library.
160
161 Each element of the list is a tuple. The first element is the name of the
162 group under which the list of elements in the second element will be
163 registered. A group name of None corresponds to the [DEFAULT] group in
164 config files.
165
166 The purpose of this is to allow tools like the Oslo sample config file
167 generator to discover the options exposed to users by this library.
168
169 :returns: a list of (group_name, opts) tuples
170 """
171 return [('database', copy.deepcopy(database_opts))]
diff --git a/ec2api/openstack/common/db/sqlalchemy/__init__.py b/ec2api/openstack/common/db/sqlalchemy/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/ec2api/openstack/common/db/sqlalchemy/__init__.py
diff --git a/ec2api/openstack/common/db/sqlalchemy/migration.py b/ec2api/openstack/common/db/sqlalchemy/migration.py
new file mode 100644
index 0000000..4c44fb8
--- /dev/null
+++ b/ec2api/openstack/common/db/sqlalchemy/migration.py
@@ -0,0 +1,278 @@
1# coding: utf-8
2#
3# Copyright (c) 2013 OpenStack Foundation
4# All Rights Reserved.
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17#
18# Base on code in migrate/changeset/databases/sqlite.py which is under
19# the following license:
20#
21# The MIT License
22#
23# Copyright (c) 2009 Evan Rosson, Jan Dittberner, Domen Ko┼żar
24#
25# Permission is hereby granted, free of charge, to any person obtaining a copy
26# of this software and associated documentation files (the "Software"), to deal
27# in the Software without restriction, including without limitation the rights
28# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
29# copies of the Software, and to permit persons to whom the Software is
30# furnished to do so, subject to the following conditions:
31# The above copyright notice and this permission notice shall be included in
32# all copies or substantial portions of the Software.
33#
34# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
35# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
36# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
37# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
38# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
39# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
40# THE SOFTWARE.
41
42import os
43import re
44
45from migrate.changeset import ansisql
46from migrate.changeset.databases import sqlite
47from migrate import exceptions as versioning_exceptions
48from migrate.versioning import api as versioning_api
49from migrate.versioning.repository import Repository
50import sqlalchemy
51from sqlalchemy.schema import UniqueConstraint
52
53from ec2api.openstack.common.db import exception
54from ec2api.openstack.common.gettextutils import _
55
56
57def _get_unique_constraints(self, table):
58 """Retrieve information about existing unique constraints of the table
59
60 This feature is needed for _recreate_table() to work properly.
61 Unfortunately, it's not available in sqlalchemy 0.7.x/0.8.x.
62
63 """
64
65 data = table.metadata.bind.execute(
66 """SELECT sql
67 FROM sqlite_master
68 WHERE
69 type='table' AND
70 name=:table_name""",
71 table_name=table.name
72 ).fetchone()[0]
73
74 UNIQUE_PATTERN = "CONSTRAINT (\w+) UNIQUE \(([^\)]+)\)"
75 return [
76 UniqueConstraint(
77 *[getattr(table.columns, c.strip(' "')) for c in cols.split(",")],
78 name=name
79 )
80 for name, cols in re.findall(UNIQUE_PATTERN, data)
81 ]
82
83
84def _recreate_table(self, table, column=None, delta=None, omit_uniques=None):
85 """Recreate the table properly
86
87 Unlike the corresponding original method of sqlalchemy-migrate this one
88 doesn't drop existing unique constraints when creating a new one.
89
90 """
91
92 table_name = self.preparer.format_table(table)
93
94 # we remove all indexes so as not to have
95 # problems during copy and re-create
96 for index in table.indexes:
97 index.drop()
98
99 # reflect existing unique constraints
100 for uc in self._get_unique_constraints(table):
101 table.append_constraint(uc)
102 # omit given unique constraints when creating a new table if required
103 table.constraints = set([
104 cons for cons in table.constraints
105 if omit_uniques is None or cons.name not in omit_uniques
106 ])
107
108 self.append('ALTER TABLE %s RENAME TO migration_tmp' % table_name)
109 self.execute()
110
111 insertion_string = self._modify_table(table, column, delta)
112
113 table.create(bind=self.connection)
114 self.append(insertion_string % {'table_name': table_name})
115 self.execute()
116 self.append('DROP TABLE migration_tmp')
117 self.execute()
118
119
120def _visit_migrate_unique_constraint(self, *p, **k):
121 """Drop the given unique constraint
122
123 The corresponding original method of sqlalchemy-migrate just
124 raises NotImplemented error
125
126 """
127
128 self.recreate_table(p[0].table, omit_uniques=[p[0].name])
129
130
131def patch_migrate():
132 """A workaround for SQLite's inability to alter things
133
134 SQLite abilities to alter tables are very limited (please read
135 http://www.sqlite.org/lang_altertable.html for more details).
136 E. g. one can't drop a column or a constraint in SQLite. The
137 workaround for this is to recreate the original table omitting
138 the corresponding constraint (or column).
139
140 sqlalchemy-migrate library has recreate_table() method that
141 implements this workaround, but it does it wrong:
142
143 - information about unique constraints of a table
144 is not retrieved. So if you have a table with one
145 unique constraint and a migration adding another one
146 you will end up with a table that has only the
147 latter unique constraint, and the former will be lost
148
149 - dropping of unique constraints is not supported at all
150
151 The proper way to fix this is to provide a pull-request to
152 sqlalchemy-migrate, but the project seems to be dead. So we
153 can go on with monkey-patching of the lib at least for now.
154
155 """
156
157 # this patch is needed to ensure that recreate_table() doesn't drop
158 # existing unique constraints of the table when creating a new one
159 helper_cls = sqlite.SQLiteHelper
160 helper_cls.recreate_table = _recreate_table
161 helper_cls._get_unique_constraints = _get_unique_constraints
162
163 # this patch is needed to be able to drop existing unique constraints
164 constraint_cls = sqlite.SQLiteConstraintDropper
165 constraint_cls.visit_migrate_unique_constraint = \
166 _visit_migrate_unique_constraint
167 constraint_cls.__bases__ = (ansisql.ANSIColumnDropper,
168 sqlite.SQLiteConstraintGenerator)
169
170
171def db_sync(engine, abs_path, version=None, init_version=0, sanity_check=True):
172 """Upgrade or downgrade a database.
173
174 Function runs the upgrade() or downgrade() functions in change scripts.
175
176 :param engine: SQLAlchemy engine instance for a given database
177 :param abs_path: Absolute path to migrate repository.
178 :param version: Database will upgrade/downgrade until this version.
179 If None - database will update to the latest
180 available version.
181 :param init_version: Initial database version
182 :param sanity_check: Require schema sanity checking for all tables
183 """
184
185 if version is not None:
186 try:
187 version = int(version)
188 except ValueError:
189 raise exception.DbMigrationError(
190 message=_("version should be an integer"))
191
192 current_version = db_version(engine, abs_path, init_version)
193 repository = _find_migrate_repo(abs_path)
194 if sanity_check:
195 _db_schema_sanity_check(engine)
196 if version is None or version > current_version:
197 return versioning_api.upgrade(engine, repository, version)
198 else:
199 return versioning_api.downgrade(engine, repository,
200 version)
201
202
203def _db_schema_sanity_check(engine):
204 """Ensure all database tables were created with required parameters.
205
206 :param engine: SQLAlchemy engine instance for a given database
207
208 """
209
210 if engine.name == 'mysql':
211 onlyutf8_sql = ('SELECT TABLE_NAME,TABLE_COLLATION '
212 'from information_schema.TABLES '
213 'where TABLE_SCHEMA=%s and '
214 'TABLE_COLLATION NOT LIKE "%%utf8%%"')
215
216 # NOTE(morganfainberg): exclude the sqlalchemy-migrate and alembic
217 # versioning tables from the tables we need to verify utf8 status on.
218 # Non-standard table names are not supported.
219 EXCLUDED_TABLES = ['migrate_version', 'alembic_version']
220
221 table_names = [res[0] for res in
222 engine.execute(onlyutf8_sql, engine.url.database) if
223 res[0].lower() not in EXCLUDED_TABLES]
224
225 if len(table_names) > 0:
226 raise ValueError(_('Tables "%s" have non utf8 collation, '
227 'please make sure all tables are CHARSET=utf8'
228 ) % ','.join(table_names))
229
230
231def db_version(engine, abs_path, init_version):
232 """Show the current version of the repository.
233
234 :param engine: SQLAlchemy engine instance for a given database
235 :param abs_path: Absolute path to migrate repository
236 :param version: Initial database version
237 """
238 repository = _find_migrate_repo(abs_path)
239 try:
240 return versioning_api.db_version(engine, repository)
241 except versioning_exceptions.DatabaseNotControlledError:
242 meta = sqlalchemy.MetaData()
243 meta.reflect(bind=engine)
244 tables = meta.tables
245 if len(tables) == 0 or 'alembic_version' in tables:
246 db_version_control(engine, abs_path, version=init_version)
247 return versioning_api.db_version(engine, repository)
248 else:
249 raise exception.DbMigrationError(
250 message=_(
251 "The database is not under version control, but has "
252 "tables. Please stamp the current version of the schema "
253 "manually."))
254
255
256def db_version_control(engine, abs_path, version=None):
257 """Mark a database as under this repository's version control.
258
259 Once a database is under version control, schema changes should
260 only be done via change scripts in this repository.
261
262 :param engine: SQLAlchemy engine instance for a given database
263 :param abs_path: Absolute path to migrate repository
264 :param version: Initial database version
265 """
266 repository = _find_migrate_repo(abs_path)
267 versioning_api.version_control(engine, repository, version)
268 return version
269
270
271def _find_migrate_repo(abs_path):
272 """Get the project's change script repository
273
274 :param abs_path: Absolute path to migrate repository
275 """
276 if not os.path.exists(abs_path):
277 raise exception.DbMigrationError("Path %s not found" % abs_path)
278 return Repository(abs_path)
diff --git a/ec2api/openstack/common/db/sqlalchemy/models.py b/ec2api/openstack/common/db/sqlalchemy/models.py
new file mode 100644
index 0000000..db401b7
--- /dev/null
+++ b/ec2api/openstack/common/db/sqlalchemy/models.py
@@ -0,0 +1,119 @@
1# Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
2# Copyright 2010 United States Government as represented by the
3# Administrator of the National Aeronautics and Space Administration.
4# Copyright 2011 Piston Cloud Computing, Inc.
5# Copyright 2012 Cloudscaling Group, Inc.
6# All Rights Reserved.
7#
8# Licensed under the Apache License, Version 2.0 (the "License"); you may
9# not use this file except in compliance with the License. You may obtain
10# a copy of the License at
11#
12# http://www.apache.org/licenses/LICENSE-2.0
13#
14# Unless required by applicable law or agreed to in writing, software
15# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17# License for the specific language governing permissions and limitations
18# under the License.
19"""
20SQLAlchemy models.
21"""
22
23import six
24
25from sqlalchemy import Column, Integer
26from sqlalchemy import DateTime
27from sqlalchemy.orm import object_mapper
28
29from ec2api.openstack.common import timeutils
30
31
32class ModelBase(six.Iterator):
33 """Base class for models."""
34 __table_initialized__ = False
35
36 def save(self, session):
37 """Save this object."""
38
39 # NOTE(boris-42): This part of code should be look like:
40 # session.add(self)
41 # session.flush()
42 # But there is a bug in sqlalchemy and eventlet that
43 # raises NoneType exception if there is no running
44 # transaction and rollback is called. As long as
45 # sqlalchemy has this bug we have to create transaction
46 # explicitly.
47 with session.begin(subtransactions=True):
48 session.add(self)
49 session.flush()
50
51 def __setitem__(self, key, value):
52 setattr(self, key, value)
53
54 def __getitem__(self, key):
55 return getattr(self, key)
56
57 def get(self, key, default=None):
58 return getattr(self, key, default)
59
60 @property
61 def _extra_keys(self):
62 """Specifies custom fields
63
64 Subclasses can override this property to return a list
65 of custom fields that should be included in their dict
66 representation.
67
68 For reference check tests/db/sqlalchemy/test_models.py
69 """
70 return []
71
72 def __iter__(self):
73 columns = list(dict(object_mapper(self).columns).keys())
74 # NOTE(russellb): Allow models to specify other keys that can be looked
75 # up, beyond the actual db columns. An example would be the 'name'
76 # property for an Instance.
77 columns.extend(self._extra_keys)
78 self._i = iter(columns)
79 return self
80
81 # In Python 3, __next__() has replaced next().
82 def __next__(self):
83 n = six.advance_iterator(self._i)
84 return n, getattr(self, n)
85
86 def next(self):
87 return self.__next__()
88
89 def update(self, values):
90 """Make the model object behave like a dict."""
91 for k, v in six.iteritems(values):
92 setattr(self, k, v)
93
94 def iteritems(self):
95 """Make the model object behave like a dict.
96
97 Includes attributes from joins.
98 """
99 local = dict(self)
100 joined = dict([(k, v) for k, v in six.iteritems(self.__dict__)
101 if not k[0] == '_'])
102 local.update(joined)
103 return six.iteritems(local)
104
105
106class TimestampMixin(object):
107 created_at = Column(DateTime, default=lambda: timeutils.utcnow())
108 updated_at = Column(DateTime, onupdate=lambda: timeutils.utcnow())
109
110
111class SoftDeleteMixin(object):
112 deleted_at = Column(DateTime)
113 deleted = Column(Integer, default=0)
114
115 def soft_delete(self, session):
116 """Mark this object as deleted."""
117 self.deleted = self.id
118 self.deleted_at = timeutils.utcnow()
119 self.save(session=session)
diff --git a/ec2api/openstack/common/db/sqlalchemy/provision.py b/ec2api/openstack/common/db/sqlalchemy/provision.py
new file mode 100644
index 0000000..2412bc6
--- /dev/null
+++ b/ec2api/openstack/common/db/sqlalchemy/provision.py
@@ -0,0 +1,157 @@
1# Copyright 2013 Mirantis.inc
2# All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16"""Provision test environment for specific DB backends"""
17
18import argparse
19import logging
20import os
21import random
22import string
23
24from six import moves
25import sqlalchemy
26
27from ec2api.openstack.common.db import exception as exc
28
29
30LOG = logging.getLogger(__name__)
31
32
33def get_engine(uri):
34 """Engine creation
35
36 Call the function without arguments to get admin connection. Admin
37 connection required to create temporary user and database for each
38 particular test. Otherwise use existing connection to recreate connection
39 to the temporary database.
40 """
41 return sqlalchemy.create_engine(uri, poolclass=sqlalchemy.pool.NullPool)
42