diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 15725ae..0000000 --- a/.gitignore +++ /dev/null @@ -1,100 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# IPython Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# dotenv -.env - -# virtualenv -venv/ -ENV/ - -# Spyder project settings -.spyderproject - -# Rope project settings -.ropeproject - -# pbr generates these -AUTHORS -ChangeLog - -# Editors -*~ -.*.swp -.*sw? - -# Files created by releasenotes build -extension.xml diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 8dada3e..0000000 --- a/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index a6e4a82..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,7 +0,0 @@ -include AUTHORS -include ChangeLog -exclude .gitignore -exclude .gitreview -recursive-include fuel_external_git * - -global-exclude *.pyc diff --git a/README.md b/README.md deleted file mode 100644 index 67f389a..0000000 --- a/README.md +++ /dev/null @@ -1,182 +0,0 @@ -# Nailgun API Extension with External Git Server -### About -Nailgun extension that generates deployment data based on configuration files published in external -git repository -### Requirements -Operational Fuel 9.x (Mitaka) Master Node - -### Installation -Execute following commands on Fuel Master node -``` -# yum install git python-pip -# git clone https://github.com/openstack/fuel-nailgun-extension-iac -# cd fuel-nailgun-extension-iac -# pip install -r requirements.txt -# python setup.py install -# nailgun_syncdb -# systemctl restart nailgun.service -``` -Since the 9. version of Fuel extension should be enabled. -To list all available extensions execute following command -``` -# fuel2 extension list -``` -Than enable extension for a particular environment -``` -# fuel2 env extension enable -E fuel_external_git -``` - -### How to Use - -This extension introduces two sets of additional Fuel CLI commands. The first set allows the -operator to associate a git repo with a particular environment and preform CRUD operations on -this repo. The second set allows the operator to execute audit and enforce operations on the -environment as well as list the changes made to configuration. It also allows to manage white -lists for these changes. -See details [here](./doc/cli.md). -``` - gitrepo create - gitrepo delete - gitrepo get configs - gitrepo list - gitrepo update - - audit enforce - audit noop - audit list outofsync - audit whitelist show - audit whitelist add - audit whitelist delete -``` -Create repository and configure nailgun to use it. -``` -fuel2 gitrepo create --env 1 --name oscnf1 --url git@github.com:dukov/oscnf.git --ref master \ - --key .ssh/id_rsa -``` -In order to track configuration evolution it is possible to download all configuration files from -the environment into **separate** branch of configured Git repository. User which has been -configured to access repository must have write permissions to it -``` -fuel2 gitrepo get configs --env 1 -``` -#### Git Repo structure -Here is the example repo structure -``` -. -|-- cluster.yaml -|-- nodes -| `-- node-1.domain.local.yaml -`-- roles - |-- compute.yaml - |-- controller.yaml - `-- primary-controller.yaml -``` -There are three levels of configuration: Cluster, Role, Node. Each level has higher priority in terms -of configuration parameters. -* Cluster - configuration parameters from all configs from this level will be applied to all nodes - in environment. -* Role - configuration parameters from all configs from this level will be applied to nodes with a - particular role. Parameters from this level will override parameters from Global level -* Node - configuration parameters from all configs from this level will be applied to node with a - particular id. Parameters from this level will override parameters from Global and Role levels - -For example we have following contents of the files -``` -# cat cluster.yaml -configuration: - nova_config: - 'DEFAULT/nova_test': - value: cluster_param - 'DEFAULT/another_param': - value: another_param_value - -# cat roles/primary-controller.yaml -configuration: - nova_config: - 'DEFAULT/nova_test': - value: controller_param -``` -Resulting configuration Hash will be: -``` -configuration: - nova_config: - 'DEFAULT/nova_test': - value: controller_param - 'DEFAULT/another_param': - value: another_param_value -``` - -### Audit and enforcement -This feature enables the operator to audit the changes made to the environment as well as enforce -configuration. - -``` -fuel2 audit noop --env || --repo -``` -Audit is basically a Fuel graph run with noop flag set. This runs the whole graph and records Puppet resources, that would have changed their state. The command above is equivalent to -``` -fuel2 env redeploy --noop -``` - -After the audit run, the operator is able to list the changes to the state of Puppet resources on the environment via following command: -``` -fuel2 audit list outofsync --task || --repo -``` -This is a convenient alternative to the stock command: -``` -fuel2 task history show --include-summary -``` - -To enforce configuration state, the operator can issue a stock redeploy command: -``` -fuel2 env redeploy -``` - -To perform the whole audit-enforce process automatically, this extension provides the following command: -``` -fuel2 audit enforce --env || --repo -``` -This command will run audit, check the changes and will enforce configuration, if needed. - -### Audit changes whitelisting -Since fuel-library contains non-idempotent tasks, that contain Puppet resources, which will be -triggered on each deployment run, this extension provides the operator the ability to filter such changes out. - -A whitelist rule is a pair of strings. The first one is a fuel task name to match. The second one is what should be included into a Puppet report line for the whitelisted resource change, e.g. for -``` -Openstack_tasks::Swift::Proxy_storage/Package[mc]/ensure -``` -the whitelist rule could be -``` -Package[mc]/ensure -``` -A rule with empty fuel_task filter will match to all tasks. - -Whitelist rules for an environment can be listed by -``` -fuel2 audit whitelist show -``` -These rules can be managed by following commands: -``` -fuel2 audit whitelist add --task --rule -fuel2 audit whitelist delete [ ...] -fuel2 audit whitelist load fromfile -``` - -Example YAML file with whitelist rules: -``` -- fuel_task: netconfig - rule: L23_stored_configs -- fuel_task: top-role-compute - rule: Service[nova-compute]/ensure -``` - -The default whitelist can be loaded with following command -``` -fuel2 audit whitelist load fromfile /usr/lib/python2.7/site-packages/fuel_external_git/default_whitelist.yaml -``` -Note: this whitelist is not complete as it has been put together on following configuration: -MOS 9.1, Ubuntu, 1 controller, 1 compute+cinder lvm, Neutron GRE. - -### REST API -API documentation can be found [here](./doc/api.md) diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..86e34d6 --- /dev/null +++ b/README.rst @@ -0,0 +1,10 @@ +This project is no longer maintained. + +The contents of this repository are still available in the Git +source code management system. To see the contents of this +repository before it reached its end of life, please check out the +previous commit with "git checkout HEAD^1". + +For any further questions, please email +openstack-discuss@lists.openstack.org or join #openstack-dev on +Freenode. diff --git a/doc/api.md b/doc/api.md deleted file mode 100644 index 97756d0..0000000 --- a/doc/api.md +++ /dev/null @@ -1,94 +0,0 @@ -### API -Extension supports following REST API calls -#### GET /clusters/git-repos -Returns list of configured git repos for all clusters/environments -Example -``` -curl -H "X-Auth-Token: $(fuel token)" http://localhost:8000/api/v1/clusters/git-repos -``` - -#### POST /clusters/git-repos -Create new repo for particular cluster -Input data schema: -``` -"$schema": "http://json-schema.org/draft-04/schema#", - "title": "Cluster", - "description": "Serialized GitRepo object", - "type": "object", - "properties": { - "id": {"type": "number"}, - "repo_name": {"type": "string"}, - "env_id": {"type": "number"}, - "git_url": {"type": "string"}, - "ref": {"type": "string"}, - "user_key": {"type": "string"} -``` - -Example -``` -curl -X POST -H "X-Auth-Token: $(fuel token)" http://localhost:8000/api/v1/clusters/git-repos -d '{"user_key": "", "git_url": "https://github.com/dukov/openstack-configs", "env_id": 5, "ref": "master", "repo_name": "osconf1"}' -``` - -#### PUT /clusters/(cluster_id)/git-repos/(obj_id) -Updates repo with obj_id info for cluster cluster_id -Example: -``` -curl -X PUT -H 'X-Auth-Token: $(fuel token)' http://localhost:8000/api/v1/clusters/4/git-repos/2 -d '{"ref": "master"}' -``` - - -#### GET /clusters/changes-whitelist/(obj_id) -Returns the serialized whitelist rule object -Example -``` -curl -H "X-Auth-Token: $(fuel token)" http://localhost:8000/api/v1/clusters/changes-whitelist/1 -``` - -#### PUT /clusters/changes-whitelist/(obj_id) -Updates a whitelist rule -Input data schema: -``` -"$schema": "http://json-schema.org/draft-04/schema#", -"title": "ChangesWhitelistRule", -"description": "Serialized ChangesWhitelistRule object", -"type": "object", -"properties": { - "rule": {"type": "string"}, - "fuel_task": {"type": "string"}, -} -``` -Example -``` -curl -H "X-Auth-Token: $(fuel token)" -X PUT http://localhost:8000/api/v1/clusters/changes-whitelist/1 -d '{"rule": "new-rule-string", "fuel_task": "fuel-task-id"}' -``` - -#### DELETE /clusters/changes-whitelist/(obj_id) -Deletes a whitelist rule -Example -``` -curl -H "X-Auth-Token: $(fuel token)" -X DELETE http://localhost:8000/api/v1/clusters/changes-whitelist/1 -``` - -#### GET /clusters/(env_id)/changes-whitelist/ -Returns the whitelist rules for a specified environment -Example -``` -curl -H "X-Auth-Token: $(fuel token)" http://localhost:8000/api/v1/clusters/1/changes-whitelist/ -``` -#### POST /clusters/(env_id)/changes-whitelist/ -Creates one or more whitelist rule(s) -Input data schema: -``` -"$schema": "http://json-schema.org/draft-04/schema#", -"title": "ChangesWhitelistRule Collection", -"description": "Serialized ChangesWhitelistRule collection", -"type": "object", -"items": { - "rule": {"type": "string"}, - "fuel_task": {"fuel_task": "string"}, -} -``` -Example -``` -curl -H "X-Auth-Token: $(fuel token)" -X POST http://localhost:8000/api/v1/clusters/1/changes-whitelist/ -d '[{"rule": "new-rule-string", "fuel_task": "task1"}, {"rule": "new-rule-2", "fuel_task": ""}]' -``` diff --git a/doc/cli.md b/doc/cli.md deleted file mode 100644 index 9b12c9f..0000000 --- a/doc/cli.md +++ /dev/null @@ -1,81 +0,0 @@ -#### Fuel CLI -##### Create association of environment and git repository -``` -fuel2 gitrepo create [-h] --env ENV --name NAME --url URL --ref REF - [--key KEY] - - --env ENV ID of environment to configure. - --name NAME Name of the git repository. Will be used as directory name for - repository. - --url URL Url of Git repository. User should be specified in this url. - --ref REF Git ref. This can be either a branch or Gerrit refspec. - --key KEY Path to private key file for accessing repo -``` -For example: -``` -fuel2 gitrepo create --env 1 --name oscnf1 --url git@github.com:dukov/oscnf.git --ref master --key .ssh/id_rsa -``` - -##### (Optional) User can download supported config files from the environment and upload them to configured git repository -``` -fuel2 gitrepo get configs [-h] [--env ENV] [--key_path KEY_PATH] - [--repo_dir REPO_DIR] - - --env ENV Env ID - --key_path KEY_PATH Path to nodes private key file - --repo_dir REPO_DIR Directory to Git repo download -``` -For example download configs from all environment and push them to configured repo: -``` -fuel2 gitrepo get configs -``` -##### Other commands -You can update,delete and list git repo association executing corresponding command -``` -fuel2 gitrepo -``` -To get more detailed description use: -``` -fuel2 help -``` - -##### Audit and enforce -These commands allow to perform audit and enforce configuration on the environment as well as to list the changes made to it. - -To perform the audit run on the environment: -``` -fuel2 audit noop --env || --repo -``` - -To list the changes: -``` -fuel2 audit list outofsync --task || --env -``` - -To perform audit run, inspect changes and enforce configuration, if needed: -``` -fuel2 audit enforce --env || --repo -``` - -##### Changes whitelist commands -These commands manage the rules, that allow to ignore certain changes to configuration. - -To show rules whitelist for the environment: -``` -fuel2 audit whitelist show -``` - -To add a rule: -``` -fuel2 audit whitelist add --task --rule -``` - -To delete a rule or set of rules: -``` -fuel2 audit whitelist delete [ ... ] -``` - -To add rules from YAML file: -``` -fuel2 audit whitelist load fromfile -``` diff --git a/fuel_external_git/__init__.py b/fuel_external_git/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/fuel_external_git/alembic.ini b/fuel_external_git/alembic.ini deleted file mode 100644 index 7d13235..0000000 --- a/fuel_external_git/alembic.ini +++ /dev/null @@ -1,68 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -script_location = migrations/ - -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# max length of characters to apply to the -# "slug" field -#truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; this defaults -# to migrations//versions. When using multiple version -# directories, initial revisions must be specified with --version-path -# version_locations = %(here)s/bar %(here)s/bat migrations//versions - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 - -sqlalchemy.url = driver://user:pass@localhost/dbname - - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/fuel_external_git/const.py b/fuel_external_git/const.py deleted file mode 100644 index 968bbdf..0000000 --- a/fuel_external_git/const.py +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -AUDIT_TASK_CHECK_INTERVAL = 20 -REPOS_DIR = '/var/lib/fuel_repos' -REPO_TTL = 1200 -TASK_RETRIES = 10 -TASK_RETRY_DELAY = 2 diff --git a/fuel_external_git/default_whitelist.yaml b/fuel_external_git/default_whitelist.yaml deleted file mode 100644 index 40d1530..0000000 --- a/fuel_external_git/default_whitelist.yaml +++ /dev/null @@ -1,54 +0,0 @@ -- fuel_task: netconfig - rule: 'Service[irqbalance]/ensure' -- fuel_task: netconfig - rule: 'Notify[SDN]' -- fuel_task: netconfig - rule: 'Disable_hotplug[global]' -- fuel_task: netconfig - rule: 'L23_stored_config' -- fuel_task: netconfig - rule: 'vendor_specific' -- fuel_task: netconfig - rule: 'external_ids' -- fuel_task: netconfig - rule: 'L3_clear_route[default]' -- fuel_task: netconfig - rule: 'br-floating' -- fuel_task: netconfig - rule: 'Exec[wait-for-interfaces]' -- fuel_task: configuration_symlink - rule: 'Exec[configuration_symlink_shell]' -- fuel_task: sync_time - rule: 'Exec[sync_time_shell]' -- fuel_task: generate_keys - rule: 'Exec[generate_keys_shell]' -- fuel_task: sriov_iommu_check - rule: 'Exec[sriov_iommu_check]' -- fuel_task: primary-database - rule: 'Mysql::Server::Root_password/File[/root/.my.cnf]/target' -- fuel_task: openstack-network-common-config - rule: 'Sysctl_runtime[net.ipv4.neigh.default.gc_thresh1]/val' -- fuel_task: openstack-network-common-config - rule: 'Sysctl_runtime[net.ipv4.neigh.default.gc_thresh2]/val' -- fuel_task: openstack-network-common-config - rule: 'Sysctl_runtime[net.ipv4.neigh.default.gc_thresh3]/val' -- fuel_task: top-role-compute - rule: 'Notify[Module openstack_tasks cannot notify service nova-compute on packages update]' -- fuel_task: top-role-compute - rule: 'Nova/Nova_config[cinder/os_region_name]/ensure' -- fuel_task: top-role-compute - rule: 'Nova::Deps/Anchor[nova::config::end]' -- fuel_task: top-role-compute - rule: 'Nova::Deps/Anchor[nova::service::begin]' -- fuel_task: top-role-compute - rule: 'Nova::Deps/Anchor[nova::service::end]' -- fuel_task: top-role-compute - rule: 'Nova::Compute/Nova::Generic_service[compute]/Service[nova-compute]' -- fuel_task: top-role-cinder - rule: 'Cinder::Volume/Service[cinder-volume]/ensure' -- fuel_task: workloads_collector_add - rule: 'Openstack::Workloads_collector/Keystone_tenant[services]/description' -- fuel_task: generate_keys_ceph - rule: 'Main/Exec[generate_keys_ceph_shell]/returns' -- fuel_task: openrc-delete - rule: 'Openstack_tasks::Keystone::Openrc_delete/File[/root/openrc]/ensure' diff --git a/fuel_external_git/drivers/__init__.py b/fuel_external_git/drivers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/fuel_external_git/drivers/openstack_config.py b/fuel_external_git/drivers/openstack_config.py deleted file mode 100644 index bf8ddb8..0000000 --- a/fuel_external_git/drivers/openstack_config.py +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import ConfigParser -import os - -from nailgun.logger import logger - - -class OpenStackConfig(object): - def __init__(self, config_file, resource_name=None): - self.config = ConfigParser.ConfigParser() - self.config.read(config_file) - if resource_name: - self.config_name = resource_name - else: - file_name = os.path.basename(config_file) - self.config_name = "{}_config".format(file_name.split('.')[:-1]) - logger.debug("Initalized Config {0}. Config resource name {1}.". - format(config_file, self.config_name)) - - def to_config_dict(self): - """Config transformation - - Function returns OpenStack config file in dictionary form - compatible with override_configuration resource in - fuel-library - Example: - { - "nova_config": { - DEFAULT/debug: {value: True} - DEFAULT/amqp_durable_queues: {value: False} - } - } - """ - config_resource = {} - default_items = set(self.config.items('DEFAULT')) - for section in self.config.sections(): - # Config prasee includes all items from DEFAULT secton - # into items for each section. We need to exclude them - section_content = set(self.config.items(section)) - default_items - for key, value in section_content: - params = {'value': value} - config_resource["{0}/{1}".format(section, key)] = params - - # Add parameters from DEFAULT section - for key, value in default_items: - config_resource["DEFAULT/{}".format(key)] = {'value': value} - - return {self.config_name: config_resource} diff --git a/fuel_external_git/drivers/yaml_driver.py b/fuel_external_git/drivers/yaml_driver.py deleted file mode 100644 index 7190bf6..0000000 --- a/fuel_external_git/drivers/yaml_driver.py +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import yaml - -from nailgun.logger import logger - - -class YamlConfig(object): - def __init__(self, config_file, resource_name): - with open(config_file) as cfg: - self.config = yaml.load(cfg) - self.config_name = resource_name - - logger.debug("Initalized Config {0}.". - format(config_file)) - - def to_config_dict(self): - return self.config diff --git a/fuel_external_git/extension.py b/fuel_external_git/extension.py deleted file mode 100644 index ef930fe..0000000 --- a/fuel_external_git/extension.py +++ /dev/null @@ -1,198 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import os -import yaml - -from fuel_external_git import const -from fuel_external_git import handlers -from fuel_external_git.objects import GitRepo -from fuel_external_git.settings import GitExtensionSettings -from fuel_external_git import utils - -from nailgun.extensions import BaseExtension -from nailgun.extensions import BasePipeline -from nailgun.logger import logger - - -# TODO(dukov) add cluster remove callback -class OpenStackConfigPipeline(BasePipeline): - - @classmethod - def lcm_v2(cls, repo_path, node_data): - roles_dir = os.path.join(repo_path, 'roles') - nodes_dir = os.path.join(repo_path, 'nodes') - res_mapping = {} - node_role_list = node_data.get('roles', []) - node_fqdn = node_data['fqdn'] - yaml_drv = 'fuel_external_git.drivers.yaml_driver.YamlConfig' - yaml_list = [] - if os.path.isdir(roles_dir): - yaml_list += os.listdir(roles_dir) - - if os.path.isdir(nodes_dir): - yaml_list += os.listdir(nodes_dir) - - for cfg in yaml_list: - for node_role in node_role_list: - file_name = ".".join(cfg.split('.')[:-1]) - if file_name in node_role or file_name == node_fqdn: - res_mapping[cfg] = {'driver': yaml_drv, 'resource': 'yaml'} - roles_data = utils.get_config_hash(roles_dir, - res_mapping, - exts=['yaml']) - node_data = utils.get_config_hash(nodes_dir, - res_mapping, - exts=['yaml']) - utils.deep_merge(roles_data, node_data) - return roles_data - - @classmethod - def lcm_v1(cls, node, node_data, repo_path): - resource_mapping = ExternalGit.ext_settings['resource_mapping'] - exts_list = utils.get_file_exts_list(resource_mapping) - - global_config = utils.get_config_hash(repo_path, - resource_mapping, - exts=exts_list) - # Read config for overrides - # Overrides file should contain following mapping - # - role:config_file_dir - # - node_id:config_file_dir - # Config options from files for roles should override global - # configs (placed in repo root). - # Config options from files for nodes should override global - # and roles config oprions - overrides_file = os.path.join(repo_path, 'overrides.yaml') - if os.path.exists(overrides_file): - overrides = yaml.load(open(overrides_file)) - else: - overrides = {'roles': {}, 'nodes': {}} - - override_configs = {} - # ent = roles|nodes - for ent, override in overrides.items(): - override_configs[ent] = {} - # key = role_name|node_id - for key, path in override.items(): - file_dir = os.path.join(repo_path, path) - config_hash = utils.get_config_hash(file_dir, - resource_mapping, - exts=exts_list) - override_configs[ent][key] = config_hash - - logger.debug("Override configs {}".format(override_configs)) - - common = copy.deepcopy(global_config) - roles = node_data['roles'] - uid = node_data['uid'] - logger.debug("Node {0} roles {1}".format(uid, roles)) - for role in roles: - utils.deep_merge(common, - override_configs['roles'].get(role, {})) - - logger.debug("Config Node {0} with roles {1}".format(uid, common)) - - utils.deep_merge(common, - override_configs['nodes'].get(uid, {})) - logger.debug("Node {0} config from git {1}".format(uid, common)) - return {'configuration': common} - - @classmethod - def process_deployment_for_node(cls, node, node_data): - """Updating deployment info - - Genereate OpenStack configuration hash based on configuration files - stored in git repository associated with a particular environment - """ - logger.info("Started serialization for node {}".format(node.id)) - repo = GitRepo.get_by_cluster_id(node.cluster_id) - if not repo: - return node_data - - GitRepo.checkout(repo) - repo_path = os.path.join(const.REPOS_DIR, repo.repo_name) - - lcm_version = ExternalGit.ext_settings.get('lcm_version', 'v2') - if lcm_version == 'v1': - data = cls.lcm_v1(node, node_data, repo_path) - else: - data = cls.lcm_v2(repo_path, node_data) - - utils.deep_merge(node_data, data) - logger.info("Finished serialization for node {}".format(node.id)) - return node_data - - @classmethod - def process_deployment_for_cluster(self, cluster, data): - logger.info("Started serialization for cluster {}".format(cluster.id)) - repo = GitRepo.get_by_cluster_id(cluster.id) - if not repo: - return data - - repo_path = os.path.join(const.REPOS_DIR, repo.repo_name) - yaml_drv = 'fuel_external_git.drivers.yaml_driver.YamlConfig' - - if repo.manage_master: - GitRepo.checkout(repo) - resource_mapping = ExternalGit.ext_settings['master_mapping'] - master_config = utils.get_config_hash( - repo_path, - {'master_config': resource_mapping.get('master_config', {})}, - exts=['yaml'] - ) - - data['master_config'] = master_config - - overrides = {} - for override in ('global.yaml', 'cluster.yaml'): - res_mapping = { - override: { - 'driver': yaml_drv, - 'resource': 'yaml' - } - } - override_data = utils.get_config_hash(repo_path, - res_mapping, - exts=['yaml']) - utils.deep_merge(overrides, override_data) - - utils.deep_merge(data, overrides) - logger.info("Finished serialization for cluster {}".format(cluster.id)) - return data - - -class ExternalGit(BaseExtension): - name = 'fuel_external_git' - version = '1.0.0' - description = 'Nailgun extension which uses git repo for config files' - - urls = [{'uri': r'/clusters/git-repos/?$', - 'handler': handlers.GitRepoCollectionHandler}, - {'uri': - r'/clusters/(?P\d+)/git-repos/(?P\d+)?$', - 'handler': handlers.GitRepoHandler}, - {'uri': r'/clusters/(?P\d+)/changes-whitelist/$', - 'handler': handlers.ChangesWhitelistRuleCollectionHandler}, - {'uri': r'/clusters/changes-whitelist/(?P\d+)?$', - 'handler': handlers.ChangesWhitelistRuleHandler}] - - data_pipelines = [ - OpenStackConfigPipeline, - ] - - ext_settings = GitExtensionSettings().config - - @classmethod - def alembic_migrations_path(cls): - return os.path.join(os.path.dirname(__file__), 'migrations') diff --git a/fuel_external_git/fuelclient_audit.py b/fuel_external_git/fuelclient_audit.py deleted file mode 100644 index 5998c9e..0000000 --- a/fuel_external_git/fuelclient_audit.py +++ /dev/null @@ -1,398 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import - -import logging -import time - -from cliff import command -from cliff import lister -import yaml - -from fuelclient import client -if hasattr(client, 'DefaultAPIClient'): - # Handling python-fuelclient version >= 10.0 - fc_client = client.DefaultAPIClient -else: - # Handling python-fuelclient version <= 9.0 - fc_client = client.APIClient -from fuelclient.common import data_utils -from fuelclient.objects import Environment -from fuelclient.objects import Task - -from fuel_external_git.const import AUDIT_TASK_CHECK_INTERVAL -from fuel_external_git.const import TASK_RETRIES -from fuel_external_git.const import TASK_RETRY_DELAY - - -LOG = logging.getLogger(__name__) - - -class Audit(lister.Lister, command.Command): - columns = () - - @staticmethod - def get_running_task(name): - for retry in xrange(TASK_RETRIES): - tasks = Task.get_all() - try: - task = filter(lambda t: t.data['status'] == 'running' and - t.data['name'] == name, - tasks).pop() - break - except IndexError: - time.sleep(TASK_RETRY_DELAY) - continue - return task - - @staticmethod - def start_noop_run(env): - # Due to how Nailgun handles such tasks, returned - # one will not contain any deployment-related data. - # So we'll have to fetch the last noop task with progress < 100 - env.redeploy_changes(noop_run=True) - return Audit.get_running_task('dry_run_deployment') - - @staticmethod - def get_outofsync(noop_task): - history = noop_task.connection.get_request( - ('transactions/{tid}/' - 'deployment_history?include_summary=1').format(tid=noop_task.id) - ) - changes = [] - changed_tasks = filter(lambda t: t['status'] != 'skipped' and - t.get('summary', {}) and - t['summary']['resources']['out_of_sync'] > 0, - history) - for task in changed_tasks: - name = task['task_name'] - for item in task['summary']['raw_report']: - if item['source'].startswith('/Stage[main]/'): - short_item = item['source'].replace('/Stage[main]/', '') - changes.append({'task_id': name, - 'resource': short_item, - 'node_id': task['node_id']}) - return changes - - @staticmethod - def filter_changes(changes, env_id): - whitelist = fc_client.get_request( - '/clusters/{env}/changes-whitelist/'.format(env=env_id) - ) - - matched_rules = lambda change: filter( - lambda rule: rule['rule'] in change['resource'] and - (rule['fuel_task'] == change['task_id'] or - rule['fuel_task'] == ''), - whitelist - ) - - changes = filter(lambda c: len(matched_rules(c)) == 0, - changes) - - return changes - - def get_parser(self, prog_name): - parser = super(Audit, self).get_parser(prog_name) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('--env', - type=int, - help='Environment ID') - group.add_argument('--repo', - type=int, - help='Associated Repo ID') - return parser - - def take_action(self, parsed_args): - env_id = parsed_args.env - if not env_id: - repo_id = parsed_args.repo - repos = fc_client.get_request('/clusters/git-repos/') - env_id = [repo['env_id'] for repo in repos - if repo['id'] == repo_id][0] - - env = Environment(env_id) - - audit_task = Audit.start_noop_run(env) - - LOG.info("Noop task ID is {tid}".format(tid=audit_task.id)) - - while audit_task.status == 'running': - time.sleep(AUDIT_TASK_CHECK_INTERVAL) - LOG.info( - 'Current task progress is {p}'.format(p=audit_task.progress) - ) - - changes = Audit.get_outofsync(audit_task) - - if changes: - changed_tasks = [c['task_id'] for c in changes] - LOG.info( - "Following tasks have outofsync resources: {tasks}".format( - tasks=set(changed_tasks) - ) - ) - LOG.info(("To get the list of changes, run " - "fuel2 audit get outofsync --task " - "{task_id}").format(task_id=audit_task.id)) - LOG.info("Starting enforce run on environment {eid}".format( - eid=env_id - )) - env.redeploy_changes() - enforce_task = Audit.get_running_task('deployment') - LOG.info("Enforce task id is {tid}".format(tid=enforce_task.id)) - while enforce_task.status == 'running': - time.sleep(AUDIT_TASK_CHECK_INTERVAL) - LOG.info( - 'Current task progress is {p}'.format( - p=enforce_task.progress - ) - ) - LOG.info("Enforce task status is {status}".format( - status=enforce_task.status - )) - return ((), {}) - - -class AuditRun(lister.Lister, command.Command): - columns = ( - 'task_id', - ) - - def get_parser(self, prog_name): - parser = super(AuditRun, self).get_parser(prog_name) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('--env', - type=int, - help='Associated Repo ID') - group.add_argument('--repo', - type=int, - help='Associated Repo ID') - return parser - - def take_action(self, parsed_args): - env_id = parsed_args.env - if not env_id: - repo_id = parsed_args.repo - repos = fc_client.get_request('/clusters/git-repos/') - env_id = [repo['env_id'] for repo in repos - if repo['id'] == repo_id][0] - - env = Environment(env_id) - task = Audit.start_noop_run(env) - - data = {'task_id': task.id} - data = data_utils.get_display_data_multi(self.columns, [data]) - return (self.columns, data) - - -class OutOfSyncResources(lister.Lister, command.Command): - columns = ( - 'task_id', - 'node_id', - 'resource' - ) - - def get_parser(self, prog_name): - parser = super(OutOfSyncResources, self).get_parser(prog_name) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('--env', - type=int, - help='Environment to lookup changes on') - group.add_argument('--task', - type=int, - help='Task ID to lookup changes on') - return parser - - def take_action(self, parsed_args): - task_id = parsed_args.task - if not task_id: - all_tasks = Task.get_all() - env_tasks = filter( - lambda t: t.data['cluster'] == parsed_args.env and - t.data['name'] == 'dry_run_deployment', - all_tasks) - env_tasks.sort(key=lambda t: t.data['time_start']) - fuel_task = env_tasks[-1] - else: - fuel_task = Task(task_id) - - env_id = fuel_task.data['cluster'] - - changes = Audit.get_outofsync(fuel_task) - changes = Audit.filter_changes(changes, env_id) - - data = data_utils.get_display_data_multi(self.columns, changes) - return (self.columns, data) - - -class WhitelistRulesShow(lister.Lister, command.Command): - columns = ( - 'id', - 'fuel_task', - 'rule' - ) - - def get_parser(self, prog_name): - parser = super(WhitelistRulesShow, self).get_parser(prog_name) - parser.add_argument('env', - type=int, - help=('Environment to find whitelist rules ' - 'associated with')) - return parser - - def take_action(self, parsed_args): - env_id = parsed_args.env - - rules = fc_client.get_request( - '/clusters/{env}/changes-whitelist/'.format(env=env_id) - ) - - data = data_utils.get_display_data_multi(self.columns, rules) - return (self.columns, data) - - -class WhitelistRuleAdd(lister.Lister, command.Command): - columns = ( - 'id', - 'fuel_task', - 'rule' - ) - - def get_parser(self, prog_name): - parser = super(WhitelistRuleAdd, self).get_parser(prog_name) - parser.add_argument('env', - type=int, - help='Environment to add whitelist rules to') - parser.add_argument('--rule', '-r', - type=str, - required=True, - help='Rule to add') - parser.add_argument('--task', '-t', - type=str, - required=False, - help=('Fuel task for the rule. Omitting this will' - ' result in matching all Fuel tasks.')) - return parser - - def take_action(self, parsed_args): - env_id = parsed_args.env - rule = parsed_args.rule - task = parsed_args.task - data = {'rule': rule} - if task: - data['fuel_task'] = task - - ret = fc_client.post_request( - '/clusters/{env}/changes-whitelist/'.format(env=env_id), - data - ) - ret = data_utils.get_display_data_multi(self.columns, ret) - - return (self.columns, ret) - - -class WhitelistRuleDelete(command.Command): - columns = () - - def get_parser(self, prog_name): - parser = super(WhitelistRuleDelete, self).get_parser(prog_name) - parser.add_argument('rule_id', - type=int, - nargs='+', - help='Rule ID to delete') - return parser - - def take_action(self, parsed_args): - rule_ids = parsed_args.rule_id - - for rule in rule_ids: - fc_client.delete_request( - '/clusters/changes-whitelist/{rule}'.format(rule=rule) - ) - - return ((), {}) - - -class WhitelistRuleAddFromFile(lister.Lister, command.Command): - columns = ( - 'id', - 'fuel_task', - 'rule' - ) - - def get_parser(self, prog_name): - parser = super(WhitelistRuleAddFromFile, self).get_parser(prog_name) - parser.add_argument('env', - type=int, - help='Environment to add whitelist rules to') - parser.add_argument('file_name', - type=str, - help='YAML file to load rules from') - return parser - - def take_action(self, parsed_args): - env_id = parsed_args.env - file_name = parsed_args.file_name - - with open(file_name, 'r') as f: - data = yaml.load(f) - - ret = fc_client.post_request( - '/clusters/{env}/changes-whitelist/'.format(env=env_id), - data - ) - ret = data_utils.get_display_data_multi(self.columns, ret) - - return (self.columns, ret) - - -class WhitelistRuleAddAll(OutOfSyncResources): - columns = ( - 'id', - 'fuel_task', - 'rule' - ) - - def take_action(self, parsed_args): - task_id = parsed_args.task - if not task_id: - all_tasks = Task.get_all() - env_tasks = filter( - lambda t: t.data['cluster'] == parsed_args.env and - t.data['name'] == 'dry_run_deployment', - all_tasks) - env_tasks.sort(key=lambda t: t.data['time_start']) - fuel_task = env_tasks[-1] - else: - fuel_task = Task(task_id) - - env_id = fuel_task.data['cluster'] - - changes = Audit.get_outofsync(fuel_task) - - remap = lambda x: { - 'fuel_task': x['task_id'], - 'rule': x['resource'][:254] - } - remap_changes = map(remap, changes) - uniq_data = set(map(lambda x: tuple(x.items()), remap_changes)) - data = map(lambda x: dict(x), uniq_data) - - ret = fc_client.post_request( - '/clusters/{env}/changes-whitelist/'.format(env=env_id), - data - ) - ret = data_utils.get_display_data_multi(self.columns, ret) - - return (self.columns, ret) diff --git a/fuel_external_git/fuelclient_gitrepo.py b/fuel_external_git/fuelclient_gitrepo.py deleted file mode 100644 index b8dd03d..0000000 --- a/fuel_external_git/fuelclient_gitrepo.py +++ /dev/null @@ -1,234 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import - -from cliff import command -from cliff import lister - -from fuelclient import client -from fuelclient.common import data_utils -if hasattr(client, 'DefaultAPIClient'): - # Handling python-fuelclient version >= 10.0 - fc_client = client.DefaultAPIClient -else: - # Handling python-fuelclient version <= 9.0 - fc_client = client.APIClient - - -class GitRepoList(lister.Lister, command.Command): - columns = ( - 'id', - 'repo_name', - 'env_id', - 'git_url', - 'ref', - 'manage_master', - ) - - def get_parser(self, prog_name): - parser = super(GitRepoList, self).get_parser(prog_name) - parser.add_argument('--env', type=int, help='Env ID', required=False) - return parser - - def take_action(self, parsed_args): - data = fc_client.get_request('/clusters/git-repos/') - if parsed_args.env: - data = [entry for entry in data - if entry['env_id'] == parsed_args.env] - data = data_utils.get_display_data_multi(self.columns, data) - return (self.columns, data) - - -class AddRepo(command.Command): - columns = ( - 'id', - 'repo_name', - 'env_id', - 'git_url', - 'ref', - 'manage_master', - ) - - def get_parser(self, prog_name): - parser = super(AddRepo, self).get_parser(prog_name) - mm = parser.add_mutually_exclusive_group(required=False) - parser.add_argument('--env', - type=int, - help='ID of environment to configure.', - required=True) - - parser.add_argument('--name', - type=str, - help=('Name of the git repository. ' - 'Will be used as directory ' - 'name for repository.'), - required=True) - - parser.add_argument('--url', - type=str, - help=('Url of Git repository. ' - 'User should be specified in this url.'), - required=True) - - parser.add_argument('--ref', - type=str, - help=('Git ref. This can be either a branch ' - 'or Gerrit refspec.'), - required=True) - - parser.add_argument('--key', - type=str, - help='Path to private key file for accessing repo', - required=False) - - mm.add_argument('--manage-master', - dest='manage_master', - help='Enable Fuel master management from this repo', - action='store_true', - required=False) - - mm.add_argument('--no-manage-master', - dest='manage_master', - help='Disable Fuel master management from this repo', - action='store_false', - required=False) - parser.set_defaults(manage_master=False) - return parser - - def take_action(self, parsed_args): - data = { - 'repo_name': parsed_args.name, - 'env_id': parsed_args.env, - 'git_url': parsed_args.url, - 'ref': parsed_args.ref, - 'manage_master': parsed_args.manage_master, - } - - if parsed_args.key: - with open(parsed_args.key) as key_file: - data['user_key'] = key_file.read() - - fc_client.post_request('/clusters/git-repos/', data) - return (self.columns, data) - - -class DeleteRepo(command.Command): - columns = ( - 'id' - ) - - def get_parser(self, prog_name): - parser = super(DeleteRepo, self).get_parser(prog_name) - parser.add_argument('--repo', - type=int, - help='Repo ID to delete', - required=True) - - parser.add_argument('--env', - type=int, - help='Environment to delete Git repo object from', - required=False) - return parser - - def take_action(self, parsed_args): - repo_id = parsed_args.repo - if parsed_args.env: - env = parsed_args.env - else: - repos = fc_client.get_request('/clusters/git-repos/') - env = [repo['env_id'] for repo in repos - if repo['id'] == parsed_args.repo][0] - - del_path = "/clusters/{0}/git-repos/{1}" - fc_client.delete_request(del_path.format(env, repo_id)) - return (self.columns, {}) - - -class UpdateRepo(command.Command): - columns = ( - 'id', - 'repo_name', - 'git_url', - 'ref', - 'manage_master', - ) - - def get_parser(self, prog_name): - parser = super(UpdateRepo, self).get_parser(prog_name) - mm = parser.add_mutually_exclusive_group(required=False) - parser.add_argument('--repo', - type=int, - help='Repo ID to update', - required=True) - - parser.add_argument('--name', - type=str, - help=('Name of the git repository. ' - 'Will be used as directory ' - 'name for repository.'), - required=False) - - parser.add_argument('--url', - type=str, - help=('Url of Git repository. ' - 'User should be specified in this url.'), - required=False) - - parser.add_argument('--ref', - type=str, - help=('Git ref. This can be either a branch ' - 'or Gerrit refspec.'), - required=False) - - parser.add_argument('--key', - type=str, - help='Path to private key file for accessing repo', - required=False) - - mm.add_argument('--manage-master', - dest='manage_master', - help='Enable Fuel master management from this repo', - action='store_true', - required=False) - - mm.add_argument('--no-manage-master', - dest='manage_master', - help='Disable Fuel master management from this repo', - action='store_false', - required=False) - parser.set_defaults(manage_master=False) - return parser - - def take_action(self, parsed_args): - param_mapping = { - 'name': 'repo_name', - 'url': 'git_url', - 'ref': 'ref', - 'manage_master': 'manage_master', - } - - data = {} - for param, value in parsed_args.__dict__.items(): - if value is not None and param in param_mapping.keys(): - data[param_mapping[param]] = value - repos = fc_client.get_request('/clusters/git-repos/') - env = [repo['env_id'] for repo in repos - if repo['id'] == parsed_args.repo][0] - path = "/clusters/{0}/git-repos/{1}" - - if parsed_args.key: - with open(parsed_args.key) as key_file: - data['user_key'] = key_file.read() - - fc_client.put_request(path.format(env, parsed_args.repo), data) - return (self.columns, data) diff --git a/fuel_external_git/handlers.py b/fuel_external_git/handlers.py deleted file mode 100644 index 114407b..0000000 --- a/fuel_external_git/handlers.py +++ /dev/null @@ -1,245 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from fuel_external_git import json_schema -from fuel_external_git.objects import ChangesWhitelistRule -from fuel_external_git.objects import ChangesWhitelistRuleCollection -from fuel_external_git.objects import GitRepo -from fuel_external_git.objects import GitRepoCollection - -from nailgun.api.v1.handlers.base import CollectionHandler -from nailgun.api.v1.handlers.base import handle_errors -from nailgun.api.v1.handlers.base import serialize -from nailgun.api.v1.handlers.base import SingleHandler -from nailgun.api.v1.handlers.base import validate -from nailgun.api.v1.validators import base -from nailgun import errors -from nailgun import objects - - -REPOS_DIR = '/var/lib/fuel_repos' - - -class GitRepoValidator(base.BasicValidator): - - single_schema = json_schema.gitrepo_single_schema - collection_schema = json_schema.gitrepo_collection_schema - - _blocked_for_update = ( - 'env_id', - ) - - @classmethod - def _validate_master_mgmt(self, data, instance=None): - d = self.validate_json(data) - if instance: - repo_id = instance.id - else: - repo_id = d.get('id', None) - if d.get('manage_master', False): - for repo in GitRepoCollection.all(): - if repo.manage_master and repo_id != repo.id: - raise errors.InvalidData( - ("Repo {} already marked for Fuel Master management. " - "Disable it first".format(repo.id)), - log_message=True) - return d - - @classmethod - def validate(self, data): - return self._validate_master_mgmt(data) - - @classmethod - def validate_update(self, data, instance): - d = self._validate_master_mgmt(data, instance) - env_id = d.get('env_id') - if env_id: - cluster = objects.Cluster.get_by_uid(env_id) - if not cluster: - raise errors.InvalidData( - "Invalid cluster ID", log_message=True) - - for k in self._blocked_for_update: - if k in d and getattr(instance, k) != d[k]: - raise errors.InvalidData( - u"Changing '{0}' for git repo is prohibited".format(k), - log_message=True - ) - - return d - - -class ChangesWhitelistRuleValidator(base.BasicValidator): - - single_schema = json_schema.changeswhitelistrule_single_schema - collection_schema = json_schema.changeswhitelistrule_collection_schema - - _blocked_for_update = ( - 'env_id', - ) - - @classmethod - def validate_update(self, data, instance): - d = self.validate_json(data) - for k in self._blocked_for_update: - if k in d and getattr(instance, k) != d[k]: - raise errors.InvalidData( - u"Changing '{0}' for white list is prohibited".format(k), - log_message=True - ) - - return d - - # TODO(dnikishov): investigate if there's a more simple way to do this - @classmethod - def validate_one_or_multiple(self, data): - d = self.validate_json(data) - if not isinstance(d, list): - d = [d] - for item in d: - self.validate_schema(item, self.single_schema) - - return d - - # This is required for inherited handlers to work - @classmethod - def validate_delete(self, *args, **kwargs): - pass - - -class GitRepoCollectionHandler(CollectionHandler): - collection = GitRepoCollection - validator = GitRepoValidator - - -class GitRepoHandler(SingleHandler): - single = GitRepo - validator = GitRepoValidator - - @handle_errors - @validate - @serialize - def GET(self, cluster_id, obj_id): - """:returns: JSONized REST object. - - :http: * 200 (OK) - * 404 (dashboard entry not found in db) - """ - self.get_object_or_404(objects.Cluster, cluster_id) - - obj = self.get_object_or_404(self.single, obj_id) - return self.single.to_json(obj) - - @handle_errors - @validate - @serialize - def PUT(self, cluster_id, obj_id): - """:returns: JSONized REST object. - - :http: * 200 (OK) - * 400 (invalid object data specified) - * 404 (object not found in db) - """ - obj = self.get_object_or_404(self.single, obj_id) - - data = self.checked_data( - self.validator.validate_update, - instance=obj - ) - self.single.update(obj, data) - return self.single.to_json(obj) - - def PATCH(self, cluster_id, obj_id): - """:returns: JSONized REST object. - - :http: * 200 (OK) - * 400 (invalid object data specified) - * 404 (object not found in db) - """ - return self.PUT(cluster_id, obj_id) - - @handle_errors - @serialize - def DELETE(self, cluster_id, obj_id): - """:returns: JSONized REST object. - - :http: * 204 (OK) - * 404 (object not found in db) - """ - d_e = self.get_object_or_404(self.single, obj_id) - self.single.delete(d_e) - raise self.http(204) - - -class ChangesWhitelistRuleHandler(SingleHandler): - single = ChangesWhitelistRule - validator = ChangesWhitelistRuleValidator - - -class ChangesWhitelistRuleCollectionHandler(CollectionHandler): - collection = ChangesWhitelistRuleCollection - validator = ChangesWhitelistRuleValidator - - @handle_errors - @validate - @serialize - def GET(self, env_id): - """:returns: JSONized REST object. - - :http: * 200 (OK) - * 404 (dashboard entry not found in db) - """ - env_id = int(env_id) - self.get_object_or_404(objects.Cluster, env_id) - rules = self.collection.get_by_env_id(env_id) - if rules: - rules = self.collection.to_list(rules) - return rules - - @handle_errors - @serialize - def POST(self, env_id): - """:returns: JSONized REST object. - - :http: * 201 (object successfully created) - * 400 (invalid object data specified) - * 409 (object with such parameters already exists) - """ - env_id = int(env_id) - data = self.checked_data( - validate_method=self.validator.validate_one_or_multiple - ) - for item in data: - item['env_id'] = env_id - - rules = self.collection.get_by_env_id(env_id) - existing_rules = [] - if rules: - rules = self.collection.to_list(rules) - existing_rules = filter( - lambda i: len(filter(lambda o: - i['fuel_task'] == o['fuel_task'] and - i['rule'] == o['rule'], - rules)) > 0, - data - ) - if existing_rules: - raise self.http(409, existing_rules) - - new_objs = [] - try: - for item in data: - new_objs.append(self.collection.create(item)) - except errors.CannotCreate as exc: - raise self.http(400, exc.message) - - raise self.http(201, self.collection.to_json(new_objs)) diff --git a/fuel_external_git/json_schema.py b/fuel_external_git/json_schema.py deleted file mode 100644 index 716d63f..0000000 --- a/fuel_external_git/json_schema.py +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -gitrepo_single_schema = { - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "GitRepo", - "description": "Serialized GitRepo object", - "type": "object", - "properties": { - "id": {"type": "number"}, - "repo_name": {"type": "string"}, - "env_id": {"type": "number"}, - "git_url": {"type": "string"}, - "ref": {"type": "string"}, - "user_key": {"type": "string"} - } -} - -gitrepo_collection_schema = { - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "GitRepo Collection", - "description": "Serialized GitRepo collection", - "type": "object", - "items": gitrepo_single_schema["properties"] -} - -changeswhitelistrule_single_schema = { - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "ChangesWhitelistRule", - "description": "Serialized ChangesWhitelistRule object", - "type": "object", - "properties": { - "id": {"type": "number"}, - "env_id": {"type": "number"}, - "rule": {"type": "string"}, - "fuel_task": {"type": "string"}, - } -} - -changeswhitelistrule_collection_schema = { - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "ChangesWhitelistRule Collection", - "description": "Serialized ChangesWhitelistRule collection", - "type": "object", - "items": changeswhitelistrule_single_schema["properties"] -} diff --git a/fuel_external_git/migrations/README b/fuel_external_git/migrations/README deleted file mode 100644 index 98e4f9c..0000000 --- a/fuel_external_git/migrations/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file diff --git a/fuel_external_git/migrations/__init__.py b/fuel_external_git/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/fuel_external_git/migrations/env.py b/fuel_external_git/migrations/env.py deleted file mode 100644 index 238c1d1..0000000 --- a/fuel_external_git/migrations/env.py +++ /dev/null @@ -1,73 +0,0 @@ - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging.config - -from alembic import context -import sqlalchemy - - -config = context.config -if config.get_main_option('table_prefix') is None: - config.set_main_option('table_prefix', '') -if config.config_file_name: - logging.config.fileConfig(config.config_file_name) -target_metadata = None - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - Calls to context.execute() here emit the given string to the - script output. - """ - context.configure( - url=config.get_main_option('sqlalchemy.url'), - version_table=config.get_main_option('version_table'), - literal_binds=True, - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - """ - connectable = sqlalchemy.engine_from_config( - config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=sqlalchemy.pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=target_metadata, - version_table=config.get_main_option('version_table'), - ) - - with context.begin_transaction(): - context.run_migrations() - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/fuel_external_git/migrations/script.py.mako b/fuel_external_git/migrations/script.py.mako deleted file mode 100644 index 43c0940..0000000 --- a/fuel_external_git/migrations/script.py.mako +++ /dev/null @@ -1,24 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} -branch_labels = ${repr(branch_labels)} -depends_on = ${repr(depends_on)} - -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -def upgrade(): - ${upgrades if upgrades else "pass"} - - -def downgrade(): - ${downgrades if downgrades else "pass"} diff --git a/fuel_external_git/migrations/versions/8736ad38ca31_add_whitelist_table.py b/fuel_external_git/migrations/versions/8736ad38ca31_add_whitelist_table.py deleted file mode 100644 index 4b121d6..0000000 --- a/fuel_external_git/migrations/versions/8736ad38ca31_add_whitelist_table.py +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""add whitelist table - -Revision ID: 8736ad38ca31 -Revises: adb78f70605d -Create Date: 2016-11-07 10:50:38.168018 - -""" - -# revision identifiers, used by Alembic. -revision = '8736ad38ca31' -down_revision = 'adb78f70605d' -branch_labels = None -depends_on = None - -from alembic import context -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.create_table( - table_prefix + 'changes_whitelist', - sa.Column('id', sa.Integer(), nullable=False, primary_key=True), - sa.Column('env_id', sa.Integer(), nullable=False), - sa.Column('rule', sa.String(255), - server_default='', nullable=False) - ) - - -def downgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.drop_table(table_prefix + 'changes_whitelist') diff --git a/fuel_external_git/migrations/versions/954d4c3a76be_add_unique_constaint_to_whitelist_rule_.py b/fuel_external_git/migrations/versions/954d4c3a76be_add_unique_constaint_to_whitelist_rule_.py deleted file mode 100644 index 13f46c5..0000000 --- a/fuel_external_git/migrations/versions/954d4c3a76be_add_unique_constaint_to_whitelist_rule_.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""add unique constaint to whitelist rule-task-env - -Revision ID: 954d4c3a76be -Revises: fc4f164a7b6c -Create Date: 2016-11-15 10:36:06.950577 - -""" - -# revision identifiers, used by Alembic. -revision = '954d4c3a76be' -down_revision = 'fc4f164a7b6c' -branch_labels = None -depends_on = None - -from alembic import context -from alembic import op - - -def upgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.create_unique_constraint('_env_id_rule_task_unique', - table_prefix + 'changes_whitelist', - ['env_id', 'rule', 'fuel_task']) - - -def downgrade(): - pass diff --git a/fuel_external_git/migrations/versions/__init__.py b/fuel_external_git/migrations/versions/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/fuel_external_git/migrations/versions/adb78f70605d_manage_fuel_node_flag.py b/fuel_external_git/migrations/versions/adb78f70605d_manage_fuel_node_flag.py deleted file mode 100644 index f9ffa5f..0000000 --- a/fuel_external_git/migrations/versions/adb78f70605d_manage_fuel_node_flag.py +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Manage fuel node flag - -Revision ID: adb78f70605d -Revises: d59114c46ac4 -Create Date: 2016-09-26 10:10:37.779555 - -""" - -# revision identifiers, used by Alembic. -revision = 'adb78f70605d' -down_revision = 'd59114c46ac4' -branch_labels = None -depends_on = None - -import sqlalchemy as sa - -from alembic import context -from alembic import op - - -def upgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.add_column( - table_prefix + 'repos', - sa.Column('manage_master', sa.Boolean(), nullable=True) - ) - - -def downgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.drop_column(table_prefix + 'repos', 'manage_master') diff --git a/fuel_external_git/migrations/versions/d59114c46ac4_change_constrains.py b/fuel_external_git/migrations/versions/d59114c46ac4_change_constrains.py deleted file mode 100644 index cd91861..0000000 --- a/fuel_external_git/migrations/versions/d59114c46ac4_change_constrains.py +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""change_constrains - -Revision ID: d59114c46ac4 -Revises: e3b840e64e53 -Create Date: 2016-08-26 14:33:57.385961 - -""" - -# revision identifiers, used by Alembic. -revision = 'd59114c46ac4' -down_revision = 'e3b840e64e53' -branch_labels = None -depends_on = None - -import sqlalchemy as sa - -from alembic import context -from alembic import op - - -def upgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.create_unique_constraint('_repo_name_unique', - table_prefix + 'repos', - ['repo_name']) - - op.alter_column(table_prefix + 'repos', - 'user_key', - type_=sa.UnicodeText(), - existing_type=sa.String(255)) - - -def downgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.drop_constraint('_repo_name_unique', - table_prefix + 'repos') - - op.alter_column(table_prefix + 'repos', - 'user_key', - existing_type=sa.UnicodeText(), - type_=sa.String(255)) diff --git a/fuel_external_git/migrations/versions/e3b840e64e53_init.py b/fuel_external_git/migrations/versions/e3b840e64e53_init.py deleted file mode 100644 index 9b42cc5..0000000 --- a/fuel_external_git/migrations/versions/e3b840e64e53_init.py +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Init - -Revision ID: e3b840e64e53 -Revises: -Create Date: 2016-08-09 16:59:36.504052 - -""" - -# revision identifiers, used by Alembic. -revision = 'e3b840e64e53' -down_revision = None -branch_labels = None -depends_on = None - -import sqlalchemy as sa - -from alembic import context -from alembic import op - - -def upgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.create_table( - table_prefix + 'repos', - sa.Column('id', sa.Integer(), nullable=False, primary_key=True), - sa.Column('repo_name', sa.Unicode(100), nullable=False), - sa.Column('env_id', sa.Integer(), nullable=False), - sa.Column('git_url', sa.String(255), - server_default='', nullable=False), - sa.Column('ref', sa.String(255), - server_default='', nullable=False), - sa.Column('user_key', sa.String(255), - server_default='', nullable=False), - sa.UniqueConstraint('env_id', name='_env_id_unique')) - - -def downgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.drop_table(table_prefix + 'repos') diff --git a/fuel_external_git/migrations/versions/fc4f164a7b6c_extended_whitelist_with_task_name.py b/fuel_external_git/migrations/versions/fc4f164a7b6c_extended_whitelist_with_task_name.py deleted file mode 100644 index 659e697..0000000 --- a/fuel_external_git/migrations/versions/fc4f164a7b6c_extended_whitelist_with_task_name.py +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""extended whitelist with task name - -Revision ID: fc4f164a7b6c -Revises: 8736ad38ca31 -Create Date: 2016-11-15 09:09:46.300987 - -""" - -# revision identifiers, used by Alembic. -revision = 'fc4f164a7b6c' -down_revision = '8736ad38ca31' -branch_labels = None -depends_on = None - -from alembic import context -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.add_column( - table_prefix + 'changes_whitelist', - sa.Column('fuel_task', sa.String, server_default='', nullable=False) - ) - - -def downgrade(): - table_prefix = context.config.get_main_option('table_prefix') - op.drop_column(table_prefix + 'changes_whitelist', 'fuel_task') diff --git a/fuel_external_git/models.py b/fuel_external_git/models.py deleted file mode 100644 index b581e7c..0000000 --- a/fuel_external_git/models.py +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import Boolean -from sqlalchemy import Column -from sqlalchemy import Integer -from sqlalchemy import String -from sqlalchemy import UnicodeText - -from nailgun.db.sqlalchemy.models.base import Base - - -class GitRepo(Base): - __tablename__ = 'fuel_external_git_repos' - id = Column(Integer, primary_key=True) - repo_name = Column(UnicodeText, nullable=False) - env_id = Column(Integer, unique=True, nullable=False) - git_url = Column(String(255), default='', server_default='', - nullable=False) - ref = Column(String(255), default='', server_default='', nullable=False) - user_key = Column(String(255), default='', server_default='', - nullable=False) - manage_master = Column(Boolean(), nullable=False) - - -class ChangesWhitelistRule(Base): - __tablename__ = 'fuel_external_git_changes_whitelist' - id = Column(Integer, primary_key=True) - env_id = Column(Integer, nullable=False) - rule = Column(String(255), server_default='', nullable=False) - fuel_task = Column(String(255), server_default='', nullable=False) diff --git a/fuel_external_git/objects.py b/fuel_external_git/objects.py deleted file mode 100644 index e9c82c0..0000000 --- a/fuel_external_git/objects.py +++ /dev/null @@ -1,180 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import shutil -import time - -from fuel_external_git import const -from fuel_external_git.models import ChangesWhitelistRule -from fuel_external_git.models import GitRepo - -from git import exc -from git import Repo - -from nailgun.consts import CLUSTER_STATUSES -from nailgun.db import db -from nailgun.logger import logger -from nailgun.objects import Cluster -from nailgun.objects import NailgunCollection -from nailgun.objects import NailgunObject -from nailgun.objects.serializers.base import BasicSerializer - - -class GitRepoSerializer(BasicSerializer): - fields = ( - "id", - "repo_name", - "env_id", - "git_url", - "ref", - "user_key", - "manage_master" - ) - - -class ChangesWhitelistRuleSerializer(BasicSerializer): - fields = ( - "id", - "env_id", - "rule", - "fuel_task" - ) - - -class GitRepo(NailgunObject): - model = GitRepo - serializer = GitRepoSerializer - - @classmethod - def get_by_cluster_id(self, cluster_id): - instance = db().query(self.model).\ - filter(self.model.env_id == cluster_id).first() - if instance is not None: - try: - instance.repo = Repo(os.path.join(const.REPOS_DIR, - instance.repo_name)) - except exc.NoSuchPathError: - logger.debug("Repo folder does not exist. Cloning repo") - self._create_key_file(instance.repo_name, instance.user_key) - if instance.user_key: - os.environ['GIT_SSH'] = \ - self._get_ssh_cmd(instance.repo_name) - - repo_path = os.path.join(const.REPOS_DIR, instance.repo_name) - repo = Repo.clone_from(instance.git_url, repo_path) - instance.repo = repo - return instance - - @classmethod - def create(self, data): - if not os.path.exists(const.REPOS_DIR): - os.mkdir(const.REPOS_DIR) - repo_path = os.path.join(const.REPOS_DIR, data['repo_name']) - if os.path.exists(repo_path): - logger.debug('Repo directory exists. Removing...') - shutil.rmtree(repo_path) - - user_key = data.get('user_key', '') - if user_key: - self._create_key_file(data['repo_name'], user_key) - os.environ['GIT_SSH'] = self._get_ssh_cmd(data['repo_name']) - repo = Repo.clone_from(data['git_url'], repo_path) - - instance = super(GitRepo, self).create(data) - instance.repo = repo - return instance - - @classmethod - def update(self, instance, data): - super(GitRepo, self).update(instance, data) - if 'user_key' in data: - instance = GitRepo.get_by_cluster_id(instance.env_id) - self._create_key_file(instance.repo_name, instance.user_key) - - @classmethod - def checkout(self, instance): - fetch_file = os.path.join( - const.REPOS_DIR, - instance.repo_name, - '.git/FETCH_HEAD' - ) - if os.path.exists(fetch_file): - current_ts = time.time() - cluster = Cluster.get_by_uid(instance.env_id) - last_fetch = os.stat(fetch_file).st_mtime - if cluster.status != CLUSTER_STATUSES.deployment and \ - current_ts - last_fetch < const.REPO_TTL: - return - - logger.debug("Repo TTL exceeded. Fetching code...") - git_shell_env = {} - if instance.user_key: - ssh_cmd = self._get_ssh_cmd(instance.repo_name) - - if not os.path.exists(self._get_key_path(instance.repo_name)): - logger.debug('Key file does not exist. Creating...') - self._create_key_file(instance.repo_name) - - logger.debug("Updating ENV with ssh command") - git_shell_env['GIT_SSH'] = ssh_cmd - - with instance.repo.git.custom_environment(**git_shell_env): - commit = instance.repo.remotes.origin.fetch(refspec=instance.ref) - commit = commit[0].commit - instance.repo.head.reference = commit - instance.repo.head.reset(index=True, working_tree=True) - - @classmethod - def _create_key_file(self, repo_name, data): - key_path = self._get_key_path(repo_name) - with open(key_path, 'w') as key_file: - key_file.write(data) - os.chmod(key_path, 0o600) - - @classmethod - def _get_key_path(self, repo_name): - return os.path.join(const.REPOS_DIR, repo_name + '.key') - - @classmethod - def _get_ssh_cmd(self, repo_name): - key_path = self._get_key_path(repo_name) - git_ssh_file = os.path.join(const.REPOS_DIR, repo_name + '.sh') - with open(git_ssh_file, 'w') as ssh_wrap: - ssh_wrap.write("#!/bin/bash\n") - ssh_wrap.write(( - "exec /usr/bin/ssh " - "-o UserKnownHostsFile=/dev/null " - "-o StrictHostKeyChecking=no " - "-i {0} \"$@\"".format(key_path) - )) - os.chmod(git_ssh_file, 0o755) - return git_ssh_file - - -class GitRepoCollection(NailgunCollection): - single = GitRepo - - -class ChangesWhitelistRule(NailgunObject): - model = ChangesWhitelistRule - serializer = ChangesWhitelistRuleSerializer - - -class ChangesWhitelistRuleCollection(NailgunCollection): - single = ChangesWhitelistRule - - @classmethod - def get_by_env_id(self, env_id): - whitelist = filter(lambda r: r.env_id == env_id, - ChangesWhitelistRuleCollection.all()) - return whitelist diff --git a/fuel_external_git/settings.py b/fuel_external_git/settings.py deleted file mode 100644 index 3f0d077..0000000 --- a/fuel_external_git/settings.py +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# tst -import os -import yaml - -from nailgun.logger import logger - - -class GitExtensionSettings(object): - def __init__(self): - settings_files = [] - project_path = os.path.dirname(__file__) - project_settings_file = os.path.join(project_path, 'settings.yaml') - settings_files.append(project_settings_file) - settings_files.append('/etc/git-exension-settings.yaml') - settings_files.append('/etc/nailgun/git-exension-settings.yaml') - self.config = {} - for sf in settings_files: - try: - logger.debug("Trying to read config file %s" % sf) - with open(sf) as custom_config: - self.config.update(yaml.load(custom_config.read())) - except Exception as e: - logger.error("Error while reading config file %s: %s" % - (sf, str(e))) diff --git a/fuel_external_git/settings.yaml b/fuel_external_git/settings.yaml deleted file mode 100644 index 29235e6..0000000 --- a/fuel_external_git/settings.yaml +++ /dev/null @@ -1,69 +0,0 @@ -master_mapping: - master_config.yaml: - resource: master_config - driver: fuel_external_git.drivers.yaml_driver.YamlConfig - path: '' -resource_mapping: - ceilometer-api-paste.ini: - path: /etc/ceilometer/api-paste.ini - resource: ceilometer_api_paste_ini - ceilometer.conf: - path: /etc/ceilometer/ceilometer.conf - resource: ceilometer - cinder-api-paste.ini: - path: /etc/cinder/api-paste.ini - resource: cinder_api_paste_ini - cinder.conf: - path: /etc/cinder/cinder.conf - resource: cinder - dhcp_agent.ini: - path: /etc/neutron/dhcp_agent.ini - resource: neutron_dhcp_agent_config - glance-api.conf: - path: /etc/glance/glance-api.conf - resource: glance_api - glance-cache.conf: - path: /etc/glance/glance-cache.conf - resource: glance_cache - glance-glare.conf: - path: /etc/glance/glance-glare.conf - resource: glare_config - glance-registry.conf: - path: /etc/glance/glance-registry.conf - resource: glance_registry - heat-api-paste.ini: - path: /etc/heat/api-paste.ini - resource: heat_api_paste_ini - heat.conf: - path: /etc/heat/heat.conf - resource: heat - keystone.conf: - path: /etc/keystone/keystone.conf - resource: keystone_config - l3_agent.ini: - path: /etc/neutron/l3_agent.ini - resource: neutron_l3_agent_config - metadata_agent.ini: - path: /etc/neutron/metadata_agent.ini - resource: neutron_metadata_agent_config - ml2_conf.ini: - path: /etc/neutron/plugins/ml2/ml2_conf.ini - resource: neutron_plugin_ml2 - neutron-api-paste.ini: - path: /etc/neutron/api-paste.ini - resource: neutron_api_config - neutron.conf: - path: /etc/neutron/neutron.conf - resource: neutron_config - nova-api-paste.ini: - path: /etc/nova/api-paste.ini - resource: nova_paste_api_ini - nova.conf: - path: /etc/nova/nova.conf - resource: nova_config - openvswitch_agent.ini: - path: /etc/neutron/plugins/ml2/openvswitch_agent.ini - resource: neutron_agent_ovs - sriov_agent.ini: - path: /etc/neutron/plugins/ml2/sriov_agent.ini - resource: neutron_sriov_agent_config diff --git a/fuel_external_git/tests/__init__.py b/fuel_external_git/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/fuel_external_git/tests/base.py b/fuel_external_git/tests/base.py deleted file mode 100644 index 7a5a379..0000000 --- a/fuel_external_git/tests/base.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -from oslotest import base - -from fuel_external_git import settings - - -class TestCase(base.BaseTestCase): - - """Test case base class for all unit tests.""" - def setUp(self): - super(TestCase, self).setUp() - self.config = settings.GitExtensionSettings().config - self.cfg_sample_dir = os.path.join(os.path.dirname(__file__), 'cfgs') diff --git a/fuel_external_git/tests/cfgs/controller_configs/nova.conf b/fuel_external_git/tests/cfgs/controller_configs/nova.conf deleted file mode 100644 index d2a641b..0000000 --- a/fuel_external_git/tests/cfgs/controller_configs/nova.conf +++ /dev/null @@ -1,5 +0,0 @@ -[DEFAULT] -global_param = global_param -global_param_to_override = override_from_role -role_param = role_param -role_param_to_override = role_param_to_override diff --git a/fuel_external_git/tests/cfgs/node_1_configs/nova.conf b/fuel_external_git/tests/cfgs/node_1_configs/nova.conf deleted file mode 100644 index a2020e1..0000000 --- a/fuel_external_git/tests/cfgs/node_1_configs/nova.conf +++ /dev/null @@ -1,3 +0,0 @@ -[DEFAULT] -role_param_to_override = override_from_node -node_param = node_param diff --git a/fuel_external_git/tests/cfgs/nova.conf b/fuel_external_git/tests/cfgs/nova.conf deleted file mode 100644 index c623083..0000000 --- a/fuel_external_git/tests/cfgs/nova.conf +++ /dev/null @@ -1,3 +0,0 @@ -[DEFAULT] -global_param = global_param -global_param_to_override = global_param_to_override diff --git a/fuel_external_git/tests/cfgs/overrides.yaml b/fuel_external_git/tests/cfgs/overrides.yaml deleted file mode 100644 index c148405..0000000 --- a/fuel_external_git/tests/cfgs/overrides.yaml +++ /dev/null @@ -1,8 +0,0 @@ -nodes: - '1': node_1_configs - '2': node_2_configs -roles: - 'cinder': 'cinder_configs' - 'compute': 'compute_configs' - 'controller': 'controller_configs' - 'primary-controller': 'controller_configs' diff --git a/fuel_external_git/tests/test_utils.py b/fuel_external_git/tests/test_utils.py deleted file mode 100644 index 162902a..0000000 --- a/fuel_external_git/tests/test_utils.py +++ /dev/null @@ -1,97 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy - -from fuel_external_git.tests import base -from fuel_external_git import utils - - -class TestUtils(base.TestCase): - mapping = { - 'ceilometer-api-paste.ini': { - 'resource': 'ceilometer_api_paste_ini', - 'path': '/etc/ceilometer/api-paste.ini', - }, - 'ceilometer.conf': { - 'resource': 'ceilometer', - 'path': '/etc/ceilometer/ceilometer.conf', - }, - 'nova.conf': { - 'path': '/etc/nova/nova.conf', - 'resource': 'nova_config', - } - } - - def test_extension_list(self): - ext_list = utils.get_file_exts_list(self.mapping) - self.assertEqual(ext_list, set(['ini', 'conf'])) - - def test_deep_merge_two_empy(self): - a = {} - b = {} - utils.deep_merge(a, b) - self.assertEqual(a, {}) - - def test_deep_merge_one_empy(self): - sample_dict = { - 'a': {'b': {'c': 'd'}}, - 'e': {'f': {'g': 'h'}}, - } - new_dict = copy.deepcopy(sample_dict) - utils.deep_merge(new_dict, {}) - self.assertEqual(new_dict, sample_dict) - - new_dict = {} - utils.deep_merge(new_dict, sample_dict) - self.assertEqual(new_dict, sample_dict) - - def test_merge_two_discts(self): - a = { - 'a': {'b': {'c': 'd'}}, - 'e': {'f': {'g': 'h'}}, - } - - b = { - 'x': {'b': {'c': 'd'}}, - 'y': {'f': {'g': 'h'}}, - } - - result = { - 'a': {'b': {'c': 'd'}}, - 'e': {'f': {'g': 'h'}}, - 'x': {'b': {'c': 'd'}}, - 'y': {'f': {'g': 'h'}}, - } - - utils.deep_merge(a, b) - self.assertEqual(a, result) - - def test_config_hash_dir_ne(self): - file_dir = '/a/b/c' - resource_mapping = {} - cfg = utils.get_config_hash(file_dir, resource_mapping) - self.assertEqual(cfg, {}) - - def test_config_hash(self): - res = { - 'nova_config': { - 'DEFAULT/global_param': { - 'value': 'global_param' - }, - 'DEFAULT/global_param_to_override': { - 'value': 'global_param_to_override' - } - } - } - actual_res = utils.get_config_hash(self.cfg_sample_dir, self.mapping) - self.assertEqual(res, actual_res) diff --git a/fuel_external_git/utils.py b/fuel_external_git/utils.py deleted file mode 100644 index 2bb4f43..0000000 --- a/fuel_external_git/utils.py +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -from oslo_utils import importutils - -from nailgun.logger import logger - - -def get_file_exts_list(resource_mapping): - return set(map(lambda key: key.split('.')[-1], resource_mapping.keys())) - - -def get_config_hash(file_dir, resource_mapping, exts=['conf']): - res = {} - if not os.path.isdir(file_dir): - logger.debug( - "Directory {} not found. Returning emty dict".format(file_dir)) - return {} - - conf_files = [conf for conf in os.listdir(file_dir) - if conf.split('.')[-1] in exts] - - for conf_file in conf_files: - if conf_file in resource_mapping.keys(): - drv = resource_mapping[conf_file].get( - 'driver', - 'fuel_external_git.drivers.openstack_config.OpenStackConfig' - ) - drv_class = importutils.import_class(drv) - config = drv_class( - os.path.join(file_dir, conf_file), - resource_mapping[conf_file]['resource'] - ) - deep_merge(res, config.to_config_dict()) - return res - - -def deep_merge(dct, merge_dct): - for k, v in merge_dct.iteritems(): - if (k in dct and isinstance(dct[k], dict)): - deep_merge(dct[k], merge_dct[k]) - else: - dct[k] = merge_dct[k] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d7a7441..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -GitPython==2.0.8 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c010d77..0000000 --- a/setup.cfg +++ /dev/null @@ -1,38 +0,0 @@ -[metadata] -name = fuel_external_git -summary = Nailgun extension which uses git repo for config files -description-file = README.md -author = Dmitry Ukov -author-email = dukov@mirantis.com -home-page = http://mirantis.com -classifier = - Development Status :: 3 - Alpha - Environment :: OpenStack - Intended Audience :: Information Technology - Intended Audience :: System Administrators - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 - -[files] -packages = - fuel_external_git - -[entry_points] -nailgun.extensions: - fuel_external_git = fuel_external_git.extension:ExternalGit -fuelclient: - gitrepo_list = fuel_external_git.fuelclient_gitrepo:GitRepoList - gitrepo_create = fuel_external_git.fuelclient_gitrepo:AddRepo - gitrepo_delete = fuel_external_git.fuelclient_gitrepo:DeleteRepo - gitrepo_update = fuel_external_git.fuelclient_gitrepo:UpdateRepo - audit_enforce = fuel_external_git.fuelclient_audit:Audit - audit_noop = fuel_external_git.fuelclient_audit:AuditRun - audit_list_outofsync = fuel_external_git.fuelclient_audit:OutOfSyncResources - audit_whitelist_show = fuel_external_git.fuelclient_audit:WhitelistRulesShow - audit_whitelist_add = fuel_external_git.fuelclient_audit:WhitelistRuleAdd - audit_whitelist_load_fromfile = fuel_external_git.fuelclient_audit:WhitelistRuleAddFromFile - audit_whitelist_delete = fuel_external_git.fuelclient_audit:WhitelistRuleDelete - audit_whitelist_load_all = fuel_external_git.fuelclient_audit:WhitelistRuleAddAll diff --git a/setup.py b/setup.py deleted file mode 100644 index 056c16c..0000000 --- a/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT -import setuptools - -# In python < 2.7.4, a lazy loading of package `pbr` will break -# setuptools if some other modules registered functions in `atexit`. -# solution from: http://bugs.python.org/issue15881#msg170215 -try: - import multiprocessing # noqa -except ImportError: - pass - -setuptools.setup( - setup_requires=['pbr'], - pbr=True) diff --git a/specs/fuel-nailgun-extension-iac.spec b/specs/fuel-nailgun-extension-iac.spec deleted file mode 100644 index ec447b1..0000000 --- a/specs/fuel-nailgun-extension-iac.spec +++ /dev/null @@ -1,45 +0,0 @@ -Name: fuel-nailgun-extension-iac -Version: 9.0.0 -Release: 2%{?dist}~mos0 -Summary: Infrastructure as Code extension for Fuel -License: Apache-2.0 -Url: https://git.openstack.org/cgit/openstack/fuel-nailgun-extension-iac/ -Source0: %{name}-%{version}.tar.gz -BuildArch: noarch - -BuildRequires: python-devel -BuildRequires: python-pbr -BuildRequires: python-setuptools - -Requires: python-pbr -Requires: GitPython -Requires: git - -%description -Nailgun extension that generates deployment data based on configuration files -published in external git repository - -%prep -%setup -q -c -n %{name}-%{version} - -%build -export OSLO_PACKAGE_VERSION=%{version} %py2_build - -%install -export OSLO_PACKAGE_VERSION=%{version} %py2_install - -%files -%license LICENSE -%{python2_sitelib}/fuel_external_git -%{python2_sitelib}/*.egg-info - -%changelog -* Fri Nov 18 2016 Dmitrii Nikishov - 9.0.0-2.el7~mos0 -- Added git to dependencies - -* Wed Nov 09 2016 Ivan Udovichenko - 9.0.0-1.el7~mos0 -- Rebuild for MOS 9.2 -- LP#1640433 - -* Fri Sep 23 2016 Vladimir Maliaev - 9.0.0 -- Initial package. diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index bb31766..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - -hacking<0.11,>=0.10.0 - -coverage>=3.6 -discover -oslo.db[fixtures,mysql,postgresql] -oslotest>=1.10.0 # Apache-2.0 -testrepository>=0.0.18 -testscenarios>=0.4 -testtools>=1.4.0 -requests-mock -python-fuelclient -os-testr>=0.4.1 # Apache-2.0 -mock>=1.2 # BSD diff --git a/tools/test-setup.sh b/tools/test-setup.sh deleted file mode 100755 index 07a0785..0000000 --- a/tools/test-setup.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -xe - -# This script will be run by OpenStack CI before unit tests are run, -# it sets up the test system as needed. -# Developers should setup their test systems in a similar way. - -# This setup needs to be run as a user that can run sudo. - -# The root password for the MySQL database; pass it in via -# MYSQL_ROOT_PW. -DB_ROOT_PW=${MYSQL_ROOT_PW:-insecure_slave} - -# This user and its password are used by the tests, if you change it, -# your tests might fail. -DB_USER=openstack_citest -DB_PW=openstack_citest - -sudo -H mysqladmin -u root password $DB_ROOT_PW - -# It's best practice to remove anonymous users from the database. If -# a anonymous user exists, then it matches first for connections and -# other connections from that host will not work. -sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e " - DELETE FROM mysql.user WHERE User=''; - FLUSH PRIVILEGES; - GRANT ALL PRIVILEGES ON *.* - TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;" - -# Now create our database. -mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " - SET default_storage_engine=MYISAM; - DROP DATABASE IF EXISTS openstack_citest; - CREATE DATABASE openstack_citest CHARACTER SET utf8;" - -# Same for PostgreSQL -# The root password for the PostgreSQL database; pass it in via -# POSTGRES_ROOT_PW. -DB_ROOT_PW=${POSTGRES_ROOT_PW:-insecure_slave} - -# Setup user -root_roles=$(sudo -H -u postgres psql -t -c " - SELECT 'HERE' from pg_roles where rolname='$DB_USER'") -if [[ ${root_roles} == *HERE ]];then - sudo -H -u postgres psql -c "ALTER ROLE $DB_USER WITH SUPERUSER LOGIN PASSWORD '$DB_PW'" -else - sudo -H -u postgres psql -c "CREATE ROLE $DB_USER WITH SUPERUSER LOGIN PASSWORD '$DB_PW'" -fi - -# Store password for tests -cat << EOF > $HOME/.pgpass -*:*:*:$DB_USER:$DB_PW -EOF -chmod 0600 $HOME/.pgpass - -# Now create our database -psql -h 127.0.0.1 -U $DB_USER -d template1 -c "DROP DATABASE IF EXISTS openstack_citest" -createdb -h 127.0.0.1 -U $DB_USER -l C -T template0 -E utf8 openstack_citest diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 4f55efc..0000000 --- a/tox.ini +++ /dev/null @@ -1,41 +0,0 @@ -[tox] -minversion = 1.6 -envlist = py27,pep8 -skipsdist = True - -[base] -NAILGUN_REPO = git+https://github.com/openstack/fuel-web.git -NAILGUN_CONFIG = {toxinidir}/nailgun-test-settings.yaml -NAILGUN_BRANCH={env:ZUUL_BRANCH:stable/mitaka} - -[testenv] -usedevelop = True -install_command = pip install -U {opts} {packages} -setenv = - VIRTUAL_ENV={envdir} - PYTHONWARNINGS=ignore:Unmanaged access of declarative attribute __tablename__ from non-mapped class ModelMixin - OS_TEST_DBAPI_ADMIN_CONNECTION=mysql+pymysql://openstack_citest:openstack_citest@localhost/;postgresql://openstack_citest:openstack_citest@localhost/postgres;sqlite:///testdb -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt - -e{[base]NAILGUN_REPO}@{[base]NAILGUN_BRANCH}#egg=nailgun[test]&subdirectory=nailgun -commands = py.test -v --junit-xml {toxinidir}/extension.xml {posargs} - -[testenv:pep8] -commands = flake8 - -[testenv:venv] -commands = {posargs} - -[testenv:cover] -commands = python setup.py test --coverage --testr-args='{posargs}' - -[testenv:debug] -commands = oslo_debug_helper {posargs} - -[flake8] -# E123, E125 skipped as they are invalid PEP-8. - -show-source = True -ignore = E123,E125 -builtins = _ -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build