diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 7cb539c4..00000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[run] -branch = True -source = monasca_log_api -omit = monasca_log_api/tests/* - -[report] -ignore_errors = True \ No newline at end of file diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 1a636224..00000000 --- a/.gitignore +++ /dev/null @@ -1,39 +0,0 @@ -*.py[co] -*~ -doc/build/* -dist -build -cover/ -.coverage -.coverage.* -*.egg -*.egg-info -.eggs/ -.stestr/ -.tox/ -MANIFEST -AUTHORS -ChangeLog -monasca-log-api.log -etc/monasca/*.sample - -*.swp -*.iml -.DS_Store -.cache -.classpath -.idea -.project -.target/ -java/debs/* -.settings/ -target -test-output/ -logs/ -*config*.yml -db/config.yml -.venv -.vagrant - -# Logs from devstack directory should not be committed -devstack/*.log diff --git a/.stestr.conf b/.stestr.conf deleted file mode 100644 index 6107277d..00000000 --- a/.stestr.conf +++ /dev/null @@ -1,4 +0,0 @@ -[DEFAULT] -test_path=$OS_TEST_PATH -top_dir=./ -group_regex=monasca_log_api\.tests(?:\.|_)([^_]+) diff --git a/.zuul.yaml b/.zuul.yaml deleted file mode 100644 index a41df255..00000000 --- a/.zuul.yaml +++ /dev/null @@ -1,62 +0,0 @@ -- project: - templates: - - check-requirements - - openstack-cover-jobs - - openstack-lower-constraints-jobs - - openstack-python3-victoria-jobs - - publish-openstack-docs-pti - - release-notes-jobs-python3 - check: - jobs: - - monasca-tempest-log-oldapi-python3-influxdb - - build-monasca-docker-image - gate: - queue: monasca - jobs: - - monasca-tempest-log-oldapi-python3-influxdb - post: - jobs: - - publish-monasca-log-api-docker-image - periodic: - jobs: - - publish-monasca-log-api-docker-image - release: - jobs: - - publish-monasca-log-api-docker-image - -- job: - name: publish-monasca-log-api-docker-image - parent: build-monasca-docker-image - post-run: playbooks/docker-publish.yml - required-projects: - - openstack/monasca-common - vars: - publisher: true - secrets: - - doker_hub_login_log_api - -- secret: - name: doker_hub_login_log_api - data: - user: !encrypted/pkcs1-oaep - - mR5DkiEi0lGjpUHgFYlt779KdX0UhPK7+/uJIiawmM5ASJSx90s/DMMDUrPbGc9cpJrOH - zRxBRryUrXuLGpKwQvLAC8uQC3rd9rGrlTVwjszl3nDNFYxKJ3tBMjppG6dBTIo58v6Go - Xo9GdNmaeS3jh6wh9hUqmMlHF4fzaX49GUpZyxTwufA0Y4h/pbq1r7ImbOCLiNUEwxWqH - dPJ9aJMQ5u8qus2r8cs7EHU/KAsFfPEteNtOz3egDSb1/SFYs5oI0VeIs4RvidHe6M56v - vEYx2KjzKKIE4s5TAiU8jU2Qs0RROv+6s7Y23ciqOfFBhVKJg+0PAoge1AwmL6g3FNrW0 - sy0uEslPltkbF3Pae9zFe0VU6ZIxYC62kAIBEJj96zc0TIfFYHx4NRbIvkoKopnun7dqF - 6hg5k8lZ1kN5RMT2nPNemj9esh0eqCBvLjf4ljy7veYLN9M/GSKSP5A+2chBWFxloWuwB - 7PRf9GbDbHA+H+Y6WASK38haHlo9zsoi/D7oJK02LHTwgj4ED3qN34Z9zCVBBr1jN3TUH - 9aVOsm+OT1OsUszLQ3BM1HqBZLszrOxuPoqDQ5CURrxEtzrSnX+fsoLyv1tzT/vVnrbAT - /zJxBXGiiE5/3H0isFkK6GUVwsmnLUFYCq5ye6NyLpwdWfTEx6eQDlO5OIE/Qo= - password: !encrypted/pkcs1-oaep - - DALBrHoOI3iZAKArnbPL59d3Mk1OHeDNREq9CJXs601l70AwPmNnK0y1XixbUoAmB8y9C - hkadR+a2s8Lc2+xCzRvr+4tt8vZ+gjgl77WTs43mpb+DfssdP6s5847t6cnE1WKI7GgCS - 4as41aOz4HDIdLG6Oe5Jq8HYDx+qcBZhQaeCzObLnbmxdDop6SkpjfziV8+TWxag56joE - V7masPLtYofzQpqB0hRhD3RF7T5fW8uOUlKKzSKhjYuwul1sbOKn6izjufxQs0iimhJpI - HkWCPSj9emr889BfO2DR9/OvTjwSdWZmEtup3RAyWcd6JUESa3oLkG1OFqB0vp6hgx9/F - zFXhBxtEyvs1Odj1ecudr7l+zO4kPVG9cIMHHzd7Tlg1XX2O1/+houVmXd3I1GtxwRYZa - B3Q+yecTLleR1coMM4nZgGtbiky5q6zcttTQCxeUEeCpTUWbxhnLPZyXV+pdFMxsLeUjP - 9GcnjuNCgOKHYHyHcXwDoytsqgoTaAS4O+FR5tCmc9dREWsaRCd0dBMy6Xx+ynVJE/1jp - XwtKG/yaFdHyTXwXGQ6ZlW9XnesBejLmetZHx8j7atiMOtKqmM4+N9i0O+1wbl3A2Glxs - K1orRrDJBdLNYVdczq/TUvzl4XBXi+36f1i6auL/WqHUmQy62w1S9WvaqGzEKM= diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index dafe5518..00000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,21 +0,0 @@ -If you would like to contribute to the development of OpenStack, -you must follow the steps documented at: - - http://docs.openstack.org/infra/manual/developers.html#getting-started - -Once those steps have been completed, changes to OpenStack -should be submitted for review via the Gerrit tool, following -the workflow documented at: - - http://docs.openstack.org/infra/manual/developers.html#development-workflow - -Pull requests submitted through GitHub will be ignored. - -Bugs should be filed on Storyboard (using tag **bug**), not GitHub: - - https://storyboard.openstack.org/#!/project/869 - -Additionally, specific guidelines for contributing to Monasca-Log-Api may be found in -Monasca-Log-Api's Documentation: - - https://docs.openstack.org/monasca-log-api/latest/contributor/ diff --git a/HACKING.rst b/HACKING.rst deleted file mode 100644 index 3fb9088d..00000000 --- a/HACKING.rst +++ /dev/null @@ -1,5 +0,0 @@ -monasca-log-api Style Commandments -================================== - -- Step 1: Read the OpenStack Style Commandments https://docs.openstack.org/hacking/latest/ -- Step 2: Read on diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 67db8588..00000000 --- a/LICENSE +++ /dev/null @@ -1,175 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. diff --git a/README.deprecated.md b/README.deprecated.md deleted file mode 100644 index f109f85e..00000000 --- a/README.deprecated.md +++ /dev/null @@ -1,193 +0,0 @@ -Team and repository tags -======================== - -[![Team and repository tags](https://governance.openstack.org/badges/monasca-log-api.svg)](https://governance.openstack.org/reference/tags/index.html) - - - -# Overview - -`monasca-log-api` is a RESTful API server that is designed with a layered architecture [layered architecture](http://en.wikipedia.org/wiki/Multilayered_architecture). - -The full API Specification can be found in [documentation/monasca-log-api-spec.md](documentation/monasca-log-api-spec.md) - -## Monasca-log-api Python - -### Installation - -To install the python api implementation, git clone the source and run the -following command: -```sh - sudo python setup.py install -``` - -### Configuration - -If it installs successfully, you will need to make changes to the following -two files to reflect your system settings, especially where kafka server is -located:: - -```sh - /etc/monasca/log-api-config.conf - /etc/monasca/log-api-config.ini - /etc/monasca/log-api-logging.conf -``` - -Once the configurations are modified to match your environment, you can start -up the server using either Gunicorn or Apache. - -### Start the Server -- for Gunicorn - -The server can be run in the foreground, or as daemons: - -Running the server in foreground mode with Gunicorn: - -```sh - gunicorn -k eventlet --worker-connections=2000 --backlog=1000 - --paste /etc/monasca/log-api-config.ini -``` - -Running the server as daemons with Gunicorn: - -```sh - gunicorn -k eventlet --worker-connections=2000 --backlog=1000 - --paste /etc/monasca/log-api-config.ini -D -``` - -### Start the Server -- for Apache - -To start the server using Apache: create a modwsgi file, -create a modwsgi config file, and enable the wsgi module -in Apache. - -The modwsgi conf file may look something like this, and the site will need to be enabled: - -```sh - Listen myhost:8082 - Listen 127.0.0.1:8082 - - - WSGIDaemonProcess log-api processes=4 threads=4 socket-timeout=120 user=log group=log python-path=/usr/local/lib/python2.7/site-packages - WSGIProcessGroup log-api - WSGIApplicationGroup log-api - WSGIScriptAlias / /var/www/log/log-api.wsgi - - ErrorLog /var/log/log-api/wsgi.log - LogLevel info - CustomLog /var/log/log-api/wsgi-access.log combined - - - Options Indexes FollowSymLinks MultiViews - Require all granted - AllowOverride None - Order allow,deny - allow from all - LimitRequestBody 102400 - - - SetEnv no-gzip 1 - - - -``` - -The wsgi file may look something like this: - -```sh - from monasca_log_api.server import get_wsgi_app - - application = get_wsgi_app(config_base_path='/etc/monasca') -``` - -## Testing - -### Commandline run -To check the server from the commandline: - -```sh - python server.py -``` - -### PEP8 guidelines -To check if the code follows python coding style, run the following command -from the root directory of this project: - -```sh - tox -e pep8 -``` - -### Unit Tests -To run all the unit test cases, run the following command from the root -directory of this project: - -```sh - tox -e py27 - tox -e py35 -``` - -### Coverage -To generate coverage results, run the following command from the root -directory of this project: - -```sh - tox -e cover -``` - -### Building - -To build an installable package, run the following command from the root -directory of this project: - -```sh - python setup.py sdist -``` - -### Documentation - -To generate documentation, run the following command from the root -directory of this project: - -```sh - tox -e docs -``` - -That will create documentation under build folder relative to root of the -project. - -## Architectural layers - -Requests flow through the following architectural layers from top to bottom: - -* Resource - * Serves as the entrypoint into the service. - * Responsible for handling web service requests, and performing structural request validation. -* Application - * Responsible for providing application level implementations for specific use cases. -* Domain - * Contains the technology agnostic core domain model and domain service definitions. - * Responsible for upholding invariants and defining state transitions. -* Infrastructure - * Contains technology specific implementations of domain services. - -## Documentation - -* API Specification: [/documentation/monasca-log-api-spec.md](/documentation/monasca-log-api-spec.md). -* Kafka communication: [/documentation/monasca-log-api-kafka.md](/documentation/monasca-log-api-kafka.md). -* API Monitoring: [/documentation/monasca-log-api-metrics.md](/documentation/monasca-log-api-metrics.md). - -# License - - # Copyright 2015 kornicameister@gmail.com - # Copyright 2015-2017 FUJITSU LIMITED - # - # Licensed under the Apache License, Version 2.0 (the "License"); you may - # not use this file except in compliance with the License. You may obtain - # a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - # License for the specific language governing permissions and limitations - # under the License. diff --git a/README.rst b/README.rst index 762e301d..c00729d0 100644 --- a/README.rst +++ b/README.rst @@ -1,73 +1,17 @@ -======================== -Team and repository tags -======================== - -.. image:: https://governance.openstack.org/tc/badges/monasca-log-api.svg - :target: https://governance.openstack.org/tc/reference/tags/index.html - -.. Change things from this point on - OpenStack Monasca-Log-Api ========================= -.. important:: +This project is no longer maintained. + +The contents of this repository are still available in the Git +source code management system. To see the contents of this +repository before it reached its end of life, please check out the +previous commit with "git checkout HEAD^1". - This API is deprecated. Last maintained release is OpenStack Train. - Please use `Monasca API `_ - for newer versions. This repository is kept only for maintenance purposes. +This API is deprecated. Last maintained release is OpenStack Train. +Please use `Monasca API `_ +for newer versions. This repository is kept only for maintenance purposes. -OpenStack Monasca-Log-Api provides RESTful Api to collect logs -from the OpenStack cloud. - -OpenStack Monasca-Log-Api is distributed under the terms of the Apache -License, Version 2.0. The full terms and conditions of this -license are detailed in the LICENSE file. - -Api ---- - -To learn how to use Monasca-Log-Api, consult the documentation -available online at: - -* `Api Guide `_ -* `Api Ref `_ - -For more information on OpenStack Apis, SDKs and CLIs, -please see: - -* `OpenStack Application Development `_ -* `OpenStack Developer Documentation `_ - -Developers ----------- - -For information on how to contribute to Monasca-Log-Api, please see the -contents of the CONTRIBUTING.rst. - -Any new code must follow the development guidelines detailed -in the HACKING.rst file, and pass all unit tests as well as linters. - -Further developer focused documentation is available at: - -* `Monasca-Log-Api `_ - -Operators ---------- - -To learn how to deploy and configure OpenStack Monasca-Log-Api, consult the -documentation available online at: - -* `Installation `_ -* `Configuration `_ - -Bug tracking ------------- - -In the unfortunate event that bugs are discovered, they should -be reported to the appropriate bug tracker. If you obtained -the software from a 3rd party operating system vendor, it is -often wise to use their own bug tracker for reporting problems. -In all other cases use the master OpenStack bug tracker, -available at: - -* `Storyboard `_ +For any further questions, please email +openstack-discuss@lists.openstack.org or join #openstack-monasca on +Freenode. diff --git a/api-guide/source/conf.py b/api-guide/source/conf.py deleted file mode 100644 index ca432ea1..00000000 --- a/api-guide/source/conf.py +++ /dev/null @@ -1,252 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Key Manager API documentation build configuration file -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.6' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'openstackdocstheme' -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -source_encoding = 'utf-8' - -# The master toctree document. -master_doc = 'index' - -# General details about project -openstackdocs_repo_name = 'openstack/monasca-log-api' -openstackdocs_auto_name = False -project = 'Monasca Log API Guide' -openstackdocs_bug_project = 'monasca-log-api' -openstackdocs_bug_tag = 'api-guide' -copyright = '2014, OpenStack Foundation' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'native' - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'openstackdocs' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# A shorter title for the navigation bar. Default is the same as html_title. -html_short_title = 'API Guide' - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# html_extra_path = [] - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'monascalogapi-api-guide' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # 'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'MonascaLogApiAPI.tex', u'Key Manager API Documentation', - u'OpenStack Foundation', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'monascalogapiapi', u'Monasca Log API Documentation', - [u'OpenStack Foundation'], 1) -] - -# If true, show URL addresses after external links. -# man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'MonascaLogApiAPIGuide', u'Monasca Log API Guide', - u'OpenStack Foundation', 'APIGuide', - 'This guide teaches OpenStack Monasca Log service users concepts about ' - 'managing keys in an OpenStack cloud with the Monasca Log API.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -# texinfo_appendices = [] - -# If false, no module index is generated. -# texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -# texinfo_no_detailmenu = False - -# -- Options for Internationalization output ------------------------------ -locale_dirs = ['locale/'] - -# -- Options for PDF output -------------------------------------------------- - -pdf_documents = [ - ('index', u'MonascaLogApiAPIGuide', u'Key Manager API Guide', u'OpenStack ' - 'contributors') -] diff --git a/api-guide/source/general_info.rst b/api-guide/source/general_info.rst deleted file mode 100644 index ae6cad6a..00000000 --- a/api-guide/source/general_info.rst +++ /dev/null @@ -1,50 +0,0 @@ -.. - Copyright 2014-2017 Fujitsu LIMITED - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -======================== -Monasca Log API Concepts -======================== - -The Monasca Log API is defined as a RESTful HTTP service. The API -takes advantage of all aspects of the HTTP protocol (methods, URIs, -media types, response codes, etc.) and providers are free to use -existing features of the protocol such as caching, persistent -connections, and content compression among others. - -Providers can return information identifying requests in HTTP response -headers, for example, to facilitate communication between the provider -and client applications. - -Monasca LOG is a service that provides log collection capabilities over cloud. - -User Concepts -============= - -To use the Monasca Log API effectively, you should understand several -key concepts: - -- **Log** - -- **Dimensions** - -Relationship with Metric API -============================ - -The Monasca Log API follow similar concept as Monasca Metric API. -Both are using the same meta-like language to describie entities that are -sent over wire. Below list enumerates those meta properties: - -- dimensions -- meta_value diff --git a/api-guide/source/index.rst b/api-guide/source/index.rst deleted file mode 100644 index 111fef95..00000000 --- a/api-guide/source/index.rst +++ /dev/null @@ -1,70 +0,0 @@ -.. - Copyright 2014-2017 Fujitsu LIMITED - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -=============== -Monasca Log API -=============== - -The monasca-log-api project has a RESTful HTTP service called the -Monasca Log API. Through this API users are able to send logs from entire -cloud. - -This guide covers the concepts in the Monasca Log API. -For a full reference listing, please see: -`Monasca Log API Reference `__. - -We welcome feedback, comments, and bug reports at -`storyboard/monasca `__. - -Intended audience -================= - -This guide assists software developers who want to develop applications -using the Monasca Log API. To use this information, you should -have access to an account from an OpenStack Compute provider, or have -access to your own deployment, and you should also be familiar with the -following concepts: - -* Monasca services -* RESTful HTTP services -* HTTP/1.1 -* JSON data serialization formats - -End User and Operator APIs -========================== - -The Log API includes all end user and operator API calls. -The API works with keystone and, at the monent, uses custom RBAC to -enforce API security. - -API Versions -============ - -Following the Newton release, every Nova deployment should have -the following endpoints: - -* / - list of available versions -* /v2.0 - the first version, permitted only single log to be send per request -* /v3.0 - the next version, allows sending multiple logs at once - -Contents -======== - -.. toctree:: - :maxdepth: 2 - - general_info - logs - diff --git a/api-guide/source/logs.rst b/api-guide/source/logs.rst deleted file mode 100644 index db0306f7..00000000 --- a/api-guide/source/logs.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. - Copyright 2014-2017 Fujitsu LIMITED - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -==== -Logs -==== diff --git a/api-ref/source/conf.py b/api-ref/source/conf.py deleted file mode 100644 index a056160b..00000000 --- a/api-ref/source/conf.py +++ /dev/null @@ -1,247 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Key Manager API documentation build configuration file -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.6' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'os_api_ref', - 'openstackdocstheme' -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -source_encoding = 'utf-8' - -# The master toctree document. -master_doc = 'index' - -# General details about project -openstackdocs_repo_name = 'openstack/monasca-log-api' -openstackdocs_auto_name = False -project = 'Monasca Log Ref Guide' -openstackdocs_bug_project = 'monasca-log-api' -openstackdocs_bug_tag = 'api-ref' -copyright = '2014, OpenStack Foundation' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'native' - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'openstackdocs' - -# To use the API Reference sidebar dropdown menu, -# uncomment the html_theme_options parameter. The theme -# variable, sidebar_dropdown, should be set to `api_ref`. -# Otherwise, the list of links for the User and Ops docs -# appear in the sidebar dropdown menu. -html_theme_options = {"sidebar_dropdown": "api_ref", - "sidebar_mode": "toc"} - -# A shorter title for the navigation bar. Default is the same as html_title. -html_short_title = 'API Ref' - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# html_extra_path = [] - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'monascalogapi-api-ref' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # 'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'MonascaLogApi.tex', u'Monasca Log API Documentation', - u'OpenStack Foundation', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'monascalogapi', u'Monasca Log API Documentation', - [u'OpenStack Foundation'], 1) -] - -# If true, show URL addresses after external links. -# man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'MonascaLogAPI', u'Monasca Log API Documentation', - u'OpenStack Foundation', 'MonascaLogAPI', 'Monasca Log API', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -# texinfo_appendices = [] - -# If false, no module index is generated. -# texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -# texinfo_no_detailmenu = False - -# -- Options for Internationalization output ------------------------------ -locale_dirs = ['locale/'] diff --git a/api-ref/source/healthcheck.inc b/api-ref/source/healthcheck.inc deleted file mode 100644 index b1bfdb36..00000000 --- a/api-ref/source/healthcheck.inc +++ /dev/null @@ -1,54 +0,0 @@ -.. -*- rst -*- -.. - Copyright 2017 Fujitsu LIMITED - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -=========== -Healthcheck -=========== - -The *Monasca Log API* comes with a built-in health check mechanism. -It is available in two flavors (simple and complex). - -Simple check -============ - -The simple check only returns response only if API is up -and running. It does not return any data because it is accessible only -for ```HEAD``` requests. - -.. rest_method:: HEAD /healthcheck - -.. rest_status_code:: success http_codes.yaml - - - 204: la_up - -Complex check -============= - -# TODO(trebskit) add note to api-guide about peripheral checks - -The complex check not only returns a response with success code if API -is up and running but it also verifies if peripheral components are -in expected condition. - -.. rest_method:: GET /healthcheck - -.. rest_status_code:: success http_codes.yaml - - - 200: la_up - -.. rest_status_code:: error http_codes.yaml - - - 503: no_health diff --git a/api-ref/source/http_codes.yaml b/api-ref/source/http_codes.yaml deleted file mode 100644 index 9e12b04b..00000000 --- a/api-ref/source/http_codes.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2017 Fujitsu LIMITED - -200: - la_up: | - API is up and running. -204: - default: | - Normal response code, everything went as expected (or even better). - la_up: | - API is up and running. -400: - default: | - Sent data was malformed. -401: - default: | - User must authenticate before making a request. -403: - default: | - Policy does not allow current user to do this operation. -411: - default: | - Content-Length header was not found in request. -413: - default: | - Sent body is too large to be processed. -422: - default: | - Send data could not be processed properly. - no_dims: | - Dimensions are required. - dim_name_too_long: | - Dimension name {name} must be 255 characters or less. - dim_name_underscore: | - Dimension name {name} cannot start with underscore (_). - dim_name_forbidden_chars: | - Dimension name {name} may not contain: "> < = { } ( ) \' " , ; &". - dim_name_empty: | - Dimension name cannot be empty. - dim_value_too_long: | - Dimension value {value} must be 255 characters or less. - dim_value_empty: | - Dimension value cannot be empty. - app_type_too_long: | - Application type {type} must be {length} characters or less. - log_no_msg: | - Log property must have message. - bad_envelope: | - Failed to create an envelope. - -503: - default: | - The server is currently unable to handle the request due to a - temporary overload or scheduled maintenance. - This will likely be alleviated after some delay. - no_health: - API is running but there are problems with peripheral components. diff --git a/api-ref/source/index.rst b/api-ref/source/index.rst deleted file mode 100644 index 807149a7..00000000 --- a/api-ref/source/index.rst +++ /dev/null @@ -1,35 +0,0 @@ -:tocdepth: 2 - -.. - Copyright 2014-2017 Fujitsu LIMITED - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -======================== -Monasca Log Service APIs -======================== - -.. rest_expand_all:: - -.. include:: logs.inc -.. include:: version.inc -.. include:: healthcheck.inc - -=============== -Deprecated APIs -=============== - -This section contains the reference for APIs that are -depracted in the Monasca Log Service - -.. include:: log.inc diff --git a/api-ref/source/locale/.gitkeep b/api-ref/source/locale/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/api-ref/source/log.inc b/api-ref/source/log.inc deleted file mode 100644 index 881ffab5..00000000 --- a/api-ref/source/log.inc +++ /dev/null @@ -1,73 +0,0 @@ -.. -*- rst -*- -.. - Copyright 2017 Fujitsu LIMITED - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -==== -Log -==== - -Accepts just a single log entry sent from log-agent of another client. -Can work with logs specified as json (application/json) and text (text/plain) - -Send logs -========= - -.. rest_method:: POST /v2.0/log/single - -Accepts single log entry. - -.. rest_status_code:: success http_codes.yaml - - - 204 - -.. rest_status_code:: error http_codes.yaml - - - 422: no_dims - - 422: app_type_too_long - - 422: dim_name_too_long - - 422: dim_name_underscore - - 422: dim_name_forbidden_chars - - 422: dim_name_empty - - 422: dim_value_too_long - - 422: dim_value_empty - - 422: log_no_msg - - 422: bad_envelope - - 503 - -Request -------- - - .. rest_parameters:: parameters.yaml - - - log_json: log_json - - log_text: log_text - - X_Dimensions: X_Dimensions - - X_Application_Type: X_Application_Type - -**Example 1: Simple request with single log (json)** - -.. literalinclude:: ../../doc/api_samples/v2/req_json.json - :language: javascript - -**Example 2: Simple request with single log (text)** - -.. literalinclude:: ../../doc/api_samples/v2/req_text.txt - :language: text - - -Response --------- - -No body content is returned on a successful POST diff --git a/api-ref/source/logs.inc b/api-ref/source/logs.inc deleted file mode 100644 index 5b47d07d..00000000 --- a/api-ref/source/logs.inc +++ /dev/null @@ -1,75 +0,0 @@ -.. -*- rst -*- -.. - Copyright 2017 Fujitsu LIMITED - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -==== -Logs -==== - -Accepts logs send from log-agents. Logs are basically raw lines, -as collected from physical resources, enriched with dimensions. - -Send logs -========= - -.. rest_method:: POST /v3.0/logs - -Accepts multiple logs (i.e. bulk mode). Each log can be enriched with set -of dimensions. If necessary some of the dimensions can be specified as global -dimensions (that is particularly useful, to make request smaller, -if there is a lot of duplicates among each log entry dimensions) - -.. rest_status_code:: success http_codes.yaml - - - 204 - -.. rest_status_code:: error http_codes.yaml - - - 400 - - 401 - - 403 - - 411 - - 413 - - 422: log_no_msg - - 422: bad_envelope - - 503 - -Request -------- - - .. rest_parameters:: parameters.yaml - - - dimensions: dimensions - - logs: logs - -**Example 1: Simple request with single log** - -.. literalinclude:: ../../doc/api_samples/v3/req_single_log.json - :language: javascript - -**Example 2: Send multiple logs at once** - -.. literalinclude:: ../../doc/api_samples/v3/req_multiple_logs.json - :language: javascript - -**Example 3: Specify global dimensions for each log entry** - -.. literalinclude:: ../../doc/api_samples/v3/req_global_dims.json - :language: javascript - -Response --------- - -No body content is returned on a successful POST diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml deleted file mode 100644 index dc9038ca..00000000 --- a/api-ref/source/parameters.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2017 Fujitsu LIMITED - -# header params -X_Application_Type: - description: | - A single string value representing the application that has generated - given log entry - in: header - required: true - required: true - type: string - min_version: 2.0 -X_Dimensions: - description: | - A dictionary consisting of (key, value) pairs used to uniquely - identify a log. - in: header - required: true - type: dict - min_version: 2.0 - -# body params -dimensions: - description: | - Dimensions sent in request body are known as global dimensions. - Each dimension applies to each log entry sent in a bulk request. - Dimensions are simple map (thus having key-value structure). - in: body - required: false - type: object - min_version: 3.0 -log_json: - description: | - Single log entry specified as application/json - in: body - required: true - type: object -log_text: - description: | - Single log entry specified as text/plain - in: body - required: true - type: string -logs: - description: | - Array containing each log entry, sent in bulk request. - in: body - required: true - type: object - min_version: 3.0 diff --git a/api-ref/source/version.inc b/api-ref/source/version.inc deleted file mode 100644 index 6bce887e..00000000 --- a/api-ref/source/version.inc +++ /dev/null @@ -1,19 +0,0 @@ -.. -*- rst -*- -.. - Copyright 2017 Fujitsu LIMITED - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -======= -Version -======= diff --git a/config-generator/README.md b/config-generator/README.md deleted file mode 100644 index 477d8a36..00000000 --- a/config-generator/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# config-generator - -To generate sample configuration execute - -```sh -tox -e genconfig -``` - -To generate the sample policies execute - -```sh -tox -e genpolicy -``` diff --git a/config-generator/monasca-log-api.conf b/config-generator/monasca-log-api.conf deleted file mode 100644 index b946a254..00000000 --- a/config-generator/monasca-log-api.conf +++ /dev/null @@ -1,8 +0,0 @@ -[DEFAULT] -output_file = etc/monasca/monasca-log-api.conf.sample -wrap_width = 79 -format = ini -summarize = True -namespace = monasca_log_api -namespace = oslo.log -namespace = oslo.policy diff --git a/config-generator/policy.conf b/config-generator/policy.conf deleted file mode 100644 index 075c2b99..00000000 --- a/config-generator/policy.conf +++ /dev/null @@ -1,4 +0,0 @@ -[DEFAULT] -output_file = etc/monasca/log-api.policy.yaml.sample -format = yaml -namespace = monasca_log_api \ No newline at end of file diff --git a/contrib/post_test_hook.sh b/contrib/post_test_hook.sh deleted file mode 100644 index b41cc929..00000000 --- a/contrib/post_test_hook.sh +++ /dev/null @@ -1,103 +0,0 @@ -# -# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP -# (C) Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -#    http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Sleep some time until all services are starting -sleep 6 - -function load_devstack_utilities { - source $BASE/new/devstack/stackrc - source $BASE/new/devstack/functions - source $BASE/new/devstack/openrc admin admin - - # print OS_ variables - env | grep OS_ -} - -function setup_monasca_log { - local constraints="-c $REQUIREMENTS_DIR/upper-constraints.txt" - - pushd $TEMPEST_DIR - sudo -EH pip install $constraints -r requirements.txt -r test-requirements.txt - popd; - - pushd $MONASCA_LOG_API_DIR - sudo -EH pip install $constraints -r requirements.txt -r test-requirements.txt - sudo -EH python setup.py install - popd; -} - -function set_tempest_conf { - - local conf_file=$TEMPEST_DIR/etc/tempest.conf - pushd $TEMPEST_DIR - oslo-config-generator \ - --config-file tempest/cmd/config-generator.tempest.conf \ - --output-file $conf_file - popd - - cp -f $DEST/tempest/etc/logging.conf.sample $DEST/tempest/etc/logging.conf - - # set identity section - iniset $conf_file identity admin_domain_scope True - - iniset $conf_file identity user_unique_last_password_count 2 - iniset $conf_file identity user_locakout_duration 5 - iniset $conf_file identity user_lockout_failure_attempts 2 - - iniset $conf_file identity uri $OS_AUTH_URL/v2.0 - iniset $conf_file identity uri_v3 $OS_AUTH_URL/v3 - iniset $conf_file identity auth_version v$OS_IDENTITY_API_VERSION - iniset $conf_file identity region $OS_REGION_NAME - - # NOTE(trebskit) we're processing a lot here, increase http timeout - iniset $conf_file identity-feature-enabled http_timeout 120 - - # set auth section - iniset $conf_file auth use_dynamic_credentials True - iniset $conf_file auth admin_username $OS_USERNAME - iniset $conf_file auth admin_password $OS_PASSWORD - iniset $conf_file auth admin_domain_name $OS_PROJECT_DOMAIN_ID - iniset $conf_file auth admin_project_name $OS_PROJECT_NAME - iniset $conf_file auth tempest_roles monasca-user - - # set monitoring section - iniset $conf_file monitoring kibana_version 4.6.3 -} - -function function_exists { - declare -f -F $1 > /dev/null -} - -if ! function_exists echo_summary; then - function echo_summary { - echo $@ - } -fi - -echo_summary "monasca-log's post_test_hook.sh was called..." -(set -o posix; set) - -# save ref to monasca-api dir -export MONASCA_LOG_API_DIR="$BASE/new/monasca-log-api" -export TEMPEST_DIR="$BASE/new/tempest" - -sudo chown -R "${USER}":stack $MONASCA_LOG_API_DIR -sudo chown -R "${USER}":stack $TEMPEST_DIR - -load_devstack_utilities -setup_monasca_log -set_tempest_conf diff --git a/devstack/README.md b/devstack/README.md deleted file mode 100644 index d6bd2b5f..00000000 --- a/devstack/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# Monasca Log Management DevStack Plugin - -The Monasca Log Management DevStack plugin currently only works on Ubuntu 14.04 (Trusty). -More Linux Distributions will be supported in the future. - -Monasca Log Management Devstack plugin requires Monasca Devstack plugin. -Running the Monasca DevStack plugin and Monasca Log Management Devstack plugin requires a machine with 14GB of RAM. - -Directions for installing and running Devstack can be found here: -``` -https://docs.openstack.org/devstack/latest/ -``` - -To run Monasca Log Management in DevStack, do the following three steps. - -1. Clone the DevStack repo. - -``` -git clone https://opendev.org/openstack/devstack -``` - -2. Add the following to the DevStack local.conf file in the root of the devstack directory. You may - need to create the local.conf if it does not already exist. - -``` -[[local|localrc]] -MYSQL_PASSWORD=secretmysql -DATABASE_PASSWORD=secretdatabase -RABBIT_PASSWORD=secretrabbit -ADMIN_PASSWORD=secretadmin -SERVICE_PASSWORD=secretservice -SERVICE_TOKEN=111222333444 - -LOGFILE=$DEST/logs/stack.sh.log -LOGDIR=$DEST/logs -LOG_COLOR=False - -# The following two variables allow switching between Java and Python for the implementations -# of the Monasca API and the Monasca Persister. If these variables are not set, then the -# default is to install the Python implementations of both the Monasca API and the Monasca Persister. - -# Uncomment one of the following two lines to choose Java or Python for the Monasca API. -MONASCA_API_IMPLEMENTATION_LANG=${MONASCA_API_IMPLEMENTATION_LANG:-java} -# MONASCA_API_IMPLEMENTATION_LANG=${MONASCA_API_IMPLEMENTATION_LANG:-python} - -# Uncomment of the following two lines to choose Java or Python for the Monasca Pesister. -MONASCA_PERSISTER_IMPLEMENTATION_LANG=${MONASCA_PERSISTER_IMPLEMENTATION_LANG:-java} -# MONASCA_PERSISTER_IMPLEMENTATION_LANG=${MONASCA_PERSISTER_IMPLEMENTATION_LANG:-python} - -# Uncomment one of the following two lines to choose either InfluxDB or Vertica. -MONASCA_METRICS_DB=${MONASCA_METRICS_DB:-influxdb} -# MONASCA_METRICS_DB=${MONASCA_METRICS_DB:-vertica} - -# This line will enable all of Monasca. -enable_plugin monasca-api https://opendev.org/openstack/monasca-api -enable_plugin monasca-log-api https://opendev.org/openstack/monasca-log-api.git -``` - -3. Run './stack.sh' from the root of the devstack directory. - - -After finishing the installation, you can find the "Log Management" button on -"Overview" of "Monitoring" tab, if you log in OpenStack as admin. -At first time, you need to specify the index pattern and time-field name. - -The index name is created as the following format. - logs-\[mini-mon tenant id\]-YYYY-MM-DD -For example: - logs-20c4fbd37a2345dd84266dfc92da7bd1-2016-04-07 - -Set the value as the above to index pattern. -Or you can use "\*" as a wild card, like below. - logs-20c4fbd37a2345dd84266dfc92da7bd1-\* - -Select @timestamp as time-field name. - -4. Extra settings - -## Using WSGI - -monasca-log-api can be deployed with Apache using mod_uwsgi. -By default monasca-log-api by default runs under gunicorn. -If you wish to use Apache make sure that ```devstack/local.conf``` -contains: - -```sh -MONASCA_LOG_API_USE_MOD_WSGI=True -``` - -Actual ```MONASCA_LOG_API_DEPLOY``` value is determined using devstack`s -```WSGI_MODE``` variable. Nevertheless there are only three possible values, -that ```MONASCA_LOG_API_DEPLOY``` can take: - -* ```gunicorn``` if ```MONASCA_LOG_API_USE_MOD_WSGI=False``` -* ```mod_wsgi``` if ```MONASCA_LOG_API_USE_MOD_WSGI=True && WSGI_MODE="mod_wsgi""``` -* ```uwsgi``` if ```MONASCA_LOG_API_USE_MOD_WSGI=True && WSGI_MODE="uwsgi""``` - -# Using Vagrant - -Vagrant can be used to deploy a VM with Devstack and Monasca Logging -running in it using the Vagrantfile. After installing Vagrant, -just run the command `vagrant up` as usual in the `monasca-log-api/devstack` -directory. - -``` -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -``` diff --git a/devstack/Vagrantfile b/devstack/Vagrantfile deleted file mode 100644 index a4a0d456..00000000 --- a/devstack/Vagrantfile +++ /dev/null @@ -1,123 +0,0 @@ -require 'vagrant.rb' -Vagrant.configure(2) do |config| - - config.cache.scope = :box if Vagrant.has_plugin?("vagrant-cachier") - config.timezone.value = :host if Vagrant.has_plugin?('vagrant-timezone') - if Vagrant.has_plugin?('vagrant-proxyconf') - config.proxy.http = ENV['http_proxy'] if ENV['http_proxy'] - config.proxy.https = ENV['https_proxy'] if ENV['https_proxy'] - if ENV['no_proxy'] - local_no_proxy = ",192.168.10.6,10.0.2.15" - config.proxy.no_proxy = ENV['no_proxy'] + local_no_proxy - end - end - - config.ssh.forward_agent = true - - config.vm.hostname = "devstack" - config.vm.box = "bento/ubuntu-18.04" - config.vm.network "private_network",ip:"192.168.10.6" - config.vm.synced_folder "~/", "/vagrant_home" - config.vm.provider "virtualbox" do |vb| - vb.gui = false - vb.memory = "12800" - vb.cpus = 4 - end - - config.vm.provision "shell", privileged: false, inline: <<-SHELL - sudo apt-get -y install git - - if [ $http_proxy ]; then - git config --global url.https://opendev.org/.insteadOf https://opendev.org/ - sudo git config --global url.https://opendev.org/.insteadOf https://opendev.org/ - - protocol=`echo $http_proxy | awk -F: '{print $1}'` - host=`echo $http_proxy | awk -F/ '{print $3}' | awk -F: '{print $1}'` - port=`echo $http_proxy | awk -F/ '{print $3}' | awk -F: '{print $2}'` - - echo " - - - $host - true - $protocol - $host - $port - - - " > ./maven_proxy_settings.xml - - mkdir ~/.m2 - cp ./maven_proxy_settings.xml ~/.m2/settings.xml - - sudo mkdir /root/.m2 - sudo cp ./maven_proxy_settings.xml /root/.m2/settings.xml - fi - - git clone https://opendev.org/openstack/devstack --branch master --depth 1 - cd devstack - echo '[[local|localrc]] - -GIT_DEPTH=1 -DEST=/opt/stack -USE_VENV=False - -SERVICE_HOST=192.168.10.6 -HOST_IP=192.168.10.6 -DATABASE_HOST=192.168.10.6 -MYSQL_HOST=192.168.10.6 -HOST_IP_IFACE=eth1 - -MYSQL_PASSWORD=secretmysql -DATABASE_PASSWORD=secretdatabase -RABBIT_PASSWORD=secretrabbit -ADMIN_PASSWORD=secretadmin -SERVICE_PASSWORD=secretservice - -LOGFILE=$DEST/logs/stack.sh.log -LOGDIR=$DEST/logs -LOG_COLOR=False - -disable_all_services -enable_service zookeeper rabbit mysql key tempest horizon - -# Enable more OpenStack services if neccessary: -# https://github.com/openstack-dev/devstack/blob/master/stackrc#L56-L81 -# Nova - services to support libvirt based openstack clouds -# enable_service n-api n-cpu n-cond n-sch n-novnc n-cauth n-api-meta -# Placement and Glance services needed for Nova -# enable_service placement-api placement-client -# enable_service g-api g-reg -# Cinder, Neutron -# enable_service cinder c-api c-vol c-sch c-bak -# enable_service neutron q-svc q-agt q-dhcp q-meta q-l3 - -# The following two variables allow switching between Java and Python for the implementations -# of the Monasca API and the Monasca Persister. If these variables are not set, then the -# default is to install the Python implementations of both the Monasca API and the Monasca Persister. - -# Uncomment one of the following two lines to choose Java or Python for the Monasca API. -# MONASCA_API_IMPLEMENTATION_LANG=${MONASCA_API_IMPLEMENTATION_LANG:-java} -MONASCA_API_IMPLEMENTATION_LANG=${MONASCA_API_IMPLEMENTATION_LANG:-python} - -# Uncomment one of the following two lines to choose Java or Python for the Monasca Pesister. -# MONASCA_PERSISTER_IMPLEMENTATION_LANG=${MONASCA_PERSISTER_IMPLEMENTATION_LANG:-java} -MONASCA_PERSISTER_IMPLEMENTATION_LANG=${MONASCA_PERSISTER_IMPLEMENTATION_LANG:-python} - -# Uncomment one of the following two lines to choose either InfluxDB or Vertica. -# MONASCA_METRICS_DB=${MONASCA_METRICS_DB:-vertica} -# MONASCA_METRICS_DB=${MONASCA_METRICS_DB:-cassandra} -MONASCA_METRICS_DB=${MONASCA_METRICS_DB:-influxdb} - -# Uncomment following line to deploy monasca-log-api with Apache -# MONASCA_LOG_API_USE_MOD_WSGI=True - -# Uncomment one of the following lines and modify accordingly to enable the Monasca DevStack Plugin -enable_plugin monasca-api https://opendev.org/openstack/monasca-api.git -enable_plugin monasca-log-api https://opendev.org/openstack/monasca-log-api.git - -' > local.conf - ./stack.sh - SHELL - -end diff --git a/devstack/files/apache-log-api.template b/devstack/files/apache-log-api.template deleted file mode 100644 index 946b42d2..00000000 --- a/devstack/files/apache-log-api.template +++ /dev/null @@ -1,25 +0,0 @@ -Listen %PUBLICPORT% - - - WSGIDaemonProcess monasca-log-api processes=%APIWORKERS% threads=1 user=%USER% display-name=%{GROUP} %VIRTUALENV% - WSGIProcessGroup monasca-log-api - WSGIScriptAlias / %PUBLICWSGI% - WSGIApplicationGroup %{GLOBAL} - WSGIPassAuthorization On - = 2.4> - ErrorLogFormat "%M" - - ErrorLog /var/log/%APACHE_NAME%/monasca-log-api.log - %SSLENGINE% - %SSLCERTFILE% - %SSLKEYFILE% - - -Alias /logs %PUBLICWSGI% - - SetHandler wsgi-script - Options +ExecCGI - WSGIProcessGroup monasca-log-api - WSGIApplicationGroup %{GLOBAL} - WSGIPassAuthorization On - diff --git a/devstack/files/elasticsearch/elasticsearch.yml b/devstack/files/elasticsearch/elasticsearch.yml deleted file mode 100644 index 6d013bb8..00000000 --- a/devstack/files/elasticsearch/elasticsearch.yml +++ /dev/null @@ -1,360 +0,0 @@ -##################### Elasticsearch Configuration Example ##################### - -# This file contains an overview of various configuration settings, -# targeted at operations staff. Application developers should -# consult the guide at . -# -# The installation procedure is covered at -# . -# -# Elasticsearch comes with reasonable defaults for most settings, -# so you can try it out without bothering with configuration. -# -# Most of the time, these defaults are just fine for running a production -# cluster. If you're fine-tuning your cluster, or wondering about the -# effect of certain configuration option, please _do ask_ on the -# mailing list or IRC channel [http://elasticsearch.org/community]. - -# Any element in the configuration can be replaced with environment variables -# by placing them in ${...} notation. For example: -# -#node.rack: ${RACK_ENV_VAR} - -# For information on supported formats and syntax for the config file, see -# - - -################################### Cluster ################################### - -# Cluster name identifies your cluster for auto-discovery. If you're running -# multiple clusters on the same network, make sure you're using unique names. -# -cluster.name: monasca_elastic - - -#################################### Node ##################################### - -# Node names are generated dynamically on startup, so you're relieved -# from configuring them manually. You can tie this node to a specific name: -# -node.name: "devstack" - -# Allow this node to be eligible as a master node (enabled by default): -node.master: true - -# Allow this node to store data (enabled by default) -node.data: true - -# You can exploit these settings to design advanced cluster topologies. -# -# 1. You want this node to never become a master node, only to hold data. -# This will be the "workhorse" of your cluster. -# -#node.master: false -#node.data: true -# -# 2. You want this node to only serve as a master: to not store any data and -# to have free resources. This will be the "coordinator" of your cluster. -# -#node.master: true -#node.data: false -# -# 3. You want this node to be neither master nor data node, but -# to act as a "search load balancer" (fetching data from nodes, -# aggregating results, etc.) -# -#node.master: false -#node.data: false - -# Use the Cluster Health API [http://localhost:9200/_cluster/health], the -# Node Info API [http://localhost:9200/_nodes] or GUI tools -# such as , -# , -# and -# to inspect the cluster state. - -# A node can have generic attributes associated with it, which can later be used -# for customized shard allocation filtering, or allocation awareness. An attribute -# is a simple key value pair, similar to node.key: value, here is an example: -# -#node.rack: rack314 - -# By default, multiple nodes are allowed to start from the same installation location -# to disable it, set the following: -#node.max_local_storage_nodes: 1 - - -#################################### Index #################################### - -# You can set a number of options (such as shard/replica options, mapping -# or analyzer definitions, translog settings, ...) for indices globally, -# in this file. -# -# Note, that it makes more sense to configure index settings specifically for -# a certain index, either when creating it or by using the index templates API. -# -# See and -# -# for more information. - -# Set the number of shards (splits) of an index (5 by default): -# -#index.number_of_shards: 5 - -# Set the number of replicas (additional copies) of an index (1 by default): -# -#index.number_of_replicas: 1 - -# Note, that for development on a local machine, with small indices, it usually -# makes sense to "disable" the distributed features: -# -#index.number_of_shards: 1 -#index.number_of_replicas: 0 - -# These settings directly affect the performance of index and search operations -# in your cluster. Assuming you have enough machines to hold shards and -# replicas, the rule of thumb is: -# -# 1. Having more *shards* enhances the _indexing_ performance and allows to -# _distribute_ a big index across machines. -# 2. Having more *replicas* enhances the _search_ performance and improves the -# cluster _availability_. -# -# The "number_of_shards" is a one-time setting for an index. -# -# The "number_of_replicas" can be increased or decreased anytime, -# by using the Index Update Settings API. -# -# Elasticsearch takes care about load balancing, relocating, gathering the -# results from nodes, etc. Experiment with different settings to fine-tune -# your setup. - -# Use the Index Status API () to inspect -# the index status. - - -#################################### Paths #################################### - -# Path to directory where to store index data allocated for this node. -path.data: %ES_DATA_DIR% - -# Path to log files: -path.logs: %ES_LOG_DIR% - -# Path to where plugins are installed: -#path.plugins: /path/to/plugins - -# Path to temporary files -#path.work: /path/to/work - -# Path to directory containing configuration (this file and logging.yml): -#path.conf: /path/to/conf - - -#################################### Plugin ################################### - -# If a plugin listed here is not installed for current node, the node will not start. -# -#plugin.mandatory: mapper-attachments,lang-groovy - - -################################### Memory #################################### - -# Elasticsearch performs poorly when JVM starts swapping: you should ensure that -# it _never_ swaps. -# -# Set this property to true to lock the memory: -# -#bootstrap.mlockall: true - -# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set -# to the same value, and that the machine has enough memory to allocate -# for Elasticsearch, leaving enough memory for the operating system itself. -# -# You should also make sure that the Elasticsearch process is allowed to lock -# the memory, eg. by using `ulimit -l unlimited`. - - -############################## Network And HTTP ############################### - -# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens -# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node -# communication. (the range means that if the port is busy, it will automatically -# try the next port). - -# Set the bind address specifically (IPv4 or IPv6): -network.bind_host: %ES_SERVICE_BIND_HOST% - -# Set the address other nodes will use to communicate with this node. If not -# set, it is automatically derived. It must point to an actual IP address. -network.publish_host: %ES_SERVICE_PUBLISH_HOST% - -# Set a custom port for the node to node communication (9300 by default): -transport.tcp.port: %ES_SERVICE_PUBLISH_PORT% - -# Enable compression for all communication between nodes (disabled by default): -# -#transport.tcp.compress: true - -# Set a custom port to listen for HTTP traffic: -# -http.port: %ES_SERVICE_BIND_PORT% - -# Set a custom allowed content length: -# -#http.max_content_length: 100mb - -# Disable HTTP completely: -# -#http.enabled: false - - -################################### Gateway ################################### - -# The gateway allows for persisting the cluster state between full cluster -# restarts. Every change to the state (such as adding an index) will be stored -# in the gateway, and when the cluster starts up for the first time, -# it will read its state from the gateway. - -# There are several types of gateway implementations. For more information, see -# . - -# The default gateway type is the "local" gateway (recommended): -# -#gateway.type: local - -# Settings below control how and when to start the initial recovery process on -# a full cluster restart (to reuse as much local data as possible when using shared -# gateway). - -# Allow recovery process after N nodes in a cluster are up: -# -#gateway.recover_after_nodes: 1 -# Set the timeout to initiate the recovery process, once the N nodes -# from previous setting are up (accepts time value): -# -#gateway.recover_after_time: 5m - -# Set how many nodes are expected in this cluster. Once these N nodes -# are up (and recover_after_nodes is met), begin recovery process immediately -# (without waiting for recover_after_time to expire): -# -#gateway.expected_nodes: 2 - - -############################# Recovery Throttling ############################# - -# These settings allow to control the process of shards allocation between -# nodes during initial recovery, replica allocation, rebalancing, -# or when adding and removing nodes. - -# Set the number of concurrent recoveries happening on a node: -# -# 1. During the initial recovery -# -#cluster.routing.allocation.node_initial_primaries_recoveries: 4 -# -# 2. During adding/removing nodes, rebalancing, etc -# -#cluster.routing.allocation.node_concurrent_recoveries: 2 - -# Set to throttle throughput when recovering (eg. 100mb, by default 20mb): -# -#indices.recovery.max_bytes_per_sec: 20mb - -# Set to limit the number of open concurrent streams when -# recovering a shard from a peer: -# -#indices.recovery.concurrent_streams: 5 - - -################################## Discovery ################################## - -# Discovery infrastructure ensures nodes can be found within a cluster -# and master node is elected. Multicast discovery is the default. - -# Set to ensure a node sees N other master eligible nodes to be considered -# operational within the cluster. This should be set to a quorum/majority of -# the master-eligible nodes in the cluster. -# -discovery.zen.minimum_master_nodes: 1 - -# Set the time to wait for ping responses from other nodes when discovering. -# Set this option to a higher value on a slow or congested network -# to minimize discovery failures: -# -#discovery.zen.ping.timeout: 3s - -# For more information, see -# - -# Unicast discovery allows to explicitly control which nodes will be used -# to discover the cluster. It can be used when multicast is not present, -# or to restrict the cluster communication-wise. -# -# 1. Disable multicast discovery (enabled by default): -# -discovery.zen.ping.multicast.enabled: false -# 2. Configure an initial list of master nodes in the cluster -# to perform discovery when new nodes (master or data) are started: -# -# discovery.zen.ping.unicast.hosts: [127.0.0.1] - -# EC2 discovery allows to use AWS EC2 API in order to perform discovery. -# -# You have to install the cloud-aws plugin for enabling the EC2 discovery. -# -# For more information, see -# -# -# See -# for a step-by-step tutorial. - -# GCE discovery allows to use Google Compute Engine API in order to perform discovery. -# -# You have to install the cloud-gce plugin for enabling the GCE discovery. -# -# For more information, see . - -# Azure discovery allows to use Azure API in order to perform discovery. -# -# You have to install the cloud-azure plugin for enabling the Azure discovery. -# -# For more information, see . - -################################## Slow Log ################################## - -# Shard level query and fetch threshold logging. - -#index.search.slowlog.threshold.query.warn: 10s -#index.search.slowlog.threshold.query.info: 5s -#index.search.slowlog.threshold.query.debug: 2s -#index.search.slowlog.threshold.query.trace: 500ms - -#index.search.slowlog.threshold.fetch.warn: 1s -#index.search.slowlog.threshold.fetch.info: 800ms -#index.search.slowlog.threshold.fetch.debug: 500ms -#index.search.slowlog.threshold.fetch.trace: 200ms - -#index.indexing.slowlog.threshold.index.warn: 10s -#index.indexing.slowlog.threshold.index.info: 5s -#index.indexing.slowlog.threshold.index.debug: 2s -#index.indexing.slowlog.threshold.index.trace: 500ms - -################################## GC Logging ################################ - -#monitor.jvm.gc.young.warn: 1000ms -#monitor.jvm.gc.young.info: 700ms -#monitor.jvm.gc.young.debug: 400ms - -#monitor.jvm.gc.old.warn: 10s -#monitor.jvm.gc.old.info: 5s -#monitor.jvm.gc.old.debug: 2s - -################################## Security ################################ - -# Uncomment if you want to enable JSONP as a valid return transport on the -# http server. With this enabled, it may pose a security risk, so disabling -# it unless you need it is recommended (it is disabled by default). -# -#http.jsonp.enable: true diff --git a/devstack/files/grafana/dashboards.d/09-monasca.json b/devstack/files/grafana/dashboards.d/09-monasca.json deleted file mode 100644 index 84935135..00000000 --- a/devstack/files/grafana/dashboards.d/09-monasca.json +++ /dev/null @@ -1,2138 +0,0 @@ -{ - "id": null, - "title": "Monasca Monitoring", - "originalTitle": "Monasca Monitoring", - "tags": [], - "style": "dark", - "timezone": "utc", - "editable": true, - "hideControls": false, - "sharedCrosshair": false, - "rows": [ - { - "title": "Monasca Health - metrics", - "height": "100px", - "editable": true, - "collapse": false, - "panels": [ - { - "title": "Metrics API", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 19, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "uwsgi"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - }, - { - "value": "9", - "op": "=", - "text": "UP" - }, - { - "value": "10", - "op": "=", - "text": "UP" - }, - { - "value": "11", - "op": "=", - "text": "UP" - }, - { - "value": "12", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "0.0,0.2,1.0", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "Storm", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 44, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "storm"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "Persister", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 21, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "persister"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "Metrics DB", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 51, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "http_status", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "influxdb"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "0", - "op": "=", - "text": "UP" - }, - { - "value": "1", - "op": "=", - "text": "DOWN" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(71, 212, 59, 0.4)", - "rgba(245, 150, 40, 0.73)", - "rgba(225, 40, 40, 0.59)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "Notification Engine", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 42, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "monasca-notification"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - } - ], - "showTitle": true - }, - { - "title": "Monasca Health - Logs", - "height": "100px", - "editable": true, - "collapse": false, - "panels": [ - { - "title": "Log API", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 20, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "log-api"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - }, - { - "value": "9", - "op": "=", - "text": "UP" - }, - { - "value": "10", - "op": "=", - "text": "UP" - }, - { - "value": "11", - "op": "=", - "text": "UP" - }, - { - "value": "12", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "Log Transformer", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 54, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "log-transformer"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "Log Metrics", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 102, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "log-metrics"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "Log Persister", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 53, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "log-persister"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "Log DB", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 33, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "elasticsearch.cluster_status", - "condition_filter": true - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - }, - { - "value": "9", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "Kibana", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 22, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "http_status", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "kibana"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "0", - "op": "=", - "text": "UP" - }, - { - "value": "1", - "op": "=", - "text": "DOWN" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(71, 212, 59, 0.4)", - "rgba(245, 150, 40, 0.73)", - "rgba(225, 40, 40, 0.59)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - } - ], - "showTitle": true - }, - { - "title": "Monasca Health - Common", - "height": "100px", - "editable": true, - "collapse": false, - "panels": [ - { - "title": "Kafka", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 38, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "kafka"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "ZooKeeper", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 48, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "zookeeper"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "MariaDB", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 66, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "mysqld"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - }, - { - "title": "Statsd", - "error": false, - "span": 2, - "editable": true, - "type": "singlestat", - "id": 166, - "links": [], - "maxDataPoints": 100, - "interval": null, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "monasca-statsd"}] - } - ], - "cacheTimeout": null, - "format": "none", - "prefix": "", - "postfix": "", - "nullText": null, - "valueMaps": [ - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - } - ], - "nullPointMode": "connected", - "valueName": "current", - "prefixFontSize": "50%", - "valueFontSize": "80%", - "postfixFontSize": "50%", - "thresholds": "-1.0,0.2,0.8", - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "sparkline": { - "show": false, - "full": false, - "lineColor": "rgb(31, 120, 193)", - "fillColor": "rgba(31, 118, 189, 0.18)" - } - } - ], - "showTitle": true - }, - { - "title": "System resources", - "height": "250px", - "editable": true, - "collapse": false, - "panels": [ - { - "title": "CPU usage", - "error": false, - "span": 6, - "editable": true, - "type": "graph", - "id": 23, - "datasource": null, - "renderer": "flot", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "percent", - "short" - ], - "grid": { - "leftMax": 100, - "rightMax": null, - "leftMin": 0, - "rightMin": null, - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 0, - "linewidth": 1, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false, - "alignAsTable": true - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": false - }, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "cpu.percent", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring" - }, - { - "target": "", - "aggregator": "none", - "column": "value", - "metric": "cpu.user_perc", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring" - }, - { - "target": "", - "aggregator": "none", - "column": "value", - "metric": "cpu.system_perc", - "condition_filter": true - }, - { - "target": "", - "aggregator": "none", - "column": "value", - "metric": "cpu.wait_perc", - "condition_filter": true - } - ], - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - }, - { - "title": "Memory usage", - "error": false, - "span": 6, - "editable": true, - "type": "graph", - "id": 24, - "datasource": null, - "renderer": "flot", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "none", - "short" - ], - "grid": { - "leftMax": null, - "rightMax": null, - "leftMin": 0, - "rightMin": null, - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 0, - "linewidth": 1, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false, - "alignAsTable": true - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "individual", - "shared": true - }, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "mem.total_mb", - "condition_filter": true - }, - { - "target": "", - "aggregator": "none", - "column": "value", - "metric": "mem.used_mb", - "period": "", - "condition_filter": true - }, - { - "target": "", - "aggregator": "none", - "column": "value", - "metric": "mem.swap_total_mb", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring" - }, - { - "target": "", - "aggregator": "none", - "column": "value", - "metric": "mem.swap_used_mb", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring" - }, - { - "target": "", - "aggregator": "none", - "column": "value", - "metric": "mem.used_cache", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring" - } - ], - "aliasColors": {}, - "seriesOverrides": [], - "links": [], - "leftYAxisLabel": "MB" - }, - { - "title": "Disk usage", - "error": false, - "span": 6, - "editable": true, - "type": "graph", - "id": 25, - "datasource": null, - "renderer": "flot", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "percent", - "short" - ], - "grid": { - "leftMax": 100, - "rightMax": null, - "leftMin": 0, - "rightMin": null, - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 0, - "linewidth": 1, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false, - "alignAsTable": true - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true - }, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "disk.space_used_perc", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring", - "dimensions": [ - { - "key": "mount_point", - "value": "/rootfs" - } - ] - } - ], - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - }, - { - "title": "System load", - "error": false, - "span": 6, - "editable": true, - "type": "graph", - "id": 26, - "datasource": null, - "renderer": "flot", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "none", - "short" - ], - "grid": { - "leftMax": null, - "rightMax": null, - "leftMin": 0, - "rightMin": null, - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 0, - "linewidth": 1, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false, - "alignAsTable": true - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true - }, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "load.avg_1_min", - "period": "", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring" - }, - { - "aggregator": "none", - "column": "value", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring", - "metric": "load.avg_5_min", - "period": "300", - "refId": "B", - "dimensions": [], - "error": "" - }, - { - "aggregator": "none", - "column": "value", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring", - "metric": "load.avg_15_min", - "period": "300", - "refId": "C", - "dimensions": [], - "error": "" - } - ], - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - } - ], - "showTitle": true - }, - { - "title": "Network Monitoring", - "height": "250px", - "editable": true, - "collapse": false, - "panels": [ - { - "title": "Network usage", - "error": false, - "span": 6, - "editable": true, - "type": "graph", - "id": 61, - "datasource": null, - "renderer": "flot", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "bps", - "short" - ], - "grid": { - "leftMax": null, - "rightMax": null, - "leftMin": 0, - "rightMin": null, - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 0, - "linewidth": 1, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false, - "alignAsTable": true - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true - }, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "net.in_bytes_sec", - "merge": true, - "dimensions": "", - "period": "", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring" - }, - { - "target": "", - "aggregator": "none", - "column": "value", - "metric": "net.out_bytes_sec", - "merge": true, - "dimensions": "", - "period": "", - "condition_filter": true, - "condition_key": "service", - "condition_value": "monitoring" - } - ], - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - } - ], - "showTitle": true - }, - { - "title": "Kafka", - "height": "250px", - "editable": true, - "collapse": false, - "panels": [ - { - "title": "CPU usage", - "error": false, - "span": 4, - "editable": true, - "type": "graph", - "id": 60, - "datasource": null, - "renderer": "flot", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "percent", - "short" - ], - "grid": { - "leftMax": null, - "rightMax": null, - "leftMin": 0, - "rightMin": null, - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 0, - "linewidth": 1, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false, - "alignAsTable": true - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true - }, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.cpu_perc", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "kafka"}] - } - ], - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - }, - { - "title": "Allocated memory", - "error": false, - "span": 4, - "editable": true, - "type": "graph", - "id": 59, - "datasource": null, - "renderer": "flot", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "none", - "short" - ], - "grid": { - "leftMax": null, - "rightMax": null, - "leftMin": 0, - "rightMin": null, - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 0, - "linewidth": 1, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false, - "alignAsTable": true - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true - }, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.mem.rss_mbytes", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "kafka"}] - } - ], - "aliasColors": {}, - "seriesOverrides": [], - "leftYAxisLabel": "MB", - "links": [] - }, - { - "title": "PID count", - "error": false, - "span": 4, - "editable": true, - "type": "graph", - "id": 89, - "datasource": null, - "renderer": "flot", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "none", - "short" - ], - "grid": { - "leftMax": null, - "rightMax": null, - "leftMin": 0, - "rightMin": null, - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 0, - "linewidth": 1, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false, - "alignAsTable": true - }, - "nullPointMode": "connected", - "steppedLine": true, - "tooltip": { - "value_type": "cumulative", - "shared": true - }, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "process.pid_count", - "condition_filter": true, - "dimensions": [{"key": "service", "value": "kafka"}] - } - ], - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - }, - { - "title": "Consumer lag", - "error": false, - "span": 12, - "editable": true, - "type": "graph", - "id": 58, - "datasource": null, - "renderer": "flot", - "x-axis": true, - "y-axis": true, - "y_formats": [ - "none", - "short" - ], - "grid": { - "leftMax": null, - "rightMax": null, - "leftMin": 0, - "rightMin": null, - "threshold1": null, - "threshold2": null, - "threshold1Color": "rgba(216, 200, 27, 0.27)", - "threshold2Color": "rgba(234, 112, 112, 0.22)" - }, - "lines": true, - "fill": 0, - "linewidth": 1, - "points": false, - "pointradius": 5, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false, - "alignAsTable": true - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "cumulative", - "shared": true - }, - "targets": [ - { - "aggregator": "none", - "column": "value", - "metric": "kafka.consumer_lag", - "refId": "A", - "period": "300", - "dimensions": [ - { - "key": "consumer_group", - "value": "$all" - } - ], - "error": "" - } - ], - "aliasColors": {}, - "seriesOverrides": [], - "links": [] - } - ], - "showTitle": true - }, - { - "title": "Components", - "panels": [ - { - "id": 167, - "title": "Common", - "span": 4, - "type": "dashlist", - "query": "", - "limit": 10, - "tags": [ - "common" - ], - "recent": false, - "search": true, - "starred": false, - "headings": false, - "links": [] - }, - { - "id": 168, - "title": "Metrics", - "span": 4, - "type": "dashlist", - "query": "", - "limit": 10, - "tags": [ - "metrics" - ], - "recent": false, - "search": true, - "starred": false, - "headings": false, - "links": [] - }, - { - "headings": false, - "id": 169, - "limit": 10, - "links": [], - "query": "", - "recent": false, - "search": true, - "span": 4, - "starred": false, - "tags": [ - "logs" - ], - "title": "Logs", - "type": "dashlist" - } - ] - } - ], - "time": { - "from": "now-1h", - "to": "now" - }, - "templating": { - "list": [], - "enable": false - }, - "annotations": { - "enable": false, - "list": [] - }, - "refresh": "30s", - "version": 6, - "hideAllLegends": false -} diff --git a/devstack/files/grafana/dashboards.d/20-kibana.json b/devstack/files/grafana/dashboards.d/20-kibana.json deleted file mode 100644 index 712afb00..00000000 --- a/devstack/files/grafana/dashboards.d/20-kibana.json +++ /dev/null @@ -1,624 +0,0 @@ -{ - "id": null, - "title": "Kibana", - "tags": [ - "logs" - ], - "style": "dark", - "timezone": "browser", - "editable": true, - "sharedCrosshair": false, - "hideControls": false, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "schemaVersion": 13, - "version": 7, - "links": [], - "gnetId": null, - "rows": [ - { - "title": "Dashboard Row", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 1, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "aggregator": "none", - "dimensions": [ - { - "key": "service", - "value": "kibana" - } - ], - "error": "", - "metric": "process.pid_count", - "period": "300", - "refId": "A" - } - ], - "thresholds": "0.2,0.8", - "title": "Kibana", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - }, - { - "value": "9", - "op": "=", - "text": "UP" - }, - { - "value": "10", - "op": "=", - "text": "UP" - }, - { - "value": "11", - "op": "=", - "text": "UP" - }, - { - "value": "12", - "op": "=", - "text": "UP" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "kibana" - } - ], - "error": "", - "metric": "process.cpu_perc", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "kibana" - } - ], - "error": "", - "metric": "process.mem.rss_mbytes", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": "250px", - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "kibana" - } - ], - "error": "", - "metric": "process.io.read_count", - "period": "300", - "refId": "A" - }, - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "kibana" - } - ], - "error": "", - "metric": "process.io.write_count", - "period": "300", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IO Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "kibana" - } - ], - "error": "", - "metric": "process.io.read_kbytes", - "period": "300", - "refId": "A" - }, - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "kibana" - } - ], - "error": "", - "metric": "process.io.write_kbytes", - "period": "300", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IO Read/Write [kB]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "kbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "kibana" - } - ], - "error": "", - "metric": "process.open_file_descriptors", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Open File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - } - ] -} diff --git a/devstack/files/grafana/dashboards.d/21-logapi.json b/devstack/files/grafana/dashboards.d/21-logapi.json deleted file mode 100644 index 7d6e15ef..00000000 --- a/devstack/files/grafana/dashboards.d/21-logapi.json +++ /dev/null @@ -1,624 +0,0 @@ -{ - "id": null, - "title": "Log API", - "tags": [ - "logs" - ], - "style": "dark", - "timezone": "browser", - "editable": true, - "sharedCrosshair": false, - "hideControls": false, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "schemaVersion": 13, - "version": 7, - "links": [], - "gnetId": null, - "rows": [ - { - "title": "Dashboard Row", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 1, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "aggregator": "none", - "dimensions": [ - { - "key": "service", - "value": "log-api" - } - ], - "error": "", - "metric": "process.pid_count", - "period": "300", - "refId": "A" - } - ], - "thresholds": "0.2,0.8", - "title": "Log API", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - }, - { - "value": "9", - "op": "=", - "text": "UP" - }, - { - "value": "10", - "op": "=", - "text": "UP" - }, - { - "value": "11", - "op": "=", - "text": "UP" - }, - { - "value": "12", - "op": "=", - "text": "UP" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-api" - } - ], - "error": "", - "metric": "process.cpu_perc", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-api" - } - ], - "error": "", - "metric": "process.mem.rss_mbytes", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": "250px", - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-api" - } - ], - "error": "", - "metric": "process.io.read_count", - "period": "300", - "refId": "A" - }, - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-api" - } - ], - "error": "", - "metric": "process.io.write_count", - "period": "300", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IO Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-api" - } - ], - "error": "", - "metric": "process.io.read_kbytes", - "period": "300", - "refId": "A" - }, - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-api" - } - ], - "error": "", - "metric": "process.io.write_kbytes", - "period": "300", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IO Read/Write [kB]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "kbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-api" - } - ], - "error": "", - "metric": "process.open_file_descriptors", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Open File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - } - ] -} diff --git a/devstack/files/grafana/dashboards.d/22-logtransformer.json b/devstack/files/grafana/dashboards.d/22-logtransformer.json deleted file mode 100644 index dd9dae5c..00000000 --- a/devstack/files/grafana/dashboards.d/22-logtransformer.json +++ /dev/null @@ -1,624 +0,0 @@ -{ - "id": null, - "title": "Log Transformer", - "tags": [ - "logs" - ], - "style": "dark", - "timezone": "browser", - "editable": true, - "sharedCrosshair": false, - "hideControls": false, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "schemaVersion": 13, - "version": 7, - "links": [], - "gnetId": null, - "rows": [ - { - "title": "Dashboard Row", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 1, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "aggregator": "none", - "dimensions": [ - { - "key": "service", - "value": "log-transformer" - } - ], - "error": "", - "metric": "process.pid_count", - "period": "300", - "refId": "A" - } - ], - "thresholds": "0.2,0.8", - "title": "Log Transformer", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - }, - { - "value": "9", - "op": "=", - "text": "UP" - }, - { - "value": "10", - "op": "=", - "text": "UP" - }, - { - "value": "11", - "op": "=", - "text": "UP" - }, - { - "value": "12", - "op": "=", - "text": "UP" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-transformer" - } - ], - "error": "", - "metric": "process.cpu_perc", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-transformer" - } - ], - "error": "", - "metric": "process.mem.rss_mbytes", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": "250px", - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-transformer" - } - ], - "error": "", - "metric": "process.io.read_count", - "period": "300", - "refId": "A" - }, - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-transformer" - } - ], - "error": "", - "metric": "process.io.write_count", - "period": "300", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IO Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-transformer" - } - ], - "error": "", - "metric": "process.io.read_kbytes", - "period": "300", - "refId": "A" - }, - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-transformer" - } - ], - "error": "", - "metric": "process.io.write_kbytes", - "period": "300", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IO Read/Write [kB]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "kbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-transformer" - } - ], - "error": "", - "metric": "process.open_file_descriptors", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Open File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - } - ] -} diff --git a/devstack/files/grafana/dashboards.d/23-logtmetrics.json b/devstack/files/grafana/dashboards.d/23-logtmetrics.json deleted file mode 100644 index eb44a73d..00000000 --- a/devstack/files/grafana/dashboards.d/23-logtmetrics.json +++ /dev/null @@ -1,624 +0,0 @@ -{ - "id": null, - "title": "Log Metrics", - "tags": [ - "logs" - ], - "style": "dark", - "timezone": "browser", - "editable": true, - "sharedCrosshair": false, - "hideControls": false, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "schemaVersion": 13, - "version": 7, - "links": [], - "gnetId": null, - "rows": [ - { - "title": "Dashboard Row", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 1, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "aggregator": "none", - "dimensions": [ - { - "key": "service", - "value": "log-metrics" - } - ], - "error": "", - "metric": "process.pid_count", - "period": "300", - "refId": "A" - } - ], - "thresholds": "0.2,0.8", - "title": "Log Metrics", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - }, - { - "value": "9", - "op": "=", - "text": "UP" - }, - { - "value": "10", - "op": "=", - "text": "UP" - }, - { - "value": "11", - "op": "=", - "text": "UP" - }, - { - "value": "12", - "op": "=", - "text": "UP" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-metrics" - } - ], - "error": "", - "metric": "process.cpu_perc", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-metrics" - } - ], - "error": "", - "metric": "process.mem.rss_mbytes", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": "250px", - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-metrics" - } - ], - "error": "", - "metric": "process.io.read_count", - "period": "300", - "refId": "A" - }, - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-metrics" - } - ], - "error": "", - "metric": "process.io.write_count", - "period": "300", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IO Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-metrics" - } - ], - "error": "", - "metric": "process.io.read_kbytes", - "period": "300", - "refId": "A" - }, - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-metrics" - } - ], - "error": "", - "metric": "process.io.write_kbytes", - "period": "300", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IO Read/Write [kB]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "kbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-metrics" - } - ], - "error": "", - "metric": "process.open_file_descriptors", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Open File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - } - ] -} diff --git a/devstack/files/grafana/dashboards.d/24-logpersister.json b/devstack/files/grafana/dashboards.d/24-logpersister.json deleted file mode 100644 index 28a1adec..00000000 --- a/devstack/files/grafana/dashboards.d/24-logpersister.json +++ /dev/null @@ -1,624 +0,0 @@ -{ - "id": null, - "title": "Log Persister", - "tags": [ - "logs" - ], - "style": "dark", - "timezone": "browser", - "editable": true, - "sharedCrosshair": false, - "hideControls": false, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "templating": { - "list": [] - }, - "annotations": { - "list": [] - }, - "schemaVersion": 13, - "version": 7, - "links": [], - "gnetId": null, - "rows": [ - { - "title": "Dashboard Row", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": true, - "colorValue": false, - "colors": [ - "rgba(225, 40, 40, 0.59)", - "rgba(245, 150, 40, 0.73)", - "rgba(71, 212, 59, 0.4)" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 1, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "aggregator": "none", - "dimensions": [ - { - "key": "service", - "value": "log-persister" - } - ], - "error": "", - "metric": "process.pid_count", - "period": "300", - "refId": "A" - } - ], - "thresholds": "0.2,0.8", - "title": "Log Persister", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "value": "0", - "op": "=", - "text": "DOWN" - }, - { - "value": "1", - "op": "=", - "text": "UP" - }, - { - "value": "2", - "op": "=", - "text": "UP" - }, - { - "value": "3", - "op": "=", - "text": "UP" - }, - { - "value": "4", - "op": "=", - "text": "UP" - }, - { - "value": "5", - "op": "=", - "text": "UP" - }, - { - "value": "6", - "op": "=", - "text": "UP" - }, - { - "value": "7", - "op": "=", - "text": "UP" - }, - { - "value": "8", - "op": "=", - "text": "UP" - }, - { - "value": "9", - "op": "=", - "text": "UP" - }, - { - "value": "10", - "op": "=", - "text": "UP" - }, - { - "value": "11", - "op": "=", - "text": "UP" - }, - { - "value": "12", - "op": "=", - "text": "UP" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-persister" - } - ], - "error": "", - "metric": "process.cpu_perc", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-persister" - } - ], - "error": "", - "metric": "process.mem.rss_mbytes", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": "250px", - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-persister" - } - ], - "error": "", - "metric": "process.io.read_count", - "period": "300", - "refId": "A" - }, - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-persister" - } - ], - "error": "", - "metric": "process.io.write_count", - "period": "300", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IO Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-persister" - } - ], - "error": "", - "metric": "process.io.read_kbytes", - "period": "300", - "refId": "A" - }, - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-persister" - } - ], - "error": "", - "metric": "process.io.write_kbytes", - "period": "300", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "IO Read/Write [kB]", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "kbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - }, - { - "title": "Dashboard Row", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": null, - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "aggregator": "avg", - "dimensions": [ - { - "key": "process_name", - "value": "log-persister" - } - ], - "error": "", - "metric": "process.open_file_descriptors", - "period": "300", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Open File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "titleSize": "h6", - "height": 250, - "repeat": null, - "repeatRowId": null, - "repeatIteration": null, - "collapse": false - } - ] -} diff --git a/devstack/files/grafana/grafana.py b/devstack/files/grafana/grafana.py deleted file mode 100644 index c78d119d..00000000 --- a/devstack/files/grafana/grafana.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 - -# (C) Copyright 2017 Hewlett Packard Enterprise Development LP -# (C) Copyright 2018 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import glob -import json -import logging -import os -import sys -import time - -from requests import RequestException -from requests import Session - -LOG_LEVEL = logging.getLevelName(os.environ.get('LOG_LEVEL', 'INFO')) -logging.basicConfig(level=LOG_LEVEL) - -logger = logging.getLogger(__name__) - -GRAFANA_URL = os.environ.get('GRAFANA_URL', 'http://localhost:3000') -GRAFANA_USERNAME = os.environ.get('GRAFANA_USERNAME', 'mini-mon') -GRAFANA_PASSWORD = os.environ.get('GRAFANA_PASSWORD', 'password') -GRAFANA_USERS = [{'user': GRAFANA_USERNAME, 'password': GRAFANA_PASSWORD, 'email': ''}] - -DASHBOARDS_DIR = sys.argv[1] - - -def retry(retries=5, delay=2.0, exc_types=(RequestException,)): - def decorator(func): - def f_retry(*args, **kwargs): - for i in range(retries): - try: - return func(*args, **kwargs) - except exc_types as exc: - if i < retries - 1: - logger.debug('Caught exception, retrying...', - exc_info=True) - time.sleep(delay) - else: - logger.exception('Failed after %d attempts', retries) - if isinstance(exc, RequestException): - logger.debug('Response was: %r', exc.response.text) - - raise - return f_retry - return decorator - - -def create_login_payload(): - if os.environ.get('GRAFANA_USERS'): - try: - json.loads(os.environ.get('GRAFANA_USERS')) - except ValueError: - print("Invalid type GRAFANA_USERS") - raise - grafana_users = json.loads(os.environ.get('GRAFANA_USERS')) - else: - grafana_users = GRAFANA_USERS - return grafana_users - - -@retry(retries=24, delay=5.0) -def login(session, user): - r = session.post('{url}/login'.format(url=GRAFANA_URL), - json=user, - timeout=5) - r.raise_for_status() - - -def create_dashboard_payload(json_path): - with open(json_path, 'r') as f: - dashboard = json.load(f) - dashboard['id'] = None - - return { - 'dashboard': dashboard, - 'overwrite': True - } - - -def main(): - for user in create_login_payload(): - logging.info('Opening a Grafana session...') - session = Session() - login(session, user) - - for path in sorted(glob.glob('{dir}/*.json'.format(dir=DASHBOARDS_DIR))): - logging.info('Creating dashboard from file: {path}'.format(path=path)) - r = session.post('{url}/api/dashboards/db'.format(url=GRAFANA_URL), - json=create_dashboard_payload(path)) - logging.debug('Response: %r', r.json()) - r.raise_for_status() - - logging.info('Ending %r session...', user.get('user')) - session.get('{url}/logout'.format(url=GRAFANA_URL)) - - logging.info('Finished successfully.') - - -if __name__ == '__main__': - main() diff --git a/devstack/files/kibana/kibana.yml b/devstack/files/kibana/kibana.yml deleted file mode 100644 index 0f7b02b4..00000000 --- a/devstack/files/kibana/kibana.yml +++ /dev/null @@ -1,78 +0,0 @@ -# Kibana is served by a back end server. This controls which port to use. -server.port: %KIBANA_SERVICE_PORT% - -# The host to bind the server to. -server.host: %KIBANA_SERVICE_HOST% - -# If you are running kibana behind a proxy, and want to mount it at a path, -# specify that path here. The basePath can't end in a slash. -server.basePath: /dashboard/monitoring/logs_proxy - -# The Elasticsearch instance to use for all your queries. -elasticsearch.url: http://%ES_SERVICE_BIND_HOST%:%ES_SERVICE_BIND_PORT% - -# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false, -# then the host you use to connect to *this* Kibana instance will be sent. -elasticsearch.preserveHost: True - -# Kibana uses an index in Elasticsearch to store saved searches, visualizations -# and dashboards. It will create a new index if it doesn't already exist. -kibana.index: ".kibana" - -# The default application to load. -kibana.defaultAppId: "discover" - -# If your Elasticsearch is protected with basic auth, these are the user credentials -# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana -# users will still need to authenticate with Elasticsearch (which is proxied through -# the Kibana server) -# elasticsearch.username: "user" -# elasticsearch.password: "pass" - -# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted) -# server.ssl.cert: /path/to/your/server.crt -# server.ssl.key: /path/to/your/server.key - -# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted) -# elasticsearch.ssl.cert: /path/to/your/client.crt -# elasticsearch.ssl.key: /path/to/your/client.key - -# If you need to provide a CA certificate for your Elasticsearch instance, put -# the path of the pem file here. -# elasticsearch.ssl.ca: /path/to/your/CA.pem - -# Set to false to have a complete disregard for the validity of the SSL -# certificate. -# elasticsearch.ssl.verify: true - -# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to -# request_timeout setting -elasticsearch.pingTimeout: 1500 - -# Time in milliseconds to wait for responses from the back end or elasticsearch. -# This must be > 0 -elasticsearch.requestTimeout: 300000 - -# Time in milliseconds for Elasticsearch to wait for responses from shards. -# Set to 0 to disable. -elasticsearch.shardTimeout: 0 - -# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying -elasticsearch.startupTimeout: 5000 - -# Set the path to where you would like the process id file to be created. -# pid.file: /var/run/kibana.pid - -# Set this to true to suppress all logging output. -logging.silent: false -# Set this to true to suppress all logging output except for error messages. -logging.quiet: false -# Set this to true to log all events, including system usage information and all requests. -logging.verbose: true - -# monasca-kibana-plugin configuration -monasca-kibana-plugin.auth_uri: %KEYSTONE_AUTH_URI% -monasca-kibana-plugin.enabled: True -monasca-kibana-plugin.cookie.isSecure: False - -optimize.useBundleCache: False diff --git a/devstack/files/monasca-agent/elastic.yaml b/devstack/files/monasca-agent/elastic.yaml deleted file mode 100644 index 269fdae7..00000000 --- a/devstack/files/monasca-agent/elastic.yaml +++ /dev/null @@ -1,3 +0,0 @@ -init_config: -instances: -- url: http://{{IP}}:9200 diff --git a/devstack/files/monasca-agent/http_check.yaml b/devstack/files/monasca-agent/http_check.yaml deleted file mode 100644 index 97424e55..00000000 --- a/devstack/files/monasca-agent/http_check.yaml +++ /dev/null @@ -1,27 +0,0 @@ -init_config: null -instances: -- name: keystone - dimensions: - service: keystone - timeout: 3 - url: http://127.0.0.1/identity -- name: mysql - dimensions: - service: mysql - timeout: 3 - url: http://127.0.0.1:3306 -- name: influxdb - dimensions: - service: influxdb - timeout: 3 - url: http://127.0.0.1:8086/ping -- name: elasticsearch - dimensions: - service: elasticsearch - timeout: 3 - url: http://{{IP}}:9200/_cat/health -- name: kibana - dimensions: - service: kibana - timeout: 3 - url: http://{{IP}}:5601/status diff --git a/devstack/files/monasca-agent/process.yaml b/devstack/files/monasca-agent/process.yaml deleted file mode 100644 index d8878bbd..00000000 --- a/devstack/files/monasca-agent/process.yaml +++ /dev/null @@ -1,143 +0,0 @@ -init_config: - -instances: -- name: influxd - detailed: true - dimensions: - service: influxd - exact_match: false - search_string: - - influxd -- name: monasca-statsd - detailed: true - dimensions: - service: monasca-statsd - exact_match: false - search_string: - - monasca-statsd -- name: monasca-notification - detailed: true - dimensions: - service: monasca-notification - exact_match: false - search_string: - - monasca-notification -- name: persister - detailed: true - dimensions: - service: persister - exact_match: false - search_string: - - persister -- name: storm - detailed: true - dimensions: - service: storm - exact_match: false - search_string: - - storm -- name: monasca-api - detailed: true - dimensions: - service: uwsgi - exact_match: false - search_string: - - uwsgi -- name: monasca-collector - detailed: true - dimensions: - service: monasca-collector - exact_match: false - search_string: - - monasca-collector -- name: memcached - detailed: true - dimensions: - service: memcached - exact_match: false - search_string: - - memcached -- name: monasca-forwarder - detailed: true - dimensions: - service: monasca-forwarder - exact_match: false - search_string: - - monasca-forwarder -- name: zookeeper - detailed: true - dimensions: - service: zookeeper - exact_match: false - search_string: - - zookeeper -- name: kafka - detailed: true - dimensions: - service: kafka - exact_match: false - search_string: - - kafka -- name: mysqld - detailed: true - dimensions: - service: mysqld - exact_match: false - search_string: - - mysqld -- name: logspout - detailed: true - dimensions: - service: logspout - exact_match: false - search_string: - - logspout -- name: log-agent - detailed: true - dimensions: - service: log-agent - exact_match: false - search_string: - - log-agent -- name: log-api - detailed: true - dimensions: - service: log-api - exact_match: false - search_string: - - log-api -- name: kibana - detailed: true - dimensions: - service: kibana - exact_match: false - search_string: - - kibana -- name: elasticsearch - detailed: true - dimensions: - service: elasticsearch - exact_match: false - search_string: - - elasticsearch -- name: log-transformer - detailed: true - dimensions: - service: log-transformer - exact_match: false - search_string: - - log-transformer -- name: log-persister - detailed: true - dimensions: - service: log-persister - exact_match: false - search_string: - - log-persister -- name: log-metrics - detailed: true - dimensions: - service: log-metrics - exact_match: false - search_string: - - log-metrics diff --git a/devstack/files/monasca-log-agent/agent.conf b/devstack/files/monasca-log-agent/agent.conf deleted file mode 100644 index 956fae94..00000000 --- a/devstack/files/monasca-log-agent/agent.conf +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -input { - file { - add_field => { "dimensions" => { "service" => "system" }} - path => "/var/log/syslog" - tags => ["syslog"] - } -} - -filter { - if "syslog" in [tags] { - multiline { - negate => "true" - pattern => "^%{SYSLOGTIMESTAMP}" - what => "previous" - } - } -} - -# TODO(trebskit) should use own user for log-agent -output { - monasca_log_api { - monasca_log_api_url => "%MONASCA_LOG_API_URI_V3%" - keystone_api_url => "%KEYSTONE_AUTH_URI_V3%" - project_name => "mini-mon" - username => "monasca-agent" - password => "password" - user_domain_name => "default" - project_domain_name => "default" - dimensions => [ "hostname:devstack" ] - } -} diff --git a/devstack/files/monasca-log-metrics/log-metrics.conf b/devstack/files/monasca-log-metrics/log-metrics.conf deleted file mode 100644 index b0bb5990..00000000 --- a/devstack/files/monasca-log-metrics/log-metrics.conf +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -input { - kafka { - zk_connect => "127.0.0.1:2181" - topic_id => "transformed-log" - group_id => "log-metric" - consumer_id => "monasca_log_metrics" - consumer_threads => "4" - } -} - - -filter { - - # drop logs that have not set log level - if "level" not in [log] { - drop { periodic_flush => true } - } else { - ruby { - code => " - log_level = event['log']['level'].downcase - event['log']['level'] = log_level - " - } - } - - # drop logs with log level not in warning,error - if [log][level] not in [warning,error] { - drop { periodic_flush => true } - } - - ruby { - code => " - log_level = event['log']['level'].downcase - log_ts = Time.now.to_f * 1000.0 - - # metric name - metric_name = 'log.%s' % log_level - - # build metric - metric = {} - metric['name'] = metric_name - metric['timestamp'] = log_ts - metric['value'] = 1 - metric['dimensions'] = event['log']['dimensions'] - metric['value_meta'] = {} - - event['metric'] = metric.to_hash - " - } - - mutate { - remove_field => ["log", "@version", "@timestamp", "log_level_original", "tags"] - } - -} - - -output { - kafka { - bootstrap_servers => "%KAFKA_SERVICE_HOST%:%KAFKA_SERVICE_PORT%" - topic_id => "metrics" - client_id => "monasca_log_metrics" - compression_type => "none" - } -} diff --git a/devstack/files/monasca-log-persister/persister.conf b/devstack/files/monasca-log-persister/persister.conf deleted file mode 100644 index c0f581e3..00000000 --- a/devstack/files/monasca-log-persister/persister.conf +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -input { - kafka { - zk_connect => "127.0.0.1:2181" - topic_id => "transformed-log" - group_id => "logstash-persister" - } -} - -filter { - date { - match => ["[log][timestamp]", "UNIX"] - target => "@timestamp" - } - - date { - match => ["creation_time", "UNIX"] - target => "creation_time" - } - - grok { - match => { - "[@timestamp]" => "^(?\d{4}-\d{2}-\d{2})" - } - } - - if "dimensions" in [log] { - ruby { - code => " - fieldHash = event['log']['dimensions'] - fieldHash.each do |key, value| - event[key] = value - end - " - } - } - - mutate { - add_field => { - message => "%{[log][message]}" - log_level => "%{[log][level]}" - tenant => "%{[meta][tenantId]}" - region => "%{[meta][region]}" - } - remove_field => ["@version", "host", "type", "tags" ,"_index_date", "meta", "log"] - } -} - -output { - elasticsearch { - index => "logs-%{tenant}-%{index_date}" - document_type => "log" - hosts => ["%ES_SERVICE_BIND_HOST%"] - flush_size => 500 - } -} diff --git a/devstack/files/monasca-log-transformer/transformer.conf b/devstack/files/monasca-log-transformer/transformer.conf deleted file mode 100644 index 0e2f7c7f..00000000 --- a/devstack/files/monasca-log-transformer/transformer.conf +++ /dev/null @@ -1,87 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -input { - kafka { - zk_connect => "127.0.0.1:2181" - topic_id => "log" - group_id => "transformer-logstash-consumer" - } -} - -filter { - ruby { - code => "event['message_tmp'] = event['log']['message'][0..49]" - } - grok { - match => { - "[message_tmp]" => "(?i)(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|ERR(OR)?|WARN(ING)?)|\"level\":\s?(?\d{2})" - } - } - if ! [log_level] { - grok { - match => { - "[log][message]" => "(?i)(?AUDIT|CRITICAL|DEBUG|INFO|TRACE|ERR(OR)?|WARN(ING)?)|\"level\":\s?(?\d{2})" - } - } - } - ruby { - init => " - LOG_LEVELS_MAP = { - # SYSLOG - 'warn' => :Warning, - 'err' => :Error, - # Bunyan errcodes - '10' => :Trace, - '20' => :Debug, - '30' => :Info, - '40' => :Warning, - '50' => :Error, - '60' => :Fatal - } - " - code => " - if event['log_level'] - # keep original value - log_level = event['log_level'].downcase - if LOG_LEVELS_MAP.has_key?(log_level) - event['log_level_original'] = event['log_level'] - event['log_level'] = LOG_LEVELS_MAP[log_level] - else - event['log_level'] = log_level.capitalize - end - else - event['log_level'] = 'Unknown' - end - " - } - - mutate { - add_field => { - "[log][level]" => "%{log_level}" - } - - # remove temporary fields - remove_field => ["log_level", "message_tmp"] - } -} - -output { - kafka { - bootstrap_servers => "%KAFKA_SERVICE_HOST%:%KAFKA_SERVICE_PORT%" - topic_id => "transformed-log" - } -} diff --git a/devstack/lib/config.sh b/devstack/lib/config.sh deleted file mode 100644 index 83064c24..00000000 --- a/devstack/lib/config.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash - -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Non configurable settings or settings derived from another settings - -_XTRACE_LOG_API_CONFIG=$(set +o | grep xtrace) -set +o xtrace - -if [[ ${USE_VENV} = True ]]; then - PROJECT_VENV["monasca-log-api"]=${MONASCA_LOG_API_DIR}.venv - MONASCA_LOG_API_BIN_DIR=${PROJECT_VENV["monasca-log-api"]}/bin -else - MONASCA_LOG_API_BIN_DIR=$(get_python_exec_prefix) -fi - -MONASCA_LOG_API_WSGI=$MONASCA_LOG_API_BIN_DIR/monasca-log-api-wsgi -MONASCA_LOG_API_DEPLOY=`determine_log_api_deploy_mode` -if is_service_enabled tls-proxy; then - MONASCA_LOG_API_SERVICE_PROTOCOL="https" -fi -if [ "$MONASCA_LOG_API_USE_MOD_WSGI" == "True" ]; then - MONASCA_LOG_API_BASE_URI=${MONASCA_LOG_API_SERVICE_PROTOCOL}://${MONASCA_LOG_API_SERVICE_HOST}/logs -else - MONASCA_LOG_API_BASE_URI=${MONASCA_LOG_API_SERVICE_PROTOCOL}://${MONASCA_LOG_API_SERVICE_HOST}:${MONASCA_LOG_API_SERVICE_PORT} -fi -MONASCA_LOG_API_URI_V2=${MONASCA_LOG_API_BASE_URI}/v2.0 -MONASCA_LOG_API_URI_V3=${MONASCA_LOG_API_BASE_URI}/v3.0 - -MONASCA_LOG_API_CONF_DIR=${MONASCA_LOG_API_CONF_DIR:-/etc/monasca} -MONASCA_LOG_API_LOG_DIR=${MONASCA_LOG_API_LOG_DIR:-/var/log/monasca} -MONASCA_LOG_API_CACHE_DIR=${MONASCA_LOG_API_CACHE_DIR:-/var/cache/monasca-log-api} -MONASCA_LOG_API_WSGI_DIR=${MONASCA_LOG_API_WSGI_DIR:-/var/www/monasca-log-api} - -MONASCA_LOG_API_CONF=${MONASCA_LOG_API_CONF:-$MONASCA_LOG_API_CONF_DIR/monasca-log-api.conf} -MONASCA_LOG_API_PASTE=${MONASCA_LOG_API_PASTE:-$MONASCA_LOG_API_CONF_DIR/log-api-paste.ini} -MONASCA_LOG_API_LOGGING_CONF=${MONASCA_LOG_API_LOGGING_CONF:-$MONASCA_LOG_API_CONF_DIR/log-api-logging.conf} -MONASCA_LOG_API_UWSGI_CONF=${MONASCA_LOG_API_UWSGI_CONF:-$MONASCA_LOG_API_CONF_DIR/log-api-uwsgi.ini} - -MONASCA_LOG_API_USE_MOD_WSGI=${MONASCA_LOG_API_USE_MOD_WSGI:-$ENABLE_HTTPD_MOD_WSGI_SERVICES} - -# configuration bits of various services -LOG_PERSISTER_DIR=$DEST/monasca-log-persister -LOG_TRANSFORMER_DIR=$DEST/monasca-log-transformer -LOG_METRICS_DIR=$DEST/monasca-log-metrics -LOG_AGENT_DIR=$DEST/monasca-log-agent - -ELASTICSEARCH_DIR=$DEST/elasticsearch -ELASTICSEARCH_CFG_DIR=$ELASTICSEARCH_DIR/config -ELASTICSEARCH_LOG_DIR=$LOGDIR/elasticsearch -ELASTICSEARCH_DATA_DIR=$DATA_DIR/elasticsearch - -KIBANA_DIR=$DEST/kibana -KIBANA_CFG_DIR=$KIBANA_DIR/config - -LOGSTASH_DIR=$DEST/logstash - -PLUGIN_FILES=$MONASCA_LOG_API_DIR/devstack/files -# configuration bits of various services - -# Files inside this directory will be visible in gates log -GATE_CONFIGURATION_DIR=/etc/monasca-log-api - -# clone monasca-{common,statsd} directly from repo -GITREPO["monasca-common"]=${MONASCA_COMMON_REPO} -GITBRANCH["monasca-common"]=${MONASCA_COMMON_BRANCH} -GITDIR["monasca-common"]=${MONASCA_COMMON_DIR} - -GITREPO["monasca-statsd"]=${MONASCA_STATSD_REPO} -GITBRANCH["monasca-statsd"]=${MONASCA_STATSD_BRANCH} -GITDIR["monasca-statsd"]=${MONASCA_STATSD_DIR} - -LIBS_FROM_GIT="${LIBS_FROM_GIT:-""},monasca-common,monasca-statsd" -# clone monasca-{common,statsd} directly from repo - -# public facing bits -MONASCA_LOG_API_SERVICE_HOST=${MONASCA_LOG_API_SERVICE_HOST:-${SERVICE_HOST}} -MONASCA_LOG_API_SERVICE_PORT=${MONASCA_LOG_API_SERVICE_PORT:-5607} -MONASCA_LOG_API_SERVICE_PORT_INT=${MONASCA_LOG_API_SERVICE_PORT:-15607} -MONASCA_LOG_API_SERVICE_PROTOCOL=${MONASCA_LOG_API_SERVICE_PROTOCOL:-${SERVICE_PROTOCOL}} - -ES_SERVICE_BIND_HOST=${ES_SERVICE_BIND_HOST:-${SERVICE_HOST}} -ES_SERVICE_BIND_PORT=${ES_SERVICE_BIND_PORT:-9200} -ES_SERVICE_PUBLISH_HOST=${ES_SERVICE_PUBLISH_HOST:-${SERVICE_HOST}} -ES_SERVICE_PUBLISH_PORT=${ES_SERVICE_PUBLISH_PORT:-9300} - -KIBANA_SERVICE_HOST=${KIBANA_SERVICE_HOST:-${SERVICE_HOST}} -KIBANA_SERVICE_PORT=${KIBANA_SERVICE_PORT:-5601} -KIBANA_SERVER_BASE_PATH=${KIBANA_SERVER_BASE_PATH:-"/dashboard/monitoring/logs_proxy"} - -KAFKA_SERVICE_HOST=${KAFKA_SERVICE_HOST:-${SERVICE_HOST}} -KAFKA_SERVICE_PORT=${KAFKA_SERVICE_PORT:-9092} -# public facing bits - -${_XTRACE_LOG_API_CONFIG} diff --git a/devstack/lib/util.sh b/devstack/lib/util.sh deleted file mode 100644 index b2612964..00000000 --- a/devstack/lib/util.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Set of utility-like methods that are not bound to any particular -# service - -_XTRACE_LOG_API_UTILS=$(set +o | grep xtrace) -set +o xtrace - -run_process_sleep() { - local name=$1 - local cmd=$2 - local sleepTime=${3:-1} - run_process "$name" "$cmd" - sleep ${sleepTime} -} - -is_logstash_required() { - is_service_enabled monasca-log-persister \ - || is_service_enabled monasca-log-transformer \ - || is_service_enabled monasca-log-metrics \ - || is_service_enabled monasca-log-agent \ - && return 0 -} - -# MONASCA_LOG_API_DEPLOY defines how monasca-log-api is deployed, allowed values: -# - mod_wsgi : Run monasca-log-api under Apache HTTPd mod_wsgi -# - uwsgi : Run monasca-log-api under uwsgi -# - gunicorn: Run monasca-log-api under gunicorn -determine_log_api_deploy_mode() { - MONASCA_LOG_API_USE_MOD_WSGI=$(trueorfalse False MONASCA_LOG_API_USE_MOD_WSGI) - if [ "$MONASCA_LOG_API_USE_MOD_WSGI" == "True" ]; then - if [[ "$WSGI_MODE" == "uwsgi" ]]; then - echo "uwsgi" - else - echo "mod_wsgi" - fi - else - echo "gunicorn" - fi -} - -${_XTRACE_LOG_API_UTILS} diff --git a/devstack/plugin.sh b/devstack/plugin.sh deleted file mode 100644 index ff1fcd1d..00000000 --- a/devstack/plugin.sh +++ /dev/null @@ -1,862 +0,0 @@ -#!/bin/bash - -# -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Save trace setting -_XTRACE_LOG_API=$(set +o | grep xtrace) -set -o xtrace - -_ERREXIT_LOG_API=$(set +o | grep errexit) -set -o errexit - -# source lib/* -source ${MONASCA_LOG_API_DIR}/devstack/lib/util.sh -source ${MONASCA_LOG_API_DIR}/devstack/lib/config.sh -# source lib/* - -# TOP_LEVEL functions called from devstack coordinator -############################################################################### -function pre_install { - install_elk - install_nodejs - install_gate_config_holder -} - -function install_monasca_log { - build_kibana_plugin - if is_service_enabled monasca-log-api; then - # install_monasca-log-api is not called directly - # stack_install_service calls it - if python3_enabled; then - enable_python3_package monasca-log-api - fi - stack_install_service monasca-log-api - fi - install_log_agent -} - -function install_elk { - install_logstash - install_elasticsearch - install_kibana -} - -function install_gate_config_holder { - sudo install -d -o $STACK_USER $GATE_CONFIGURATION_DIR -} - -function install_monasca_common { - if use_library_from_git "monasca-common"; then - git_clone_by_name "monasca-common" - setup_dev_lib "monasca-common" - fi -} - -function install_monasca_statsd { - if use_library_from_git "monasca-statsd"; then - git_clone_by_name "monasca-statsd" - setup_dev_lib "monasca-statsd" - fi -} - -function configure_monasca_log { - configure_kafka - configure_elasticsearch - configure_kibana - install_kibana_plugin - - configure_monasca_log_api - configure_monasca_log_transformer - configure_monasca_log_metrics - configure_monasca_log_persister - configure_monasca_log_agent -} - -function init_monasca_log { - enable_log_management - create_log_management_accounts -} - -function init_monasca_grafana_dashboards { - if is_service_enabled horizon; then - echo_summary "Init Grafana dashboards" - - sudo python3 "${PLUGIN_FILES}"/grafana/grafana.py "${PLUGIN_FILES}"/grafana/dashboards.d - fi -} - -function init_agent { - echo_summary "Init Monasca agent" - - sudo cp -f "${PLUGIN_FILES}"/monasca-agent/http_check.yaml /etc/monasca/agent/conf.d/http_check.yaml - sudo cp -f "${PLUGIN_FILES}"/monasca-agent/process.yaml /etc/monasca/agent/conf.d/process.yaml - sudo cp -f "${PLUGIN_FILES}"/monasca-agent/elastic.yaml /etc/monasca/agent/conf.d/elastic.yaml - - sudo sed -i "s/{{IP}}/$(ip -o -4 addr list eth1 | awk '{print $4}' | cut -d/ -f1 | head -1)/" /etc/monasca/agent/conf.d/*.yaml - sudo sed -i "s/127\.0\.0\.1/$(hostname)/" /etc/monasca/agent/conf.d/*.yaml - sudo systemctl restart monasca-collector -} - -function stop_monasca_log { - stop_process "monasca-log-agent" || true - stop_monasca_log_api - stop_process "monasca-log-metrics" || true - stop_process "monasca-log-persister" || true - stop_process "monasca-log-transformer" || true - stop_process "kibana" || true - stop_process "elasticsearch" || true -} - -function start_monasca_log { - start_elasticsearch - start_kibana - start_monasca_log_transformer - start_monasca_log_metrics - start_monasca_log_persister - start_monasca_log_api - start_monasca_log_agent -} - -function clean_monasca_log { - clean_monasca_log_agent - clean_monasca_log_api - clean_monasca_log_persister - clean_monasca_log_transformer - clean_kibana - clean_elasticsearch - clean_logstash - clean_nodejs - clean_gate_config_holder -} -############################################################################### - -function install_monasca-log-api { - echo_summary "Installing monasca-log-api" - - git_clone $MONASCA_LOG_API_REPO $MONASCA_LOG_API_DIR $MONASCA_LOG_API_BRANCH - setup_develop $MONASCA_LOG_API_DIR - - install_keystonemiddleware - install_monasca_common - install_monasca_statsd - - if [ "$MONASCA_LOG_API_DEPLOY" == "mod_wsgi" ]; then - install_apache_wsgi - elif [ "$MONASCA_LOG_API_DEPLOY" == "uwsgi" ]; then - pip_install uwsgi - else - pip_install gunicorn - fi - - if [ "$MONASCA_LOG_API_DEPLOY" != "gunicorn" ]; then - if is_ssl_enabled_service "monasca-log-api"; then - enable_mod_ssl - fi - fi -} - -function configure_monasca_log_api { - if is_service_enabled monasca-log-api; then - echo_summary "Configuring monasca-log-api" - - configure_monasca_log_api_core - if [ "$MONASCA_LOG_API_DEPLOY" == "mod_wsgi" ]; then - configure_monasca_log_api_mod_wsgi - elif [ "$MONASCA_LOG_API_DEPLOY" == "uwsgi" ]; then - configure_monasca_log_api_uwsgi - fi - - # link configuration for the gate - ln -sf $MONASCA_LOG_API_CONF $GATE_CONFIGURATION_DIR - ln -sf $MONASCA_LOG_API_PASTE $GATE_CONFIGURATION_DIR - ln -sf $MONASCA_LOG_API_LOGGING_CONF $GATE_CONFIGURATION_DIR - - fi -} - -function configure_monasca_log_api_core { - # Put config files in ``$MONASCA_LOG_API_CONF_DIR`` for everyone to find - sudo install -d -o $STACK_USER $MONASCA_LOG_API_CONF_DIR - sudo install -m 700 -d -o $STACK_USER $MONASCA_LOG_API_CACHE_DIR - sudo install -d -o $STACK_USER $MONASCA_LOG_API_LOG_DIR - - # ensure fresh installation of configuration files - rm -rf $MONASCA_LOG_API_CONF $MONASCA_LOG_API_PASTE $MONASCA_LOG_API_LOGGING_CONF - - $MONASCA_LOG_API_BIN_DIR/oslo-config-generator \ - --config-file $MONASCA_LOG_API_DIR/config-generator/monasca-log-api.conf \ - --output-file /tmp/monasca-log-api.conf - - install -m 600 /tmp/monasca-log-api.conf $MONASCA_LOG_API_CONF && rm -rf /tmp/monasca-log-api.conf - install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-paste.ini $MONASCA_LOG_API_PASTE - install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-logging.conf $MONASCA_LOG_API_LOGGING_CONF - - # configure monasca-log-api.conf - iniset "$MONASCA_LOG_API_CONF" DEFAULT log_config_append $MONASCA_LOG_API_LOGGING_CONF - iniset "$MONASCA_LOG_API_CONF" service region $REGION_NAME - - iniset "$MONASCA_LOG_API_CONF" log_publisher kafka_url $KAFKA_SERVICE_HOST:$KAFKA_SERVICE_PORT - iniset "$MONASCA_LOG_API_CONF" log_publisher topics log - - iniset "$MONASCA_LOG_API_CONF" kafka_healthcheck kafka_url $KAFKA_SERVICE_HOST:$KAFKA_SERVICE_PORT - iniset "$MONASCA_LOG_API_CONF" kafka_healthcheck kafka_topics log - - iniset "$MONASCA_LOG_API_CONF" roles_middleware path "/v2.0/log,/v3.0/logs" - iniset "$MONASCA_LOG_API_CONF" roles_middleware default_roles monasca-user - iniset "$MONASCA_LOG_API_CONF" roles_middleware agent_roles monasca-agent - iniset "$MONASCA_LOG_API_CONF" roles_middleware delegate_roles admin - - # configure keystone middleware - configure_auth_token_middleware "$MONASCA_LOG_API_CONF" "admin" $MONASCA_LOG_API_CACHE_DIR - iniset "$MONASCA_LOG_API_CONF" keystone_authtoken region_name $REGION_NAME - iniset "$MONASCA_LOG_API_CONF" keystone_authtoken project_name "admin" - iniset "$MONASCA_LOG_API_CONF" keystone_authtoken password $ADMIN_PASSWORD - - # insecure - if is_service_enabled tls-proxy; then - iniset "$MONASCA_LOG_API_CONF" keystone_authtoken insecure False - fi - - # configure log-api-paste.ini - iniset "$MONASCA_LOG_API_PASTE" server:main bind $MONASCA_LOG_API_SERVICE_HOST:$MONASCA_LOG_API_SERVICE_PORT - iniset "$MONASCA_LOG_API_PASTE" server:main chdir $MONASCA_LOG_API_DIR - iniset "$MONASCA_LOG_API_PASTE" server:main workers $API_WORKERS -} - -function configure_monasca_log_api_uwsgi { - rm -rf $MONASCA_LOG_API_UWSGI_CONF - install -m 600 $MONASCA_LOG_API_DIR/etc/monasca/log-api-uwsgi.ini $MONASCA_LOG_API_UWSGI_CONF - - write_uwsgi_config "$MONASCA_LOG_API_UWSGI_CONF" "$MONASCA_LOG_API_WSGI" "/logs" -} - -function configure_monasca_log_api_mod_wsgi { - sudo install -d $MONASCA_LOG_API_WSGI_DIR - - local monasca_log_api_apache_conf - monasca_log_api_apache_conf=$(apache_site_config_for monasca-log-api) - - local monasca_log_api_ssl="" - local monasca_log_api_certfile="" - local monasca_log_api_keyfile="" - local monasca_log_api_api_port=$MONASCA_LOG_API_SERVICE_PORT - local venv_path="" - - if is_ssl_enabled_service monasca_log_api; then - monasca_log_api_ssl="SSLEngine On" - monasca_log_api_certfile="SSLCertificateFile $MONASCA_LOG_API_SSL_CERT" - monasca_log_api_keyfile="SSLCertificateKeyFile $MONASCA_LOG_API_SSL_KEY" - fi - if is_service_enabled tls-proxy; then - monasca_log_api_api_port=$MONASCA_LOG_API_SERVICE_PORT_INT - fi - if [[ ${USE_VENV} = True ]]; then - venv_path="python-path=${PROJECT_VENV["monasca_log_api"]}/lib/$(python_version)/site-packages" - fi - - # copy proxy vhost and wsgi helper files - sudo cp $PLUGIN_FILES/apache-log-api.template $monasca_log_api_apache_conf - sudo sed -e " - s|%PUBLICPORT%|$monasca_log_api_api_port|g; - s|%APACHE_NAME%|$APACHE_NAME|g; - s|%PUBLICWSGI%|$MONASCA_LOG_API_BIN_DIR/monasca-log-api-wsgi|g; - s|%SSLENGINE%|$monasca_log_api_ssl|g; - s|%SSLCERTFILE%|$monasca_log_api_certfile|g; - s|%SSLKEYFILE%|$monasca_log_api_keyfile|g; - s|%USER%|$STACK_USER|g; - s|%VIRTUALENV%|$venv_path|g - s|%APIWORKERS%|$API_WORKERS|g - " -i $monasca_log_api_apache_conf -} - -function create_log_api_cache_dir { - sudo install -m 700 -d -o $STACK_USER $MONASCA_LOG_API_CACHE_DIR -} - -function clean_monasca_log_api { - if is_service_enabled monasca-log-api; then - echo_summary "Cleaning monasca-log-api" - - sudo rm -f $MONASCA_LOG_API_CONF || true - sudo rm -f $MONASCA_LOG_API_PASTE || true - sudo rm -f $MONASCA_LOG_API_LOGGING_CONF || true - sudo rm -rf $MONASCA_LOG_API_CACHE_DIR || true - sudo rm -rf $MONASCA_LOG_API_CONF_DIR || true - - sudo rm -rf $MONASCA_LOG_API_DIR || true - - if [ "$MONASCA_LOG_API_USE_MOD_WSGI" == "True" ]; then - clean_monasca_log_api_wsgi - fi - fi -} - -function clean_monasca_log_api_wsgi { - sudo rm -f $MONASCA_LOG_API_WSGI_DIR/* - sudo rm -f $(apache_site_config_for monasca-log-api) -} - -function start_monasca_log_api { - if is_service_enabled monasca-log-api; then - echo_summary "Starting monasca-log-api" - - local service_port=$MONASCA_LOG_API_SERVICE_PORT - local service_protocol=$MONASCA_LOG_API_SERVICE_PROTOCOL - if is_service_enabled tls-proxy; then - service_port=$MONASCA_LOG_API_SERVICE_PORT_INT - service_protocol="http" - fi - local service_uri - - if [ "$MONASCA_LOG_API_DEPLOY" == "mod_wsgi" ]; then - local enabled_site_file - enabled_site_file=$(apache_site_config_for monasca-log-api) - service_uri=$service_protocol://$MONASCA_LOG_API_SERVICE_HOST/logs/v3.0 - if [ -f ${enabled_site_file} ]; then - enable_apache_site monasca-log-api - restart_apache_server - tail_log monasca-log-api /var/log/$APACHE_NAME/monasca-log-api.log - fi - elif [ "$MONASCA_LOG_API_DEPLOY" == "uwsgi" ]; then - service_uri=$service_protocol://$MONASCA_LOG_API_SERVICE_HOST/logs/v3.0 - run_process "monasca-log-api" "$MONASCA_LOG_API_BIN_DIR/uwsgi --ini $MONASCA_LOG_API_UWSGI_CONF" "" - else - service_uri=$service_protocol://$MONASCA_LOG_API_SERVICE_HOST:$service_port - run_process "monasca-log-api" "$MONASCA_LOG_API_BIN_DIR/gunicorn --paste $MONASCA_LOG_API_PASTE" "" - fi - - echo "Waiting for monasca-log-api to start..." - if ! wait_for_service $SERVICE_TIMEOUT $service_uri; then - die $LINENO "monasca-log-api did not start" - fi - - if is_service_enabled tls-proxy; then - start_tls_proxy monasca-log-api '*' $MONASCA_LOG_API_SERVICE_PORT $MONASCA_LOG_API_SERVICE_HOST $MONASCA_LOG_API_SERVICE_PORT_INT - fi - - restart_service memcached - fi -} - -function stop_monasca_log_api { - if is_service_enabled monasca-log-api; then - if [ "$MONASCA_LOG_API_DEPLOY" == "mod_wsgi" ]; then - disable_apache_site monasca-log-api - restart_apache_server - else - stop_process "monasca-log-api" - if [ "$MONASCA_LOG_API_DEPLOY" == "uwsgi" ]; then - remove_uwsgi_config "$MONASCA_LOG_API_UWSGI_CONF" "$MONASCA_LOG_API_WSGI" - fi - fi - fi -} - -function install_logstash { - if is_logstash_required; then - echo_summary "Installing Logstash ${LOGSTASH_VERSION}" - - local logstash_tarball=logstash-${LOGSTASH_VERSION}.tar.gz - local logstash_url=http://download.elastic.co/logstash/logstash/${logstash_tarball} - - local logstash_dest - logstash_dest=`get_extra_file ${logstash_url}` - - tar xzf ${logstash_dest} -C $DEST - - sudo chown -R $STACK_USER $DEST/logstash-${LOGSTASH_VERSION} - ln -sf $DEST/logstash-${LOGSTASH_VERSION} $LOGSTASH_DIR - fi -} - -function clean_logstash { - if is_logstash_required; then - echo_summary "Cleaning Logstash ${LOGSTASH_VERSION}" - - sudo rm -rf $LOGSTASH_DIR || true - sudo rm -rf $FILES/logstash-${LOGSTASH_VERSION}.tar.gz || true - sudo rm -rf $DEST/logstash-${LOGSTASH_VERSION} || true - fi -} - -function install_elasticsearch { - if is_service_enabled elasticsearch; then - echo_summary "Installing ElasticSearch ${ELASTICSEARCH_VERSION}" - - local es_tarball=elasticsearch-${ELASTICSEARCH_VERSION}.tar.gz - local es_url=https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/${ELASTICSEARCH_VERSION}/${es_tarball} - - local es_dest - es_dest=`get_extra_file ${es_url}` - - tar xzf ${es_dest} -C $DEST - - sudo chown -R $STACK_USER $DEST/elasticsearch-${ELASTICSEARCH_VERSION} - ln -sf $DEST/elasticsearch-${ELASTICSEARCH_VERSION} $ELASTICSEARCH_DIR - fi -} - -function configure_elasticsearch { - if is_service_enabled elasticsearch; then - echo_summary "Configuring ElasticSearch ${ELASTICSEARCH_VERSION}" - - local templateDir=$ELASTICSEARCH_CFG_DIR/templates - - for dir in $ELASTICSEARCH_LOG_DIR $templateDir $ELASTICSEARCH_DATA_DIR; do - sudo install -m 755 -d -o $STACK_USER $dir - done - - sudo cp -f "${PLUGIN_FILES}"/elasticsearch/elasticsearch.yml $ELASTICSEARCH_CFG_DIR/elasticsearch.yml - sudo chown -R $STACK_USER $ELASTICSEARCH_CFG_DIR/elasticsearch.yml - sudo chmod 0644 $ELASTICSEARCH_CFG_DIR/elasticsearch.yml - - sudo sed -e " - s|%ES_SERVICE_BIND_HOST%|$ES_SERVICE_BIND_HOST|g; - s|%ES_SERVICE_BIND_PORT%|$ES_SERVICE_BIND_PORT|g; - s|%ES_SERVICE_PUBLISH_HOST%|$ES_SERVICE_PUBLISH_HOST|g; - s|%ES_SERVICE_PUBLISH_PORT%|$ES_SERVICE_PUBLISH_PORT|g; - s|%ES_DATA_DIR%|$ELASTICSEARCH_DATA_DIR|g; - s|%ES_LOG_DIR%|$ELASTICSEARCH_LOG_DIR|g; - " -i $ELASTICSEARCH_CFG_DIR/elasticsearch.yml - - ln -sf $ELASTICSEARCH_CFG_DIR/elasticsearch.yml $GATE_CONFIGURATION_DIR/elasticsearch.yml - fi -} - -function clean_elasticsearch { - if is_service_enabled elasticsearch; then - echo_summary "Cleaning Elasticsearch ${ELASTICSEARCH_VERSION}" - - sudo rm -rf ELASTICSEARCH_DIR || true - sudo rm -rf ELASTICSEARCH_CFG_DIR || true - sudo rm -rf ELASTICSEARCH_LOG_DIR || true - sudo rm -rf ELASTICSEARCH_DATA_DIR || true - sudo rm -rf $FILES/elasticsearch-${ELASTICSEARCH_VERSION}.tar.gz || true - sudo rm -rf $DEST/elasticsearch-${ELASTICSEARCH_VERSION} || true - fi -} - -function start_elasticsearch { - if is_service_enabled elasticsearch; then - echo_summary "Starting ElasticSearch ${ELASTICSEARCH_VERSION}" - # 5 extra seconds to ensure that ES started properly - local esSleepTime=${ELASTICSEARCH_SLEEP_TIME:-5} - run_process_sleep "elasticsearch" "$ELASTICSEARCH_DIR/bin/elasticsearch" $esSleepTime - fi -} - -function _get_kibana_version_name { - echo "kibana-${KIBANA_VERSION}-linux-x86_64" -} - -function install_kibana { - if is_service_enabled kibana; then - echo_summary "Installing Kibana ${KIBANA_VERSION}" - - local kibana_version_name - kibana_version_name=`_get_kibana_version_name` - local kibana_tarball=${kibana_version_name}.tar.gz - local kibana_tarball_url=http://download.elastic.co/kibana/kibana/${kibana_tarball} - - local kibana_tarball_dest - kibana_tarball_dest=`get_extra_file ${kibana_tarball_url}` - - tar xzf ${kibana_tarball_dest} -C $DEST - - sudo chown -R $STACK_USER $DEST/${kibana_version_name} - ln -sf $DEST/${kibana_version_name} $KIBANA_DIR - fi -} - -function configure_kibana { - if is_service_enabled kibana; then - echo_summary "Configuring Kibana ${KIBANA_VERSION}" - - sudo install -m 755 -d -o $STACK_USER $KIBANA_CFG_DIR - - sudo cp -f "${PLUGIN_FILES}"/kibana/kibana.yml $KIBANA_CFG_DIR/kibana.yml - sudo chown -R $STACK_USER $KIBANA_CFG_DIR/kibana.yml - sudo chmod 0644 $KIBANA_CFG_DIR/kibana.yml - - sudo sed -e " - s|%KIBANA_SERVICE_HOST%|$KIBANA_SERVICE_HOST|g; - s|%KIBANA_SERVICE_PORT%|$KIBANA_SERVICE_PORT|g; - s|%KIBANA_SERVER_BASE_PATH%|$KIBANA_SERVER_BASE_PATH|g; - s|%ES_SERVICE_BIND_HOST%|$ES_SERVICE_BIND_HOST|g; - s|%ES_SERVICE_BIND_PORT%|$ES_SERVICE_BIND_PORT|g; - s|%KEYSTONE_AUTH_URI%|$KEYSTONE_AUTH_URI|g; - " -i $KIBANA_CFG_DIR/kibana.yml - - ln -sf $KIBANA_CFG_DIR/kibana.yml $GATE_CONFIGURATION_DIR/kibana.yml - fi -} - -function install_kibana_plugin { - if is_service_enabled kibana; then - echo_summary "Install Kibana plugin" - - # note(trebskit) that needs to happen after kibana received - # its configuration otherwise the plugin fails to be installed - - local pkg=file://$DEST/monasca-kibana-plugin.tar.gz - - $KIBANA_DIR/bin/kibana plugin -r monasca-kibana-plugin - $KIBANA_DIR/bin/kibana plugin -i monasca-kibana-plugin -u $pkg - fi -} - -function clean_kibana { - if is_service_enabled kibana; then - echo_summary "Cleaning Kibana ${KIBANA_VERSION}" - - local kibana_tarball - kibana_tarball=`_get_kibana_version_name`.tar.gz - sudo rm -rf $KIBANA_DIR || true - sudo rm -rf $FILES/${kibana_tarball} || true - sudo rm -rf $KIBANA_CFG_DIR || true - fi -} - -function start_kibana { - if is_service_enabled kibana; then - echo_summary "Starting Kibana ${KIBANA_VERSION}" - local kibanaSleepTime=${KIBANA_SLEEP_TIME:-90} # kibana takes some time to load up - local kibanaCFG="$KIBANA_CFG_DIR/kibana.yml" - run_process_sleep "kibana" "$KIBANA_DIR/bin/kibana --config $kibanaCFG" $kibanaSleepTime - fi -} - -function configure_monasca_log_persister { - if is_service_enabled monasca-log-persister; then - echo_summary "Configuring monasca-log-persister" - - sudo install -m 755 -d -o $STACK_USER $LOG_PERSISTER_DIR - - sudo cp -f "${PLUGIN_FILES}"/monasca-log-persister/persister.conf $LOG_PERSISTER_DIR/persister.conf - sudo chown $STACK_USER $LOG_PERSISTER_DIR/persister.conf - sudo chmod 0640 $LOG_PERSISTER_DIR/persister.conf - - sudo sed -e " - s|%ES_SERVICE_BIND_HOST%|$ES_SERVICE_BIND_HOST|g; - " -i $LOG_PERSISTER_DIR/persister.conf - - ln -sf $LOG_PERSISTER_DIR/persister.conf $GATE_CONFIGURATION_DIR/log-persister.conf - fi -} - -function clean_monasca_log_persister { - if is_service_enabled monasca-log-persister; then - echo_summary "Cleaning monasca-log-persister" - sudo rm -rf $LOG_PERSISTER_DIR || true - fi -} - -function start_monasca_log_persister { - if is_service_enabled monasca-log-persister; then - echo_summary "Starting monasca-log-persister" - local logstash="$LOGSTASH_DIR/bin/logstash" - run_process "monasca-log-persister" "$logstash -f $LOG_PERSISTER_DIR/persister.conf" - fi -} - -function configure_monasca_log_transformer { - if is_service_enabled monasca-log-transformer; then - echo_summary "Configuring monasca-log-transformer" - - sudo install -m 755 -d -o $STACK_USER $LOG_TRANSFORMER_DIR - - sudo cp -f "${PLUGIN_FILES}"/monasca-log-transformer/transformer.conf $LOG_TRANSFORMER_DIR/transformer.conf - sudo chown $STACK_USER $LOG_TRANSFORMER_DIR/transformer.conf - sudo chmod 0640 $LOG_TRANSFORMER_DIR/transformer.conf - - sudo sed -e " - s|%KAFKA_SERVICE_HOST%|$KAFKA_SERVICE_HOST|g; - s|%KAFKA_SERVICE_PORT%|$KAFKA_SERVICE_PORT|g; - " -i $LOG_TRANSFORMER_DIR/transformer.conf - - ln -sf $LOG_TRANSFORMER_DIR/transformer.conf $GATE_CONFIGURATION_DIR/log-transformer.conf - fi -} - -function clean_monasca_log_transformer { - if is_service_enabled monasca-log-transformer; then - echo_summary "Cleaning monasca-log-transformer" - sudo rm -rf $LOG_TRANSFORMER_DIR || true - fi -} - -function start_monasca_log_transformer { - if is_service_enabled monasca-log-transformer; then - echo_summary "Starting monasca-log-transformer" - local logstash="$LOGSTASH_DIR/bin/logstash" - run_process "monasca-log-transformer" "$logstash -f $LOG_TRANSFORMER_DIR/transformer.conf" - fi -} - -function configure_monasca_log_metrics { - if is_service_enabled monasca-log-metrics; then - echo_summary "Configuring monasca-log-metrics" - - sudo install -m 755 -d -o $STACK_USER $LOG_METRICS_DIR - - sudo cp -f "${PLUGIN_FILES}"/monasca-log-metrics/log-metrics.conf $LOG_METRICS_DIR/log-metrics.conf - sudo chown $STACK_USER $LOG_METRICS_DIR/log-metrics.conf - sudo chmod 0640 $LOG_METRICS_DIR/log-metrics.conf - - sudo sed -e " - s|%KAFKA_SERVICE_HOST%|$KAFKA_SERVICE_HOST|g; - s|%KAFKA_SERVICE_PORT%|$KAFKA_SERVICE_PORT|g; - " -i $LOG_METRICS_DIR/log-metrics.conf - - ln -sf $LOG_METRICS_DIR/log-metrics.conf $GATE_CONFIGURATION_DIR/log-metrics.conf - fi -} - -function clean_monasca_log_metrics { - if is_service_enabled monasca-log-metrics; then - echo_summary "Cleaning monasca-log-metrics" - sudo rm -rf $LOG_METRICS_DIR || true - fi -} - -function start_monasca_log_metrics { - if is_service_enabled monasca-log-metrics; then - echo_summary "Starting monasca-log-metrics" - local logstash="$LOGSTASH_DIR/bin/logstash" - run_process "monasca-log-metrics" "$logstash -f $LOG_METRICS_DIR/log-metrics.conf" - fi -} - -function install_log_agent { - if is_service_enabled monasca-log-agent; then - echo_summary "Installing monasca-log-agent [monasca-output-plugin]" - - $LOGSTASH_DIR/bin/plugin install --version \ - "${LOGSTASH_OUTPUT_MONASCA_VERSION}" logstash-output-monasca_log_api - fi -} - -function configure_monasca_log_agent { - if is_service_enabled monasca-log-agent; then - echo_summary "Configuring monasca-log-agent" - - sudo install -m 755 -d -o $STACK_USER $LOG_AGENT_DIR - - sudo cp -f "${PLUGIN_FILES}"/monasca-log-agent/agent.conf $LOG_AGENT_DIR/agent.conf - sudo chown $STACK_USER $LOG_AGENT_DIR/agent.conf - sudo chmod 0640 $LOG_AGENT_DIR/agent.conf - - sudo sed -e " - s|%MONASCA_LOG_API_URI_V3%|$MONASCA_LOG_API_URI_V3|g; - s|%KEYSTONE_AUTH_URI_V3%|$KEYSTONE_AUTH_URI_V3|g; - " -i $LOG_AGENT_DIR/agent.conf - - ln -sf $LOG_AGENT_DIR/agent.conf $GATE_CONFIGURATION_DIR/log-agent.conf - - fi -} - -function clean_monasca_log_agent { - if is_service_enabled monasca-log-agent; then - echo_summary "Cleaning monasca-log-agent" - sudo rm -rf $LOG_AGENT_DIR || true - fi -} - -function start_monasca_log_agent { - if is_service_enabled monasca-log-agent; then - echo_summary "Starting monasca-log-agent" - local logstash="$LOGSTASH_DIR/bin/logstash" - run_process "monasca-log-agent" "$logstash -f $LOG_AGENT_DIR/agent.conf" "root" "root" - fi -} - -function install_nodejs { - if is_service_enabled kibana; then - # refresh installation - curl -sL https://deb.nodesource.com/setup_10.x | sudo bash - - apt_get install nodejs - ( - npm config set registry "http://registry.npmjs.org/"; \ - npm config set proxy "${HTTP_PROXY}"; \ - npm set strict-ssl false; - ) - fi -} - -function clean_nodejs { - if is_service_enabled kibana; then - echo_summary "Cleaning Node.js" - apt_get purge nodejs - fi -} - -function clean_gate_config_holder { - sudo rm -rf $GATE_CONFIGURATION_DIR || true -} - -function build_kibana_plugin { - if is_service_enabled kibana; then - echo "Building Kibana plugin" - - git_clone $MONASCA_KIBANA_PLUGIN_REPO \ - $MONASCA_KIBANA_PLUGIN_DIR \ - $MONASCA_KIBANA_PLUGIN_BRANCH - - pushd $MONASCA_KIBANA_PLUGIN_DIR - git_update_branch $MONASCA_KIBANA_PLUGIN_BRANCH - - local monasca_kibana_plugin_version - monasca_kibana_plugin_version="$(python -c 'import json; \ - obj = json.load(open("package.json")); print obj["version"]')" - - npm install - npm run package - - local pkg=$MONASCA_KIBANA_PLUGIN_DIR/target/monasca-kibana-plugin-${monasca_kibana_plugin_version}.tar.gz - local easyPkg=$DEST/monasca-kibana-plugin.tar.gz - - ln -sf $pkg $easyPkg - - popd - fi -} - -function configure_kafka { - echo_summary "Configuring Kafka topics" - /opt/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 \ - --replication-factor 1 --partitions 4 --topic log - /opt/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 \ - --replication-factor 1 --partitions 4 --topic transformed-log -} - -function delete_kafka_topics { - echo_summary "Deleting Kafka topics" - /opt/kafka/bin/kafka-topics.sh --delete --zookeeper localhost:2181 \ - --topic log || true - /opt/kafka/bin/kafka-topics.sh --delete --zookeeper localhost:2181 \ - --topic transformed-log || true -} - -function create_log_management_accounts { - if is_service_enabled monasca-log-api; then - echo_summary "Enable Log Management in Keystone" - - # note(trebskit) following points to Kibana which is bad, - # but we do not have search-api in monasca-log-api now - # this code will be removed in future - local log_search_url="http://$KIBANA_SERVICE_HOST:$KIBANA_SERVICE_PORT/" - - get_or_create_service "logs" "logs" "Monasca Log service" - get_or_create_endpoint \ - "logs" \ - "$REGION_NAME" \ - "$MONASCA_LOG_API_URI_V3" \ - "$MONASCA_LOG_API_URI_V3" \ - "$MONASCA_LOG_API_URI_V3" - - get_or_create_service "logs-search" "logs-search" "Monasca Log search service" - get_or_create_endpoint \ - "logs-search" \ - "$REGION_NAME" \ - "$log_search_url" \ - "$log_search_url" \ - "$log_search_url" - - fi -} - -function enable_log_management { - if is_service_enabled horizon && is_service_enabled kibana; then - echo_summary "Configure Horizon with Kibana access" - - local localSettings=${DEST}/horizon/monitoring/config/local_settings.py - - sudo sed -e " - s|KIBANA_HOST = getattr(settings, 'KIBANA_HOST', 'http://192.168.10.4:5601/')|KIBANA_HOST = getattr(settings, 'KIBANA_HOST', 'http://${KIBANA_SERVICE_HOST}:${KIBANA_SERVICE_PORT}/')|g; - " -i ${localSettings} - - if is_service_enabled monasca-log-api; then - sudo sed -e " - s|'ENABLE_LOG_MANAGEMENT_BUTTON', False|'ENABLE_LOG_MANAGEMENT_BUTTON', True|g; - " -i ${localSettings} - fi - - restart_apache_server - fi -} - -function configure_tempest_for_monasca { - iniset $TEMPEST_CONFIG monitoring kibana_version $KIBANA_VERSION -} - -# check for service enabled -if is_service_enabled monasca-log; then - - if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then - # Set up system services - echo_summary "Configuring Monasca Log Management system services" - pre_install - - elif [[ "$1" == "stack" && "$2" == "install" ]]; then - # Perform installation of service source - echo_summary "Installing Monasca Log Management" - install_monasca_log - - elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then - if is_service_enabled tempest; then - echo_summary "Configuring Tempest for Monasca" - configure_tempest_for_monasca - fi - - elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then - # Configure after the other layer 1 and 2 services have been configured - echo_summary "Configuring Monasca Log Management" - configure_monasca_log - - elif [[ "$1" == "stack" && "$2" == "extra" ]]; then - # Initialize and start the Monasca service - echo_summary "Initializing Monasca Log Management" - init_monasca_log - init_monasca_grafana_dashboards - if is_service_enabled monasca-agent; then - init_agent - fi - start_monasca_log - fi - - if [[ "$1" == "unstack" ]]; then - # Shut down Monasca services - echo_summary "Unstacking Monasca Log Management" - stop_monasca_log - delete_kafka_topics - fi - - if [[ "$1" == "clean" ]]; then - # Remove state and transient data - # Remember clean.sh first calls unstack.sh - echo_summary "Cleaning Monasca Log Management" - clean_monasca_log - fi -fi - -#Restore errexit -$_ERREXIT_LOG_API - -# Restore xtrace -$_XTRACE_LOG_API diff --git a/devstack/settings b/devstack/settings deleted file mode 100644 index adcfe789..00000000 --- a/devstack/settings +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# --- WARNING --- -# monasca-log-api DevStack plugin has been deprecated in Ussuri -# the code has been merged into monasca-api DevStack plugin -# the settings here apply only to stable/train and older branches -# please use settings in monasca-api for newer releases - -# services -enable_service zookeeper # devstack/monasca-api/zookeeper -enable_service kibana # devstack/monasca-api/kafka -enable_service elasticsearch -enable_service monasca-log -enable_service monasca-log-api -enable_service monasca-log-persister -enable_service monasca-log-transformer -enable_service monasca-log-metrics -enable_service monasca-log-agent - -# libraries/frameworks/projects monasca-log requires to install itself -KIBANA_VERSION=${KIBANA_VERSION:-4.6.3} -LOGSTASH_VERSION=${LOGSTASH_VERSION:-2.4.1} -ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION:-2.4.6} -LOGSTASH_OUTPUT_MONASCA_VERSION=${LOGSTASH_OUTPUT_MONASCA_VERSION:-1.0.4} -NODE_JS_VERSION=${NODE_JS_VERSION:-"4.0.0"} -NVM_VERSION=${NVM_VERSION:-"0.33.5"} - -# repository settings -MONASCA_LOG_API_REPO=${MONASCA_LOG_API_REPO:-${GIT_BASE}/openstack/monasca-log-api.git} -MONASCA_LOG_API_BRANCH=${MONASCA_LOG_API_BRANCH:-master} -MONASCA_LOG_API_DIR=${DEST}/monasca-log-api - -MONASCA_COMMON_REPO=${MONASCA_COMMON_REPO:-${GIT_BASE}/openstack/monasca-common.git} -MONASCA_COMMON_BRANCH=${MONASCA_COMMON_BRANCH:-master} -MONASCA_COMMON_DIR=${DEST}/monasca-common - -MONASCA_STATSD_REPO=${MONASCA_STATSD_REPO:-${GIT_BASE}/openstack/monasca-statsd.git} -MONASCA_STATSD_BRANCH=${MONASCA_STATSD_BRANCH:-master} -MONASCA_STATSD_DIR=${DEST}/monasca-statsd - -MONASCA_KIBANA_PLUGIN_REPO=${MONASCA_KIBANA_PLUGIN_REPO:-${GIT_BASE}/openstack/monasca-kibana-plugin.git} -MONASCA_KIBANA_PLUGIN_BRANCH=${MONASCA_KIBANA_PLUGIN_BRANCH:-master} -MONASCA_KIBANA_PLUGIN_DIR=${DEST}/monasca-kibana-plugin - -DOWNLOAD_FILE_TIMEOUT=${DOWNLOAD_FILE_TIMEOUT:-300} diff --git a/doc/api_samples/v2/req_json.json b/doc/api_samples/v2/req_json.json deleted file mode 100644 index d70e2db3..00000000 --- a/doc/api_samples/v2/req_json.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "message":"Hello World!" -} - diff --git a/doc/api_samples/v2/req_text.txt b/doc/api_samples/v2/req_text.txt deleted file mode 100644 index 557db03d..00000000 --- a/doc/api_samples/v2/req_text.txt +++ /dev/null @@ -1 +0,0 @@ -Hello World diff --git a/doc/api_samples/v3/req_global_dims.json b/doc/api_samples/v3/req_global_dims.json deleted file mode 100644 index b6735b68..00000000 --- a/doc/api_samples/v3/req_global_dims.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "dimensions":{ - "hostname":"mini-mon", - "service":"monitoring" - }, - "logs":[ - { - "message":"msg1", - "dimensions":{ - "component":"mysql", - "path":"/var/log/mysql.log" - } - }, - { - "message":"msg2", - "dimensions":{ - "component":"monasca-api", - "path":"/var/log/monasca/monasca-api.log" - } - } - ] -} diff --git a/doc/api_samples/v3/req_multiple_logs.json b/doc/api_samples/v3/req_multiple_logs.json deleted file mode 100644 index 4da99a16..00000000 --- a/doc/api_samples/v3/req_multiple_logs.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dimensions":{}, - "logs":[ - { - "message":"msg1", - "dimensions":{ - "component":"mysql", - "path":"/var/log/mysql.log" - } - }, - { - "message":"msg2", - "dimensions":{ - "component":"monasca-api", - "path":"/var/log/monasca/monasca-api.log" - } - }, - { - "message":"msg3", - "dimensions":{ - "component":"monasca-log-api", - "path":"/var/log/monasca/monasca-log-api.log" - } - } - ] -} diff --git a/doc/api_samples/v3/req_single_log.json b/doc/api_samples/v3/req_single_log.json deleted file mode 100644 index 201ca18a..00000000 --- a/doc/api_samples/v3/req_single_log.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "dimensions":{}, - "logs":[ - { - "message":"msg1", - "dimensions":{ - "component":"mysql", - "path":"/var/log/mysql.log" - } - } - ] -} diff --git a/doc/source/.gitignore b/doc/source/.gitignore deleted file mode 100644 index 53dafec3..00000000 --- a/doc/source/.gitignore +++ /dev/null @@ -1 +0,0 @@ -_static/*.sample diff --git a/doc/source/_static/.gitkeep b/doc/source/_static/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/doc/source/_static/images/architecture.png b/doc/source/_static/images/architecture.png deleted file mode 100644 index 7a8651fd..00000000 Binary files a/doc/source/_static/images/architecture.png and /dev/null differ diff --git a/doc/source/_static/images/architecture.svg b/doc/source/_static/images/architecture.svg deleted file mode 100644 index e185e632..00000000 --- a/doc/source/_static/images/architecture.svg +++ /dev/null @@ -1,988 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Log API - - - - - - Elasticsearch - - - - - - store logs - - - - - - Message Queue - - - - - - publish log messages - - - - - - consume logs - - - - - - filter logspublish metrics - - - - - - parse logs - - - - - - UIKibana 4.4 - - - - - - Keystoneauthenticationplugin - - - - - - Log Agents - - - - - - Monasca outputplugin - - - - - - query logs - - - - - - POST logs - - - - - - Log Persister - - - - - - Log Metrics - - - - - - Log Transformer - - - - - - Logstash / Beaver - - - - - - Kafka - - - - - - IN: KafkaOUT: Kafka - - - - - - query logs (planned) - - - - - - IN: KafkaOUT: Kafka - - - - - - IN: KafkaOUT: Elasticsearch - - - - - - Logstash - - - - - - Logstash - - - - - - Logstash - - - - - - - - \ No newline at end of file diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst deleted file mode 100644 index 36a9908c..00000000 --- a/doc/source/admin/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -====================== - Administration guide -====================== - -.. toctree:: - :maxdepth: 2 diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst deleted file mode 100644 index 21354418..00000000 --- a/doc/source/cli/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -======================== - Command Line Interface -======================== - -At the moment, monasca-log-api cannot be operated -from the CLI. diff --git a/doc/source/conf.py b/doc/source/conf.py deleted file mode 100644 index a593242b..00000000 --- a/doc/source/conf.py +++ /dev/null @@ -1,281 +0,0 @@ -# -*- coding: utf-8 -*- -# -# monasca-log-api documentation build configuration file, created by -# sphinx-quickstart on Wed Nov 18 12:02:03 2015. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import os -import sys - -sys.path = [ - os.path.abspath('../..'), - os.path.abspath('../../bin') -] + sys.path - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.6' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.coverage', - 'sphinx.ext.ifconfig', - 'sphinx.ext.graphviz', - 'sphinx.ext.autodoc', - 'sphinx.ext.viewcode', - 'oslo_config.sphinxconfiggen', - 'oslo_config.sphinxext', - 'openstackdocstheme', - 'oslo_policy.sphinxpolicygen' -] - -# geeneral information about project -openstackdocs_repo_name = u'openstack/monasca-log-api' -openstackdocs_auto_name = False -project = u'Monasca Log Dev Docs' -openstackdocs_bug_project = u'monasca-log-api' -openstackdocs_bug_tag = u'doc' -copyright = u'2014-present, OpenStack Foundation' -author = u'OpenStack Foundation' - -# sample config -config_generator_config_file = [ - ('config-generator/monasca-log-api.conf', '_static/monasca-log-api') - -] -policy_generator_config_file = [ - ('config-generator/policy.conf', '_static/log-api') - -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -source_encoding = 'utf-8' - -# The master toctree document. -master_doc = 'index' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [ - 'common', - 'doc', - 'documentation', - 'etc', - 'java' -] - -# If true, '()' will be appended to :func: etc. cross-reference text. -add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -show_authors = True - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'native' - -# A list of ignored prefixes for module index sorting. -modindex_common_prefix = ['monasca_log_api.', 'monasca'] - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'openstackdocs' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# doc. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# html_extra_path = [] - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -html_use_index = True - -# If false, no module index is generated. -html_use_modindex = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'monasca-log-apidoc' - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'monasca-log-api.tex', u'monasca-log-api Documentation', - u'Openstack Foundation \\textless{}monasca@lists.launchpad.net\\textgreater{}', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'monasca-log-api', u'monasca-log-api Documentation', - [author], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'monasca-log-api', u'monasca-log-api Documentation', - author, 'monasca-log-api', 'Rest-API to collect logs from your cloud.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://doc.python.org/': None} diff --git a/doc/source/configuration/configuring.rst b/doc/source/configuration/configuring.rst deleted file mode 100644 index 4ef47417..00000000 --- a/doc/source/configuration/configuring.rst +++ /dev/null @@ -1,160 +0,0 @@ -.. _basic-configuration: - ------------ -Configuring ------------ - -monasca-log-api has several configuration options. Some of them -are inherited from oslo libraries, others can be found in the monasca-log-api -codebase. - -The entire configuration of monasca-log-api is defined in -configuration files. - -.. note:: This is intended behaviour. One of possible ways to deploy - monasca-log-api is to use **gunicorn**. Unfortunately gunicorn's - argument parsing clashes with oslo's argument parsing. - This means that gunicorn reports the CLI options of - oslo as unknown, and vice versa. - -There are 4 configuration files. For more details on the configuration -options, see :ref:`here `. - -Configuring Keystone Authorization ----------------------------------- - -Keystone authorization (i.e. verification of the token associated -with a request) is a critical part of monasca-log-api. -It prevents from unauthorized access and provides the isolation -needed for multi-tenancy. - -The configuration for ``keystonemiddleware`` should either be provided in -``monasca-log-api.conf`` or in a file in one of the configuration directories. -For more details about configuration options, check -`here `_. - -Configuring Log Publishing --------------------------- - -monasca-log-api sends all logs to the Kafka Message Queue. -Proper configuration should include: - -* ``kafka_url`` - comma-delimited list of Kafka brokers -* ``topics`` - names of the topics to which the logs will be pushed to -* ``max_message_size`` - maximum message size that can be posted a topic - -The configuration for ``log_publisher`` should either be provided in -``monasca-log-api.conf`` or in a file in one of the configuration directories. - -Configuring Healthcheck ------------------------ - -Healthcheck is an essential part of monasca-log-api. -It allows sending HTTP requests and getting knowledge about the -availability of the API. Configuration of healthcheck includes: - -* ``kafka_url`` - comma-delimited list of Kafka brokers -* ``kafka_topics`` - list of topics that existence is verified by healthcheck - -The configuration for ``kafka_healthcheck`` should either be provided in -``monasca-log-api.conf`` or in a file in one of the configuration directories. - -Configuring Monitoring ----------------------- - -monasca-log-api is capable of self-monitoring. This is achieved -through `monasca-statsd `_. -It assumes that there is monasca-agent available on the system node and -that statsd-server has been launched. - -There are several options you may want to tweak if necessary: - -* ``statsd_host``- the host statsd-server is bound to -* ``statsd_port``- the port statsd-server is bound to -* ``statsd_buffer`` - the amount of metrics to buffer in memory before sending - any -* ``dimensions`` - additional dimensions to be sent with all - metrics for this monasca-log-api instance - -The configuration for ``monitoring`` should either be provided in -``monasca-log-api.conf`` or in a file in one of the configuration directories. - -Configuring RBAC ----------------- - -The role-based access policy can be defined in the ``log-api.policy.yaml`` file -as described in `oslo.policy documentation -`_. - -Additionally, for historical reasons, custom RBAC mechanism is provided. It can -be configured as follows: - -* ``path`` - list of URIs that RBAC applies to -* ``default_roles`` - list of roles that are permitted to access the API -* ``agent_roles`` - list of roles, that if present, means that requests come - from log-agent -* ``delegate_roles`` - list of roles required by log-agent for sending logs - on behalf of another project (tenant) - -The configuration for ``roles_middleware`` can be provided either in -``monasca-log-api.conf`` or in a file in one of the configuration directories. - -Configuring Logging -------------------- - -Logging in monasca-log-api is controlled from the single -``log-api-logging.conf`` configuration file. -Here is a short list of several modifications you may want to apply, -based on your deployment: - -* to log INFO to console:: - - [handler_console] - level = INFO - -* to log DEBUG to file:: - - [handler_file] - level = DEBUG - -* to change the log file location:: - - [handler_file] - args = ('/var/log/log-api.log', 'a') - -* if you have an external script for log rotation:: - - [handler_file] - handler = logging.handlers.WatchedFileHandler - args = ('/var/log/log-api.log', 'a') - - That will store up to 5 rotations (each having maximum size - of 100MBs) - - The configuration of ``logging`` should be presented inside - ``log-api-logging.conf`` file and referenced from ``monasca-log-api.conf`` - using ``log_config_append`` option. - - If you want to know more about possible ways to save monasca-log-api logs, - feel free to visit: - - * `oslo.log `_ - * `Python HowTo `_ - * `Logging handlers `_ - -Configuring Policies --------------------- - -The policies for accessing each service can be configured in the -``log-api.policy.yaml`` configuration file:: - - Policy Description - Method Path - "Policy string": "Roles" - -example:: - - Logs post rule - POST /logs - POST /log/single - "log_api:logs:post": "role:monasca-user" diff --git a/doc/source/configuration/files.rst b/doc/source/configuration/files.rst deleted file mode 100644 index 1ca33d5b..00000000 --- a/doc/source/configuration/files.rst +++ /dev/null @@ -1,99 +0,0 @@ -.. _configuration-files: - -------------------- -Configuration files -------------------- - -Overview of monasca-log-api's configuration files. - -monasca-log-api.conf --------------------- - -This is the main configuration file of monasca-log-api. -It can be located in several places. During startup, -monasca-log-api searches for it in the following directories: - -* ``~/.monasca`` -* ``~/`` -* ``/etc/monasca`` -* ``/etc`` - -Alternatively, you can roll with a multi-file-based configuration model. -In this case, monasca-log-api searches the configuration files -in the following directories: - -* ``~/.monasca/monasca.conf.d/`` -* ``~/.monasca/monasca-log-api.conf.d/`` -* ``~/monasca.conf.d/`` -* ``~/monasca-log-api.conf.d/`` -* ``/etc/monasca/monasca.conf.d/`` -* ``/etc/monasca/monasca-log-api.conf.d/`` -* ``/etc/monasca.conf.d/`` -* ``/etc/monasca-log-api.conf.d/`` - -Regardless of the location, the name of the main configuration file -should always be ``monasca-log-api.conf``. For files located -in ``.conf.d`` directories, the name is irrelevant, but it should -indicate the file content. - -For example, when guring keystone communication. The -`keystonemiddleware `_ -configuration would be, therefore, located in, for example, -``/etc/monasca-log-api.conf.d/keystonemiddleware.conf`` - -A sample of this configuration file is also available -:ref:`here ` - -log-api-logging.conf --------------------- - -This file contains the logging setup for monasca-log-api. It should be -referenced from ``monasca-log-api.conf`` using, for example, -the following code snippet:: - - [DEFAULT] - log_config_append = /etc/monasca/log-api-logging.conf - -A sample of this configuration file is also available -:ref:`here ` - -log-api-paste.ini ------------------ - -This file contains the `PasteDeploy `_ -configuration. It describes all pipelines that are running within a single -instance of monasca-log-api. - -There is nothing you should try and modify in this file, -apart from enabling/disabling ``oslo_middleware.debug:Debug``. - -To enable ``oslo_middleware.debug:Debug`` for ``Log v3`` pipeline, -``log-api-paste.ini`` should contain code similar to this one:: - - [composite:main] - use = egg:Paste#urlmap - /v3.0: la_api_v3 - - [pipeline:la_api_v3] - pipeline = debug {{ other pipeline members }} - - [filter:debug] - paste.filter_factory = oslo_middleware.debug:Debug.factory - -This particular filter might be useful for examining the -WSGI environment during troubleshooting or local development. - -log-api.policy.yaml -------------------- - -This is the configuration file for policies to access the services. -the path of the file can be defined in ``monasca-log-api.conf``:: - - [oslo_policy] - policy_file = log-api.policy.yaml - -More information about policy file configuration can be found at -`oslo.policy `_ - -A sample of this configuration file is also available -:ref:`here ` diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst deleted file mode 100644 index 180f2512..00000000 --- a/doc/source/configuration/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. _configuring: - -============= -Configuration -============= - -This section describes the configuration settings that can be specified. -Refer to :ref:`basic-configuration` for more details on the -available settings. - -.. toctree:: - :maxdepth: 1 - - configuring - files - options - sample diff --git a/doc/source/configuration/options.rst b/doc/source/configuration/options.rst deleted file mode 100644 index f7617956..00000000 --- a/doc/source/configuration/options.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _monasca-log-api.conf: - -------- -Options -------- - -.. show-options:: - :config-file: config-generator/monasca-log-api.conf diff --git a/doc/source/configuration/sample.rst b/doc/source/configuration/sample.rst deleted file mode 100644 index 4ee26c1a..00000000 --- a/doc/source/configuration/sample.rst +++ /dev/null @@ -1,51 +0,0 @@ -.. _sample-configuration: - -------- -Samples -------- - -The following sections show sample configuration files for monasca-log-api and -related utilities. These are generated from the code -(apart from the samples for logging and paster) and reflect the current state -of code in the monasca-log-api repository. - - -.. _sample-configuration-api: - -Sample Configuration For Application ------------------------------------- - -This sample configuration can also be viewed in `monasca-log-api.conf.sample -<../_static/monasca-log-api.conf.sample>`_. - -.. literalinclude:: ../_static/monasca-log-api.conf.sample - -.. _sample-configuration-logging: - -Sample Configuration For Logging --------------------------------- - -This sample configuration can also be viewed in `log-api-logging.conf -`_. - -.. literalinclude:: ../../../etc/monasca/log-api-logging.conf - - -Sample Configuration For Paste ------------------------------- - -This sample configuration can also be viewed in `log-api-paste.ini -`_. - -.. literalinclude:: ../../../etc/monasca/log-api-paste.ini - -.. _sample-configuration-policy: - -Sample Configuration For Policy -------------------------------- - -This sample configuration can also be viewed in `log-api-policy.yaml.sample -<../_static/log-api-policy.yaml.sample>`_. - -.. literalinclude:: ../_static/log-api.policy.yaml.sample - diff --git a/doc/source/contributor/.gitignore b/doc/source/contributor/.gitignore deleted file mode 100644 index d7f336d8..00000000 --- a/doc/source/contributor/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# codebase documentation is autogenerated -# so we do not want to track it in the tree -api/ diff --git a/doc/source/contributor/code.rst b/doc/source/contributor/code.rst deleted file mode 100644 index f8fa03f6..00000000 --- a/doc/source/contributor/code.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _codedocs: - -====================== -Codebase documentation -====================== - -Following section contains codebase documenation generated with, a little -bit of assistance, `sphinx.ext.autodoc`_. - -.. _`sphinx.ext.autodoc`: http://www.sphinx-doc.org/en/stable/ext/autodoc.html - -Modules -======= - -.. toctree:: - :maxdepth: 2 - - api/autoindex.rst diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst deleted file mode 100644 index 74449d29..00000000 --- a/doc/source/contributor/index.rst +++ /dev/null @@ -1,30 +0,0 @@ -======================= -Contribution Guidelines -======================= - -In the Contributions Guide, you will find documented policies for -developing with monasca-log. This includes the processes we use for -blueprints and specs, bugs, contributor onboarding, core reviewer -memberships, and other procedural items. - -monasca-log, as with all OpenStack projects, is written with the following -design guidelines in mind: - -* **Component based architecture**: Quickly add new behaviors -* **Highly available**: Scale to very serious workloads -* **Fault tolerant**: Isolated processes avoid cascading failures -* **Recoverable**: Failures should be easy to diagnose, debug, and rectify -* **Open standards**: Be a reference implementation for a community-driven api - -This documentation is generated by the Sphinx toolkit and lives in the source -tree. Additional documentation on monasca-log and other components of -OpenStack can be found on the `OpenStack wiki `_. - -Developer reference -------------------- - -.. toctree:: - :maxdepth: 1 - - tox - code diff --git a/doc/source/contributor/tox.rst b/doc/source/contributor/tox.rst deleted file mode 100644 index 3a47c7ab..00000000 --- a/doc/source/contributor/tox.rst +++ /dev/null @@ -1,77 +0,0 @@ -.. - monasca-log-api documentation master file - Copyright 2017 FUJITSU LIMITED - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -.. _`tox`: https://tox.readthedocs.io/en/latest/ -.. _`mandatory_tox_env`: https://github.com/openstack/monasca-log-api/blob/master/tox.ini#L2 - -=== -Tox -=== - -**monasca-log-api** uses `tox`_ to wrap up all the activities around -testing and linting the codebase. - -List of environments -==================== - -There is a rather large number of tox environments that **monasca-log-api** -is using. If necessary they can be enlisted with:: - - tox -a -v - -An output will be similar to this:: - - default environments: - py27 -> Runs unit test using Python2.7 - py35 -> Runs unit test using Python3.5 - pep8 -> Runs set of linters against codebase (flake8, bandit, - bashate, checkniceness) - cover -> Calculates code coverage - - additional environments: - api-guide -> Called from CI scripts to test and publish the API Guide - api-ref -> Called from CI scripts to test and publish the API Ref - apidocs -> Generates codebase documentation - bandit -> [no description] - bashate -> Validates (pep8-like) devstack plugins - checkjson -> Validates all json samples inside doc folder - checkniceness -> Validates (pep-like) documenation - debug -> Allows to run unit-test with debug mode enabled - docs -> Builds api-ref, api-guide, releasenotes and doc - flake8 -> [no description] - releasenotes -> Called from CI script to test and publish the Release Notes - venv -> [no description] - - -Running tox -=========== - -Running tox is as simple as:: - - tox - -That will run all **mandatory** (for details refer to `mandatory_tox_env`_) -environments. Having them passed is *a must have*. - -Running specific environments -============================= - -If you require to run specific environments, please use:: - - tox -e api-ref,api-guide,releasenotes - -Result of which will be having all documentations sub-components generated -and ready in local dev environment. diff --git a/doc/source/deprecation_note.inc b/doc/source/deprecation_note.inc deleted file mode 100644 index 3169cb92..00000000 --- a/doc/source/deprecation_note.inc +++ /dev/null @@ -1,6 +0,0 @@ -.. note:: The Log API v2 has been DEPRECATED in the Mitaka release. The - migration path is to use the `Log API v3 - `_ instead of version 2 - of the API. The Log API v2 will ultimately be removed, following the - `OpenStack standard deprecation policy - `_. diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst deleted file mode 100644 index fa106d19..00000000 --- a/doc/source/glossary.rst +++ /dev/null @@ -1,3 +0,0 @@ -======== -Glossary -======== diff --git a/doc/source/index.rst b/doc/source/index.rst deleted file mode 100644 index 9f060eaf..00000000 --- a/doc/source/index.rst +++ /dev/null @@ -1,50 +0,0 @@ -.. - monasca-log-api documentation master file - Copyright 2016-2017 FUJITSU LIMITED - - Licensed under the Apache License, Version 2.0 (the "License"); you may - not use this file except in compliance with the License. You may obtain - a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - License for the specific language governing permissions and limitations - under the License. - -=========================================== -Welcome to monasca-log-api's documentation! -=========================================== - -monasca-log-api is a RESTful API server acting as gateway for -logs collected from log-agents. - -.. include:: deprecation_note.inc - -The developer documentation provided here is continually kept up-to-date -based on the latest code, and may not represent the state of the project at -any specific prior release. - -.. note:: This is documentation for developers, if you are looking for more - general documentation including API, install, operator and user - guides see `docs.openstack.org`_ - -.. _`docs.openstack.org`: https://docs.openstack.org - -.. toctree:: - :maxdepth: 2 - - user/index - admin/index - install/index - configuration/index - cli/index - contributor/index - -.. toctree:: - :maxdepth: 1 - - glossary - diff --git a/doc/source/install/index.rst b/doc/source/install/index.rst deleted file mode 100644 index e6bc440e..00000000 --- a/doc/source/install/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -============== - Installation -============== - -.. toctree:: - :maxdepth: 2 diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst deleted file mode 100644 index 475c57c9..00000000 --- a/doc/source/user/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -============ - User guide -============ - -.. toctree:: - :maxdepth: 2 diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index efe1f5ba..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -ARG DOCKER_IMAGE=monasca/log-api -ARG APP_REPO=https://review.opendev.org/openstack/monasca-log-api - -# Branch, tag or git hash to build from. -ARG REPO_VERSION=master -ARG CONSTRAINTS_BRANCH=master - -# Extra Python3 dependencies. -ARG EXTRA_DEPS="gunicorn python-memcached gevent" - -# Always start from `monasca-base` image and use specific tag of it. -ARG BASE_TAG=master -FROM monasca/base:$BASE_TAG - -# Environment variables used for our service or wait scripts. -ENV \ - KAFKA_URI=kafka:9092 \ - KAFKA_WAIT_FOR_TOPICS=log \ - MONASCA_CONTAINER_LOG_API_PORT=5607 \ - MEMCACHED_URI=memcached:11211 \ - AUTHORIZED_ROLES=admin,domainuser,domainadmin,monasca-user \ - AGENT_AUTHORIZED_ROLES=monasca-agent \ - KEYSTONE_IDENTITY_URI=http://keystone:35357 \ - KEYSTONE_AUTH_URI=http://keystone:5000 \ - KEYSTONE_ADMIN_USER=admin \ - KEYSTONE_ADMIN_PASSWORD=secretadmin \ - KEYSTONE_ADMIN_TENANT=admin \ - KEYSTONE_ADMIN_DOMAIN=default \ - GUNICORN_WORKERS=9 \ - GUNICORN_WORKER_CLASS=gevent \ - GUNICORN_WORKER_CONNECTIONS=2000 \ - GUNICORN_BACKLOG=1000 \ - GUNICORN_TIMEOUT=10 \ - PYTHONIOENCODING=utf-8 \ - ADD_ACCESS_LOG=false \ - ACCESS_LOG_FORMAT="%(asctime)s [%(process)d] gunicorn.access [%(levelname)s] %(message)s" \ - ACCESS_LOG_FIELDS='%(h)s %(l)s %(u)s %(t)s %(r)s %(s)s %(b)s "%(f)s" "%(a)s" %(L)s' \ - LOG_LEVEL_ROOT=INFO \ - LOG_LEVEL_CONSOLE=INFO \ - LOG_LEVEL_ACCESS=INFO \ - STAY_ALIVE_ON_FAILURE="false" - -# Copy all neccessary files to proper locations. -COPY log-api* monasca-log-api* /etc/monasca/ - -# Run here all additionals steps your service need post installation. -# Stay with only one `RUN` and use `&& \` for next steps to don't create -# unnecessary image layers. Clean at the end to conserve space. -#RUN \ -# echo "Some steps to do after main installation." && \ -# echo "Hello when building." - -# Expose port for specific service. -EXPOSE ${MONASCA_CONTAINER_LOG_API_PORT} - -# Implement start script in `start.sh` file. -CMD ["/start.sh"] diff --git a/docker/README.rst b/docker/README.rst deleted file mode 100644 index d00a73fc..00000000 --- a/docker/README.rst +++ /dev/null @@ -1,85 +0,0 @@ -================================ -Docker image for Monasca Log API -================================ -The Monasca log API image is based on the monasca-base image. - - -Building monasca-base image -=========================== -See https://github.com/openstack/monasca-common/tree/master/docker/README.rst - - -Building Monasca log API image -============================== - -Example: - $ ./build_image.sh - - -Requirements from monasca-base image -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -health_check.py - This file will be used for checking the status of the Monasca Log API - application. - - -Scripts -~~~~~~~ -start.sh - In this starting script provide all steps that lead to the proper service - start. Including usage of wait scripts and templating of configuration - files. You also could provide the ability to allow running container after - service died for easier debugging. - -build_image.sh - Please read detailed build description inside the script. - - -Environment variables -~~~~~~~~~~~~~~~~~~~~~ -============================== ======================================================================= ========================================== -Variable Default Description -============================== ======================================================================= ========================================== -KAFKA_URI kafka:9092 URI to Apache Kafka (distributed streaming platform) -KAFKA_WAIT_FOR_TOPICS log The topic where log-api streams the log messages -KAFKA_WAIT_RETRIES 24 Number of kafka connect attempts -KAFKA_WAIT_DELAY 5 Seconds to wait between attempts -MONASCA_CONTAINER_LOG_API_PORT 5607 The port from the log pipeline endpoint -MEMCACHED_URI memcached:11211 URI to Keystone authentication cache -AUTHORIZED_ROLES admin,domainuser,domainadmin,monasca-user Roles for Monasca users (full API access) -AGENT_AUTHORIZED_ROLES monasca-agent Roles for Monasca agents (sending data only) -KEYSTONE_IDENTITY_URI http://keystone:35357 URI to Keystone admin endpoint -KEYSTONE_AUTH_URI http://keystone:5000 URI to Keystone public endpoint -KEYSTONE_ADMIN_USER admin OpenStack administrator user name -KEYSTONE_ADMIN_PASSWORD secretadmin OpenStack administrator user password -KEYSTONE_ADMIN_TENANT admin OpenStack administrator tenant name -KEYSTONE_ADMIN_DOMAIN default OpenStack administrator domain -GUNICORN_WORKERS 9 Number of gunicorn (WSGI-HTTP server) workers -GUNICORN_WORKER_CLASS gevent Used gunicorn worker class -GUNICORN_WORKER_CONNECTIONS 2000 Number of gunicorn worker connections -GUNICORN_BACKLOG 1000 Number of gunicorn backlogs -GUNICORN_TIMEOUT 10 Gunicorn connection timeout -PYTHONIOENCODING utf-8 Python encoding -ADD_ACCESS_LOG false Enable gunicorn request/access logging -ACCESS_LOG_FORMAT "%(asctime)s [%(process)d] gunicorn.access [%(levelname)s] %(message)s" Define the logging format -ACCESS_LOG_FIELDS '%(h)s %(l)s %(u)s %(t)s %(r)s %(s)s %(b)s "%(f)s" "%(a)s" %(L)s' Define the fields to be logged -LOG_LEVEL_ROOT WARN Log level for root logging -LOG_LEVEL_CONSOLE INFO Log level for console logging -LOG_LEVEL_ACCESS INFO Log level for access logging -STAY_ALIVE_ON_FAILURE false If true, container runs 2 hours after tests fail -============================== ======================================================================= ========================================== - - -Provide configuration templates -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -* monasca-log-api.conf.j2 -* log-api-gunicorn.conf.j2 -* log-api-logging.conf.j2 -* log-api.paste.ini.j2 - - -Links -~~~~~ -https://docs.openstack.org/monasca-log-api/latest/configuration/ - -https://github.com/openstack/monasca-log-api/blob/master/README.rst diff --git a/docker/build_image.sh b/docker/build_image.sh deleted file mode 100755 index 9e15bc02..00000000 --- a/docker/build_image.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# TODO(Dobroslaw): move this script to monasca-common/docker folder -# and leave here small script to download it and execute using env variables -# to minimize code duplication. - -set -x # Print each script step. -set -eo pipefail # Exit the script if any statement returns error. - -# This script is used for building Docker image with proper labels -# and proper version of monasca-common. -# -# Example usage: -# $ ./build_image.sh -# -# Everything after `./build_image.sh` is optional and by default configured -# to get versions from `Dockerfile`. -# -# To build from master branch (default): -# $ ./build_image.sh -# To build specific version run this script in the following way: -# $ ./build_image.sh stable/queens -# Building from specific commit: -# $ ./build_image.sh cb7f226 -# When building from a tag monasca-common will be used in version available -# in upper constraint file: -# $ ./build_image.sh 2.5.0 -# To build image from Gerrit patch sets that is targeting branch stable/queens: -# $ ./build_image.sh refs/changes/51/558751/1 stable/queens -# -# If you want to build image with custom monasca-common version you need -# to provide it as in the following example: -# $ ./build_image.sh master master refs/changes/19/595719/3 - -# Go to folder with Docker files. -REAL_PATH=$(python -c "import os,sys; print(os.path.realpath('$0'))") -cd "$(dirname "$REAL_PATH")/../docker/" - -[ -z "$DOCKER_IMAGE" ] && \ - DOCKER_IMAGE=$(\grep DOCKER_IMAGE Dockerfile | cut -f2 -d"=") - -: "${REPO_VERSION:=$1}" -[ -z "$REPO_VERSION" ] && \ - REPO_VERSION=$(\grep REPO_VERSION Dockerfile | cut -f2 -d"=") -# Let's stick to more readable version and disable SC2001 here. -# shellcheck disable=SC2001 -REPO_VERSION_CLEAN=$(echo "$REPO_VERSION" | sed 's|/|-|g') - -[ -z "$APP_REPO" ] && APP_REPO=$(\grep APP_REPO Dockerfile | cut -f2 -d"=") -GITHUB_REPO=$(echo "$APP_REPO" | sed 's/review.opendev.org/github.com/' | \ - sed 's/ssh:/https:/') - -if [ -z "$CONSTRAINTS_FILE" ]; then - CONSTRAINTS_FILE=$(\grep CONSTRAINTS_FILE Dockerfile | cut -f2 -d"=") || true - : "${CONSTRAINTS_FILE:=https://releases.openstack.org/constraints/upper/master}" -fi - -: "${CONSTRAINTS_BRANCH:=$2}" -[ -z "$CONSTRAINTS_BRANCH" ] && \ - CONSTRAINTS_BRANCH=$(\grep CONSTRAINTS_BRANCH Dockerfile | cut -f2 -d"=") - -# When using stable version of repository use same stable constraints file. -case "$REPO_VERSION" in - *stable*) - CONSTRAINTS_BRANCH_CLEAN="$REPO_VERSION" - CONSTRAINTS_FILE=${CONSTRAINTS_FILE/master/$CONSTRAINTS_BRANCH_CLEAN} - # Get monasca-common version from stable upper constraints file. - CONSTRAINTS_TMP_FILE=$(mktemp) - wget --output-document "$CONSTRAINTS_TMP_FILE" \ - $CONSTRAINTS_FILE - UPPER_COMMON=$(\grep 'monasca-common' "$CONSTRAINTS_TMP_FILE") - # Get only version part from monasca-common. - UPPER_COMMON_VERSION="${UPPER_COMMON##*===}" - rm -rf "$CONSTRAINTS_TMP_FILE" - ;; - *) - CONSTRAINTS_BRANCH_CLEAN="$CONSTRAINTS_BRANCH" - ;; -esac - -# Monasca-common variables. -if [ -z "$COMMON_REPO" ]; then - COMMON_REPO=$(\grep COMMON_REPO Dockerfile | cut -f2 -d"=") || true - : "${COMMON_REPO:=https://review.opendev.org/openstack/monasca-common}" -fi -: "${COMMON_VERSION:=$3}" -if [ -z "$COMMON_VERSION" ]; then - COMMON_VERSION=$(\grep COMMON_VERSION Dockerfile | cut -f2 -d"=") || true - if [ "$UPPER_COMMON_VERSION" ]; then - # Common from upper constraints file. - COMMON_VERSION="$UPPER_COMMON_VERSION" - fi -fi - -# Clone project to temporary directory for getting proper commit number from -# branches and tags. We need this for setting proper image labels. -# Docker does not allow to get any data from inside of system when building -# image. -TMP_DIR=$(mktemp -d) -( - cd "$TMP_DIR" - # This many steps are needed to support gerrit patch sets. - git init - git remote add origin "$APP_REPO" - git fetch origin "$REPO_VERSION" - git reset --hard FETCH_HEAD -) -GIT_COMMIT=$(git -C "$TMP_DIR" rev-parse HEAD) -[ -z "${GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1 -rm -rf "$TMP_DIR" - -# Do the same for monasca-common. -COMMON_TMP_DIR=$(mktemp -d) -( - cd "$COMMON_TMP_DIR" - # This many steps are needed to support gerrit patch sets. - git init - git remote add origin "$COMMON_REPO" - git fetch origin "$COMMON_VERSION" - git reset --hard FETCH_HEAD -) -COMMON_GIT_COMMIT=$(git -C "$COMMON_TMP_DIR" rev-parse HEAD) -[ -z "${COMMON_GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1 -rm -rf "$COMMON_TMP_DIR" - -CREATION_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - -docker build --no-cache \ - --build-arg CREATION_TIME="$CREATION_TIME" \ - --build-arg GITHUB_REPO="$GITHUB_REPO" \ - --build-arg APP_REPO="$APP_REPO" \ - --build-arg REPO_VERSION="$REPO_VERSION" \ - --build-arg GIT_COMMIT="$GIT_COMMIT" \ - --build-arg CONSTRAINTS_FILE="$CONSTRAINTS_FILE" \ - --build-arg COMMON_REPO="$COMMON_REPO" \ - --build-arg COMMON_VERSION="$COMMON_VERSION" \ - --build-arg COMMON_GIT_COMMIT="$COMMON_GIT_COMMIT" \ - --tag "$DOCKER_IMAGE":"$REPO_VERSION_CLEAN" . diff --git a/docker/health_check.py b/docker/health_check.py deleted file mode 100755 index d190cd3e..00000000 --- a/docker/health_check.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 - -# (C) Copyright 2018 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Health check will returns 0 when service is working properly.""" - -import logging -import os -import sys -from urllib import request - - -LOG_LEVEL = logging.getLevelName(os.environ.get('LOG_LEVEL', 'INFO')) -logging.basicConfig(level=LOG_LEVEL) -logger = logging.getLogger(__name__) - -API_PORT = os.environ.get('MONASCA_CONTAINER_LOG_API_PORT', '5607') -url = "http://localhost:" + API_PORT + "/healthcheck" - - -def main(): - """Send health check request to health check endpoint of log API.""" - logger.debug('Send health check request to %s', url) - try: - request.urlopen(url=url) - except Exception as ex: - logger.error('Exception during request handling: ' + repr(ex)) - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/docker/log-api-gunicorn.conf.j2 b/docker/log-api-gunicorn.conf.j2 deleted file mode 100644 index d81e2526..00000000 --- a/docker/log-api-gunicorn.conf.j2 +++ /dev/null @@ -1,13 +0,0 @@ -bind = '0.0.0.0:{{ MONASCA_CONTAINER_LOG_API_PORT }}' -proc_name = 'monasca-log-api' - -backlog = {{ GUNICORN_BACKLOG | int }} -workers = {{ GUNICORN_WORKERS | int }} -worker_class = '{{ GUNICORN_WORKER_CLASS }}' -timeout = {{ GUNICORN_TIMEOUT | int }} - -{% if ADD_ACCESS_LOG == true %} -accesslog = '-' -access_log_format = '{{ ACCESS_LOG_FIELDS }}' -{% endif %} -capture_output = True diff --git a/docker/log-api-logging.conf.j2 b/docker/log-api-logging.conf.j2 deleted file mode 100644 index 76d60f16..00000000 --- a/docker/log-api-logging.conf.j2 +++ /dev/null @@ -1,47 +0,0 @@ -[default] -disable_existing_loggers = 0 - -[loggers] -keys = root, gunicorn_access, kafka - -[handlers] -keys = console, gunicorn_access - -[formatters] -keys = context, gunicorn_access - -[logger_root] -level = {{ LOG_LEVEL_ROOT }} -handlers = console - -[logger_gunicorn_access] -level = {{ LOG_LEVEL_ACCESS }} -handlers = console -propagate = 0 -qualname = gunicorn.access - -[logger_kafka] -qualname = kafka -level = DEBUG -handlers = console -propagate = 0 - -[handler_console] -class = logging.StreamHandler -args = (sys.stdout,) -level = {{ LOG_LEVEL_CONSOLE }} -formatter = context - -[handler_gunicorn_access] -class = logging.StreamHandler -args = (sys.stdout,) -level = {{ LOG_LEVEL_ACCESS }} -formatter = gunicorn_access - -[formatter_context] -class = oslo_log.formatters.ContextFormatter - -[formatter_gunicorn_access] -class = logging.Formatter -format = {{ ACCESS_LOG_FORMAT }} -datefmt = %Y-%m-%d %H:%M:%S diff --git a/docker/log-api-paste.ini.j2 b/docker/log-api-paste.ini.j2 deleted file mode 100644 index 05ac128a..00000000 --- a/docker/log-api-paste.ini.j2 +++ /dev/null @@ -1,53 +0,0 @@ -[DEFAULT] -name = monasca_log_api - -[composite:main] -use = egg:Paste#urlmap -/: la_version -/healthcheck: la_healthcheck -/v2.0: la_api_v2 -/v3.0: la_api_v3 - -[pipeline:la_version] -pipeline = error_trap versionapp - -[pipeline:la_healthcheck] -pipeline = error_trap healthcheckapp - -[pipeline:la_api_v2] -pipeline = error_trap request_id auth roles api_v2_app - -[pipeline:la_api_v3] -pipeline = error_trap request_id auth roles api_v3_app - -[app:versionapp] -paste.app_factory = monasca_log_api.app.api:create_version_app - -[app:healthcheckapp] -paste.app_factory = monasca_log_api.app.api:create_healthcheck_app - -[app:api_v2_app] -paste.app_factory = monasca_log_api.app.api:create_api_app -set api_version=v2.0 - -[app:api_v3_app] -paste.app_factory = monasca_log_api.app.api:create_api_app -set api_version=v3.0 - -[filter:auth] -paste.filter_factory = keystonemiddleware.auth_token:filter_factory - -[filter:roles] -paste.filter_factory = monasca_log_api.middleware.role_middleware:RoleMiddleware.factory - -[filter:request_id] -paste.filter_factory = oslo_middleware.request_id:RequestId.factory - -[filter:debug] -paste.filter_factory = oslo_middleware.debug:Debug.factory - -[filter:error_trap] -paste.filter_factory = oslo_middleware.catch_errors:CatchErrors.factory - -[server:main] -use = egg:gunicorn#main diff --git a/docker/monasca-log-api.conf.j2 b/docker/monasca-log-api.conf.j2 deleted file mode 100644 index 5ae94610..00000000 --- a/docker/monasca-log-api.conf.j2 +++ /dev/null @@ -1,42 +0,0 @@ -[DEFAULT] -log_config_append=/etc/monasca/log-api-logging.conf - -[monitoring] -enable = {{ MONITORING_ENABLE | default(False) }} -statsd_host = {{ STATSD_HOST | default('127.0.0.1') }} -statsd_port = {{ STATSD_PORT | default(8125) }} -statsd_buffer = {{ STATSD_BUFFER | default(50) }} - -[service] -region = useast -max_log_size = 1048576 - -[roles_middleware] -path = /v2.0/log,/v3.0/logs -default_roles = {{ AUTHORIZED_ROLES | default('admin, domainuser, domainadmin, monasca-user') }} -agent_roles = {{ AGENT_AUTHORIZED_ROLES | default('monasca-agent') }} - -[log_publisher] -topics = log -kafka_url = {{ KAFKA_URI | default('kafka:9092') }} -max_message_size = 1048576 - -[kafka_healthcheck] -kafka_url = {{ KAFKA_URI | default('kafka:9092') }} -kafka_topics = log - -[keystone_authtoken] -auth_type = password -auth_url = {{ KEYSTONE_IDENTITY_URI }} -auth_uri = {{ KEYSTONE_AUTH_URI }} -username = {{ KEYSTONE_ADMIN_USER }} -password = {{ KEYSTONE_ADMIN_PASSWORD }} -user_domain_name = Default -project_name = {{ KEYSTONE_ADMIN_TENANT }} -project_domain_name = Default -service_token_roles_required = true -memcached_servers = {{ MEMCACHED_URI }} -insecure = false -cafile = -certfile = -keyfile = diff --git a/docker/start.sh b/docker/start.sh deleted file mode 100644 index 26d92d81..00000000 --- a/docker/start.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Starting script. -# All checks and configuration templating you need to do before service -# could be safely started should be added in this file. - -set -eo pipefail # Exit the script if any statement returns error. - -# Test services we need before starting our service. -echo "Start script: waiting for needed services" -python3 /kafka_wait_for_topics.py - -# Template all config files before start, it will use env variables. -# Read usage examples: https://pypi.org/project/Templer/ -echo "Start script: creating config files from templates" -templer -v -f /etc/monasca/monasca-log-api.conf.j2 /etc/monasca/monasca-log-api.conf -templer -v -f /etc/monasca/log-api-gunicorn.conf.j2 /etc/monasca/log-api-gunicorn.conf -templer -v -f /etc/monasca/log-api-logging.conf.j2 /etc/monasca/log-api-logging.conf -templer -v -f /etc/monasca/log-api-paste.ini.j2 /etc/monasca/log-api-paste.ini - -# Start our service. -# gunicorn --args -echo "Start script: starting container" -gunicorn \ - --config /etc/monasca/log-api-gunicorn.conf \ - --paste /etc/monasca/log-api-paste.ini - -# Allow server to stay alive in case of failure for 2 hours for debugging. -RESULT=$? -if [ $RESULT != 0 ] && [ "$STAY_ALIVE_ON_FAILURE" = "true" ]; then - echo "Service died, waiting 120 min before exiting" - sleep 7200 -fi -exit $RESULT diff --git a/documentation/monasca-log-api-kafka.md b/documentation/monasca-log-api-kafka.md deleted file mode 100644 index 615bfc7b..00000000 --- a/documentation/monasca-log-api-kafka.md +++ /dev/null @@ -1,142 +0,0 @@ -# Monasca Log API - Kafka - -Date: April 18, 2016 - -Document Version: v0.2 - -## Introduction - -**monasca-log-api** uses kafka transport to ship received logs down to the -processing pipeline. - -For more information about Kafka, please see [official documentation] -(http://kafka.apache.org/documentation.html). - -## Output message format -Messages sent to kafka should have following format -(top level object is called **envelope**) and is combined out of three -elements: -* log -* creation_time -* meta - - - "log": , - "creation_time": , - "meta": - -Log property should have at least following form: - - "message": , - "dimensions": - -Meta property should have following form: - - "tenantId": , - "region": - -Full example as json: -```json - { - "log": { - "message": "2015-11-13 12:44:42.411 27297 DEBUG kafka [-] Read 31/31 bytes from Kafka _read_bytes /opt/monasca/monasca-log-api/lib/python2.7/site-packages/kafka/conn.py:103", - "dimensions": { - "hostname": "devstack" - } - }, - "creation_time": 1447834886, - "meta": { - "tenantId": "e4bd29509eda473092d32aadfee3e7b1", - "region": "pl" - } - } -``` - -### Fields explanation - -* log - contains log specific information collected from the system. In the -most lean case that would be: **message**, **dimensions** - * message - normally that represent a single line from a log file - * dimensions - informations such as hostname where application is running -* creation_time - UNIX timestamp representing moment when log message was created -by monasca-log-api -* meta - contains tenantId and its region - -**log** entry may of course contain many more fields that are considered valid -in given case. However two mentioned in this documentation are required. - -All fields, apart from **creation_time** and **log**, are created from HTTP headers. -Description is available [here](/documentation/monasca-log-api-spec.md). - -## Truncating too large message - - Following section mostly applies to monasca-log-api v3.0 - -Each *envelope* sent to Kafka is serialized into JSON string. This string must -comply to Kafka limitation about [maximum message size](https://kafka.apache.org/08/configuration.html). -If JSON message is too big following actions are taken -1) difference between maximum allowed size and JSON message size (both in bytes). - ```diff = (size(json_envelope) + size(envelope_key) + KAFKA_METADATA_SIZE) - maximum_allowed_size + TRUNCATION_SAFE_OFFSET```. - **KAFKA_METADATA_SIZE** is amount of bytes Kafka adds during transformation - of each message prior to sending it -2) log is enriched with property **truncated** set to **true** (```log['truncated'] = True```) -3) log's message is truncated by ```diff + TRUNCATED_PROPERTY_SIZE```. - **TRUNCATED_PROPERTY_SIZE** is the size of newly added property. - -Variables explanation: - -* **envelope_key** is the key used when routing logs into specific kafka partitions. -Its byte size is always fixed (determined from the byte size of timestamp represented as string). -```len(bytearray(str(int(time.time() * 1000)).encode('utf-8')))``` -* **KAFKA_METADATA_SIZE** equals to 200 bytes. -* **TRUNCATION_SAFE_OFFSET** is equal to 1 ensuring that diff size will be always positive number -* **TRUNCATED_PROPERTY_SIZE** is calculated as byte size of expression ```log['truncated'] = True``` -for each run of log-api. - -## Configuration - -### Java - -Configuration for kafka should be placed in *.yml file and look similar to: -```yml -logTopic: logs -kafka: - brokerUris: - - localhost:8900 - zookeeperUris: - - localhost:2181 - healthCheckTopic: healthcheck -``` - -It is composed out of two relevant pieces -* logTopic - topic where data should be sent -* kafka - section containing information required to communicate in kafka. -For more details see [here](https://github.com/openstack/monasca-common/blob/master/java/monasca-common-kafka/src/main/java/monasca/common/messaging/kafka/KafkaConfiguration.java) - -### Python - -Configuration for kafka should be placed in *.conf file and look similar to: - -```conf -[log_publisher] -topics = 'logs' -kafka_url = 'localhost:8900' -``` - -There are only two relevant options: -* topics - comma delimited list of topics where data should be sent -* kafka_url - address where kafka server is running - - # Copyright 2016-2017 FUJITSU LIMITED - # - # Licensed under the Apache License, Version 2.0 (the "License"); you may - # not use this file except in compliance with the License. You may obtain - # a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - # License for the specific language governing permissions and limitations - # under the License. diff --git a/documentation/monasca-log-api-metrics.md b/documentation/monasca-log-api-metrics.md deleted file mode 100644 index fbe44bf9..00000000 --- a/documentation/monasca-log-api-metrics.md +++ /dev/null @@ -1,108 +0,0 @@ -# Monitoring for monasca-log-api - -**monasca-log-api** can be monitored by examining following metrics - -| Name | Meaning | Dimensions | -|-------------------------------------------|-------------------------|---------------| -| monasca.log.in_logs | Amount of received logs | version | -| monasca.log.in_logs_rejected | Amount of rejected logs (see below for details) | version | -| monasca.log.in_bulks_rejected | Amount of rejected bulks (see below for details) | version | -| monasca.log.in_logs_bytes | Size received logs (a.k.a. *Content-Length)* in bytes | version | -| monasca.log.out_logs | Amount of logs published to kafka | | -| monasca.log.out_logs_lost | Amount of logs lost during publish phase | | -| monasca.log.out_logs_truncated_bytes | Amount of truncated bytes, removed from message | | -| monasca.log.publish_time_ms | Time Log-Api needed to publish all logs to kafka | | -| monasca.log.processing_time_ms | Time Log-Api needed to process received logs. | version | - -Additionally each metric contains following dimensions: -- **component** - monasca-log-api -- **service** - monitoring - -## Metrics explained - -### monasca.log.in_logs - -Metric sent with amount of logs that were received by **Log-API** -and successfully passed initial validation. For **v2.0** this -metric will be always send with value one, for **v3.0** this metric's values -are equivalent to amount of element in bulk request. - -### monasca.log.in_logs_rejected - -Logs can be rejected because of: - -* checking content-type -* checking content-length -* reading payload -* retrieving logs from payload -* validating global dimensions (if set)(only valid for v3.0) - -### monasca.log.in_bulks_rejected (only v3.0) - -In **v2.0** bulk request is equivalent to single request (i.e. single-element bulk). -However in **v3.0** rejecting logs is done in two phases. - -*Phase 1* is when there is no way to determine actual amount of logs -that were sent by client (see [monasca.log.in_logs_rejected](#monasca_log_logs_rejected)) -If any of these steps was impossible to be executed entire bulk is -considered lost and thus all logs within. - -If *Phase 1* passes, *Phase 2* is executed. At this point every -piece of data is available, however still some logs can be rejected, -because of: - -* lack of certain fields (i.e. message) -* invalid local dimensions (if set) - -In *Phase 2* metric [monasca.log.in_logs_rejected](#monasca_log_logs_rejected) -is produced. - -### monasca.log.in_logs_bytes - -Metric allows to track to size of requests API receives. -In **v3.0** To simplify implementation it is equivalent to **Content-Length** value. -However amount of global dimensions and other metadata when compared -to size of logs is negligible. - -### monasca.log.out_logs - -Amount of logs successfully published to kafka queue. - -### monasca.log.out_logs_lost - -Amount of logs that were not sent to kafka and **Log-API** was unable -to recover from error situation - -### monasca.log.out_logs_truncated_bytes - -Metric is sent with the amount of bytes that log's message is shorten -by if necessary. To read more about truncation see [here](/documentation/monasca-log-api-kafka.md). -If truncation did not happen, which should be normal situation for most -of the time, metric is updated with value **0**. - -### monasca.log.publish_time_ms - -Time that was needed to send all the logs into all the topics. -*monasca.log.processing_time_ms* includes value of that metric -within. It exists to see how much does publishing take in entire -processing. - -### monasca.log.processing_time_ms - -Total amount of time logs spent inside **Log-API**. Metric does not -include time needed to communicate with Keystone to authenticate request. -As far as possible it is meant to track **Log-API** itself - - # Copyright 2016-2017 FUJITSU LIMITED - # - # Licensed under the Apache License, Version 2.0 (the "License"); you may - # not use this file except in compliance with the License. You may obtain - # a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - # License for the specific language governing permissions and limitations - # under the License. diff --git a/documentation/monasca-log-api-spec.md b/documentation/monasca-log-api-spec.md deleted file mode 100644 index 165675cc..00000000 --- a/documentation/monasca-log-api-spec.md +++ /dev/null @@ -1,319 +0,0 @@ -# Monasca Log API - -Date: May 27, 2016 - -Document Version: v2.2.2 - -# Logs -The logs resource allows logs to be created and queried. - -## Create Logs -Create logs. - -### POST /v3.0/logs - -#### Headers -* X-Auth-Token (string, required) - Keystone auth token -* Content-Type (string, required) - application/json - -#### Path Parameters -None. - -#### Query Parameters -* tenant_id (string, optional, restricted) - Tenant ID (project ID) to create - log on behalf of. Usage of this query parameter requires the role specified - in the configuration option `delegate_roles` . - -#### Request Body -JSON object which can have a maximum size of 5 MB. It consists of global -dimensions (optional) and array of logs. Each single log message with -resulting envelope can have a maximum size of 1 MB. -Dimensions is a dictionary of key-value pairs and should be consistent with -metric dimensions. - -Logs is an array of JSON objects describing the log entries. Every log object -can have individual set of dimensions which has higher precedence than global -ones. It should be noted that dimensions presented in each log record are also -optional. - - If both global (at the root level) and local (at log entry level) - dimensions would be present, they will be merged into one dictionary. - Please note that local dimensions are logically considered as more - specific thus in case of conflict (i.e. having two entries with the same - key in both global and local dimensions) local dimensions take - precedence over global dimensions. - -#### Request Examples - -POST logs - -``` -POST /v3.0/logs HTTP/1.1 -Host: 192.168.10.4:5607 -Content-Type: application/json -X-Auth-Token: 27feed73a0ce4138934e30d619b415b0 -Cache-Control: no-cache - -{ - "dimensions":{ - "hostname":"mini-mon", - "service":"monitoring" - }, - "logs":[ - { - "message":"msg1", - "dimensions":{ - "component":"mysql", - "path":"/var/log/mysql.log" - } - }, - { - "message":"msg2", - "dimensions":{ - "component":"monasca-api", - "path":"/var/log/monasca/monasca-api.log" - } - } - ] -} -``` - -### Response -#### Status Code -* 204 - No content - -#### Response Body -This request does not return a response body. - -## List logs -Get precise log listing filtered by dimensions. - - Note that this API is in development, and is not currently implemented. - -This interface can used to obtain log entries for a time range, based on -matching a set of exact dimension values. By default, entries will be returned -in descending timestamp order (newest first). The log entries returned by this -API will not necessarily be identical to those POST-ed to the service, as the -data returned will have been subjected to deployment-specific transformation -stages (i.e. the "monasca-log-transform" service). - -### GET /v3.0/logs - -#### Headers -* X-Auth-Token (string, required) - Keystone auth token -* Accept (string) - application/json - -#### Path Parameters -None. - -#### Query Parameters -* tenant_id (string, optional, restricted) - Tenant ID from which to get logs -from. This parameter can be used to get logs from a tenant other than the tenant -the request auth token is scoped to. Usage of this query parameter is restricted -to users with the monasca admin role, as defined in the monasca-log-api -configuration file, which defaults to `monasca-admin`. -* dimensions (string, optional) - A dictionary to filter logs by specified as a -comma separated array of (key, value) pairs as `key1:value1,key2:value2, ...`, -multiple values for a key may be specified as -`key1:value1|value2|...,key2:value4,...`. If the value is omitted in the form -`key1,key2, ...`, then entries are returned where the dimension exists with -any value. -* start_time (string, optional) - The start time in ISO 8601 combined date and -time format in UTC. -* end_time (string, optional) - The end time in ISO 8601 combined date and time -format in UTC. -* offset (integer, optional) - Number of log entries to skip (Default: 0). -* limit (integer, optional) - Limit number of logs returned (Default: 10). -* sort_by (string, optional) - Comma separated list of fields or dimensions to -sort by. Fields may be followed by 'asc' or 'desc' to set the direction, e.g. -'timestamp asc'. Allowed fields for sort_by are currently: 'timestamp'. -(Default: no sorting) - -#### Request Body -None. - -#### Request Examples -``` -GET /v3.0/logs?dimensions=hostname:devstack&start_time=2015-03-00T00:00:01Z HTTP/1.1 -Host: 192.168.10.4:5607 -Content-Type: application/json -X-Auth-Token: 2b8882ba2ec44295bf300aecb2caa4f7 -Cache-Control: no-cache -``` - -### Response -#### Status Code -* 200 - OK - -#### Response Body -Returns a JSON object with a 'links' array of links and an 'elements' array of -log entry objects with the following fields: - -* timestamp (timestamp) - The originating time in ISO 8601 combined date and -time format in UTC, with millisecond resolution if available. -* message (string) - The contents of the log message. -* dimensions ({string(255): string(255)}) - Dimensions of the log, either -supplied with the log or added by transformation. - -#### Response Examples -``` -{ - "links": [ - { - "rel": "prev", - "href": "http://192.168.10.4:5607/v3.0/logs?start_time=2015-03-00T00%3A00%3A00Z&dimensions=hostname%3Adevstack" - }, - { - "rel": "self", - "href": "http://192.168.10.4:5607/v3.0/logs?offset=10&start_time=2015-03-00T00%3A00%3A00Z&dimensions=hostname%3Adevstack" - }, - { - "rel": "next", - "href": "http://192.168.10.4:5607/v3.0/logs?offset=20&start_time=2015-03-00T00%3A00%3A00Z&dimensions=hostname%3Adevstack" - } - ], - "elements": [ - { - "timestamp":"2015-03-03T05:24:55.202Z", - "message":"msg1", - "dimensions":{ - "hostname":"devstack", - "component":"mysql", - "path":"/var/log/mysql.log" - } - }, - { - "timestamp":"2015-03-01T02:22:09.112Z", - "message":"msg2", - "dimensions":{ - "hostname":"devstack", - "component":"monasca-api", - "path":"/var/log/monasca/monasca-api.log" - } - } - ] -} -``` - - -# Healthcheck - - Note that following part is updated for Python implementation. - -The *Monasca Log API* comes with a built-in health check mechanism. -It is available in two flavors, both accessible under ```/healthcheck``` -endpoint. - -## Complex check -The complex check not only returns a response with success code if *Monasca Log API* -is up and running but it also verifies if peripheral components, such as **Kafka**, -are healthy too. - -*Monasca Log API* will respond with following codes: - -* 200 - both API and external components are healthy. -* 503 - API is running but problems with peripheral components have been spotted. - -Example: -```curl -XGET 192.168.10.4:5607/healthcheck``` - -### Peripheral checks - -* **Kafka** is considered healthy if connection to broker can be established -and configured topics can be found. - -## Simple check -The simple check only returns response only if *Monasca Log API* is up and running. -It does not return any data because it is accessible only for ```HEAD``` requests. -If the *Monasca Log API* is running the following response code: ```204``` is expected. - -Example: -```curl -XHEAD 192.168.10.4:5607/healthcheck``` - - -======= -### POST /v2.0/log/single (deprecated) - -#### Headers -* X-Auth-Token (string, required) - Keystone auth token -* Content-Type (string, required) - application/json; text/plain -* X-Application-Type (string(255), optional) - Type of application -* X-Dimensions ({string(255):string(255)}, required) - A dictionary consisting of (key, value) pairs used to structure logs. - -#### Path Parameters -None. - -#### Request Body -Consists of a single plain text message or a JSON object which can have a maximum length of 1048576 characters. - -#### Request Examples - -##### Plain text log - single line -POST a single line of plain text log. - -``` -POST /v2.0/log/single HTTP/1.1 -Host: 192.168.10.4:5607 -Content-Type: text/plain -X-Auth-Token: 27feed73a0ce4138934e30d619b415b0 -X-Application-Type: apache -X-Dimensions: applicationname:WebServer01,environment:production -Cache-Control: no-cache - -Hello World -``` - -##### Plain text log - multi lines -POST a multiple lines of plain text log. - -``` -POST /v2.0/log/single HTTP/1.1 -Host: 192.168.10.4:5607 -Content-Type: text/plain -X-Auth-Token: 27feed73a0ce4138934e30d619b415b0 -X-Application-Type: apache -X-Dimensions: applicationname:WebServer01,environment:production -Cache-Control: no-cache - -Hello\nWorld -``` - -##### JSON log -POST a JSON log - -``` -POST /v2.0/log/single HTTP/1.1 -Host: 192.168.10.4:5607 -Content-Type: application/json -X-Auth-Token: 27feed73a0ce4138934e30d619b415b0 -X-Application-Type: apache -X-Dimensions: applicationname:WebServer01,environment:production -Cache-Control: no-cache - -{ - "message":"Hello World!", - "from":"hoover" -} - -``` - -### Response -#### Status Code -* 204 - No content - -#### Response Body -This request does not return a response body. - - # Copyright 2016-2017 FUJITSU LIMITED - # - # Licensed under the Apache License, Version 2.0 (the "License"); you may - # not use this file except in compliance with the License. You may obtain - # a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - # License for the specific language governing permissions and limitations - # under the License. diff --git a/etc/monasca/log-api-logging.conf b/etc/monasca/log-api-logging.conf deleted file mode 100644 index 74d02ea1..00000000 --- a/etc/monasca/log-api-logging.conf +++ /dev/null @@ -1,34 +0,0 @@ -[loggers] -keys = root, kafka - -[handlers] -keys = console, file - -[formatters] -keys = context - -[logger_root] -level = INFO -handlers = console, file - -[logger_kafka] -qualname = kafka -level = INFO -handlers = console, file -propagate = 0 - -[handler_console] -class = logging.StreamHandler -args = (sys.stderr,) -level = DEBUG -formatter = context - -[handler_file] -class = logging.handlers.RotatingFileHandler -level = DEBUG -formatter = context -# store up to 5*100MB of logs -args = ('/var/log/monasca/log-api.log', 'a', 104857600, 5) - -[formatter_context] -class = oslo_log.formatters.ContextFormatter diff --git a/etc/monasca/log-api-paste.ini b/etc/monasca/log-api-paste.ini deleted file mode 100644 index 0732a87c..00000000 --- a/etc/monasca/log-api-paste.ini +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -[DEFAULT] -name = main - -[composite:main] -use = egg:Paste#urlmap -/: la_version -/healthcheck: la_healthcheck -/v2.0: la_api_v2 -/v3.0: la_api_v3 - -[pipeline:la_version] -pipeline = error_trap versionapp - -[pipeline:la_healthcheck] -pipeline = error_trap healthcheckapp - -[pipeline:la_api_v2] -pipeline = error_trap request_id auth api_v2_app - -[pipeline:la_api_v3] -pipeline = error_trap request_id auth api_v3_app - -[app:versionapp] -paste.app_factory = monasca_log_api.app.api:create_version_app - -[app:healthcheckapp] -paste.app_factory = monasca_log_api.app.api:create_healthcheck_app - -[app:api_v2_app] -paste.app_factory = monasca_log_api.app.api:create_api_app -set api_version=v2.0 - -[app:api_v3_app] -paste.app_factory = monasca_log_api.app.api:create_api_app -set api_version=v3.0 - -[filter:auth] -paste.filter_factory = keystonemiddleware.auth_token:filter_factory - -[filter:request_id] -paste.filter_factory = oslo_middleware.request_id:RequestId.factory - -# NOTE(trebskit) this is optional -# insert this into either pipeline to get some WSGI environment debug output -[filter:debug] -paste.filter_factory = oslo_middleware.debug:Debug.factory - -[filter:error_trap] -paste.filter_factory = oslo_middleware.catch_errors:CatchErrors.factory - -[server:main] -use = egg:gunicorn#main -bind = 127.0.0.1:5607 -workers = 9 -worker-connections = 2000 -worker-class = eventlet -timeout = 30 -backlog = 2048 -keepalive = 2 -proc_name = monasca-log-api -loglevel = DEBUG diff --git a/etc/monasca/log-api-uwsgi.ini b/etc/monasca/log-api-uwsgi.ini deleted file mode 100644 index 49d56191..00000000 --- a/etc/monasca/log-api-uwsgi.ini +++ /dev/null @@ -1,25 +0,0 @@ -[uwsgi] -wsgi-file = /usr/local/bin/monasca-log-api-wsgi - -# Versions of mod_proxy_uwsgi>=2.0.6 should use a UNIX socket, see -# http://uwsgi-docs.readthedocs.org/en/latest/Apache.html#mod-proxy-uwsgi -uwsgi-socket = 127.0.0.1:5607 - -# Override the default size for headers from the 4k default. -buffer-size = 65535 - -# This is running standalone -master = true - -enable-threads = true - -# Tune this to your environment. -processes = 4 - -# uwsgi recommends this to prevent thundering herd on accept. -thunder-lock = true - -plugins = python - -# This ensures that file descriptors aren't shared between keystone processes. -lazy-apps = true diff --git a/lower-constraints.txt b/lower-constraints.txt deleted file mode 100644 index 160607d7..00000000 --- a/lower-constraints.txt +++ /dev/null @@ -1,87 +0,0 @@ -alabaster==0.7.10 -appdirs==1.3.0 -Babel==2.3.4 -bandit==1.1.0 -bashate==0.5.1 -chardet==3.0.4 -configparser==3.5.0 -coverage==4.0 -debtcollector==1.2.0 -doc8==0.6.0 -docutils==0.11 -dulwich==0.15.0 -eventlet==0.18.2 -extras==1.0.0 -falcon==2.0.0 -fixtures==3.0.0 -future==0.16.0 -gevent==1.2.2 -gitdb==0.6.4 -GitPython==1.0.1 -greenlet==0.4.10 -imagesize==0.7.1 -iso8601==0.1.11 -Jinja2==2.10 -kazoo==2.2 -keystoneauth1==3.4.0 -keystonemiddleware==4.17.0 -linecache2==1.0.0 -MarkupSafe==1.0 -mccabe==0.2.1 -monasca-common==2.7.0 -monasca-statsd==1.1.0 -monotonic==0.6 -mox3==0.20.0 -msgpack-python==0.4.0 -netaddr==0.7.18 -netifaces==0.10.4 -openstackdocstheme==2.2.1 -os-api-ref==1.5.0 -os-client-config==1.28.0 -os-testr==1.0.0 -oslo.config==5.2.0 -oslo.context==2.19.2 -oslo.i18n==3.15.3 -oslo.log==3.36.0 -oslo.middleware==3.31.0 -oslo.policy==1.30.0 -oslo.serialization==2.18.0 -oslo.utils==3.33.0 -oslotest==3.2.0 -Paste==2.0.2 -PasteDeploy==1.5.0 -pbr==2.0.0 -positional==1.2.1 -pycadf==1.1.0 -pyflakes==0.8.1 -Pygments==2.2.0 -pyinotify==0.9.6 -PyMySQL==0.7.6 -pyparsing==2.1.0 -python-dateutil==2.5.3 -python-keystoneclient==3.8.0 -python-mimeparse==1.6.0 -python-statsd==2.1.0 -python-subunit==1.0.0 -pytz==2013.6 -PyYAML==3.12 -reno==3.1.0 -requests==2.14.2 -requestsexceptions==1.2.0 -restructuredtext-lint==1.1.1 -rfc3986==0.3.1 -simplejson==3.8.1 -six==1.10.0 -smmap==0.9.0 -snowballstemmer==1.2.1 -Sphinx==2.0.0 -sphinxcontrib-websupport==1.0.1 -statsd==3.2.1 -stestr==1.0.0 -stevedore==1.20.0 -tabulate==0.8.1 -testtools==2.2.0 -traceback2==1.4.0 -unittest2==1.1.0 -WebOb==1.7.1 -wrapt==1.7.0 diff --git a/monasca_log_api/__init__.py b/monasca_log_api/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/app/__init__.py b/monasca_log_api/app/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/app/api.py b/monasca_log_api/app/api.py deleted file mode 100644 index 6505d0bd..00000000 --- a/monasca_log_api/app/api.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Module contains factories to initializes various applications -of monasca-log-api -""" - -import six - -import falcon -from oslo_log import log - -from monasca_log_api.app.base import error_handlers -from monasca_log_api.app.base import request -from monasca_log_api.app.controller import healthchecks -from monasca_log_api.app.controller.v2 import logs as v2_logs -from monasca_log_api.app.controller.v3 import logs as v3_logs -from monasca_log_api.app.controller import versions -from monasca_log_api import config - - -def error_trap(app_name): - """Decorator trapping any error during application boot time""" - - @six.wraps(error_trap) - def _wrapper(func): - - @six.wraps(_wrapper) - def _inner_wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception: - logger = log.getLogger(__name__) - logger.exception('Failed to load application \'%s\'', app_name) - raise - - return _inner_wrapper - - return _wrapper - - -def singleton_config(func): - """Decorator ensuring that configuration is loaded only once.""" - - @six.wraps(singleton_config) - def _wrapper(global_config, **local_conf): - config.parse_args() - return func(global_config, **local_conf) - - return _wrapper - - -@error_trap('version') -def create_version_app(global_conf, **local_conf): - """Creates Version application""" - - ctrl = versions.Versions() - controllers = { - '/': ctrl, # redirect http://host:port/ down to Version app - # avoid conflicts with actual pipelines and 404 error - '/version': ctrl, # list all the versions - '/version/{version_id}': ctrl # display details of the version - } - - wsgi_app = falcon.API( - request_type=request.Request - ) - wsgi_app.req_options.strip_url_path_trailing_slash = True - for route, ctrl in controllers.items(): - wsgi_app.add_route(route, ctrl) - return wsgi_app - - -@error_trap('healthcheck') -def create_healthcheck_app(global_conf, **local_conf): - """Creates Healthcheck application""" - - ctrl = healthchecks.HealthChecks() - controllers = { - '/': ctrl - } - - wsgi_app = falcon.API( - request_type=request.Request - ) - for route, ctrl in controllers.items(): - wsgi_app.add_route(route, ctrl) - return wsgi_app - - -@error_trap('api') -@singleton_config -def create_api_app(global_conf, **local_conf): - """Creates MainAPI application""" - - controllers = {} - api_version = global_conf.get('api_version') - - if api_version == 'v2.0': - controllers.update({ - '/log/single': v2_logs.Logs() - }) - elif api_version == 'v3.0': - controllers.update({ - '/logs': v3_logs.Logs() - }) - - wsgi_app = falcon.API( - request_type=request.Request - ) - - for route, ctrl in controllers.items(): - wsgi_app.add_route(route, ctrl) - - error_handlers.register_error_handlers(wsgi_app) - - return wsgi_app diff --git a/monasca_log_api/app/base/__init__.py b/monasca_log_api/app/base/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/app/base/error_handlers.py b/monasca_log_api/app/base/error_handlers.py deleted file mode 100644 index db986511..00000000 --- a/monasca_log_api/app/base/error_handlers.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon - -from monasca_log_api.app.base import model - - -def log_envelope_exception_handler(ex, req, resp, params): - raise falcon.HTTPUnprocessableEntity( - title='Failed to create Envelope', - description=ex.message) - - -def register_error_handlers(app): - app.add_error_handler(model.LogEnvelopeException, - log_envelope_exception_handler) diff --git a/monasca_log_api/app/base/exceptions.py b/monasca_log_api/app/base/exceptions.py deleted file mode 100644 index 2886c0b6..00000000 --- a/monasca_log_api/app/base/exceptions.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2015 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon - - -HTTP_422 = '422 Unprocessable Entity' - - -class HTTPUnprocessableEntity(falcon.OptionalRepresentation, falcon.HTTPError): - """HTTPUnprocessableEntity http error. - - HTTPError that comes with '422 Unprocessable Entity' status - - :argument: message(str) - meaningful description of what caused an error - :argument: kwargs - any other option defined in - :py:class:`falcon.OptionalRepresentation` and - :py:class:`falcon.HTTPError` - """ - def __init__(self, message, **kwargs): - falcon.HTTPError.__init__(self, - HTTP_422, - 'unprocessable_entity', - message, - **kwargs - ) diff --git a/monasca_log_api/app/base/log_publisher.py b/monasca_log_api/app/base/log_publisher.py deleted file mode 100644 index 1394b1df..00000000 --- a/monasca_log_api/app/base/log_publisher.py +++ /dev/null @@ -1,250 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import time - -import falcon -from monasca_common.kafka import producer -from monasca_log_api.common.rest import utils as rest_utils -from oslo_log import log -from oslo_utils import encodeutils - -from monasca_log_api.app.base import model -from monasca_log_api import conf -from monasca_log_api.monitoring import client -from monasca_log_api.monitoring import metrics - -LOG = log.getLogger(__name__) -CONF = conf.CONF - -_RETRY_AFTER = 60 -_TIMESTAMP_KEY_SIZE = len( - bytearray(str(int(time.time() * 1000)).encode('utf-8'))) -_TRUNCATED_PROPERTY_SIZE = len( - bytearray('"truncated": true'.encode('utf-8'))) -_KAFKA_META_DATA_SIZE = 32 -_TRUNCATION_SAFE_OFFSET = 1 - - -class InvalidMessageException(Exception): - pass - - -class LogPublisher(object): - """Publishes log data to Kafka - - LogPublisher is able to send single message to multiple configured topic. - It uses following configuration written in conf file :: - - [log_publisher] - topics = 'logs' - kafka_url = 'localhost:8900' - - Note: - Uses :py:class:`monasca_common.kafka.producer.KafkaProducer` - to ship logs to kafka. For more details - see `monasca_common`_ github repository. - - .. _monasca_common: https://github.com/openstack/monasca-common - - """ - - def __init__(self): - self._topics = CONF.log_publisher.topics - self.max_message_size = CONF.log_publisher.max_message_size - - self._kafka_publisher = producer.KafkaProducer( - url=CONF.log_publisher.kafka_url - ) - if CONF.monitoring.enable: - self._statsd = client.get_client() - - # setup counter, gauges etc - self._logs_published_counter = self._statsd.get_counter( - metrics.LOGS_PUBLISHED_METRIC - ) - self._publish_time_ms = self._statsd.get_timer( - metrics.LOGS_PUBLISH_TIME_METRIC - ) - self._logs_lost_counter = self._statsd.get_counter( - metrics.LOGS_PUBLISHED_LOST_METRIC - ) - self._logs_truncated_gauge = self._statsd.get_gauge( - metrics.LOGS_TRUNCATED_METRIC - ) - - LOG.info('Initializing LogPublisher <%s>', self) - - def send_message(self, messages): - """Sends message to each configured topic. - - Note: - Falsy messages (i.e. empty) are not shipped to kafka - - See also - * :py:class:`monasca_log_api.common.model.Envelope` - * :py:meth:`._is_message_valid` - - :param dict|list messages: instance (or instances) of log envelope - """ - - if not messages: - return - if not isinstance(messages, list): - messages = [messages] - - sent_counter = 0 - num_of_msgs = len(messages) - - LOG.debug('About to publish %d messages to %s topics', - num_of_msgs, self._topics) - - try: - send_messages = [] - - for message in messages: - msg = self._transform_message(message) - send_messages.append(msg) - if CONF.monitoring.enable: - with self._publish_time_ms.time(name=None): - self._publish(send_messages) - else: - self._publish(send_messages) - - sent_counter = len(send_messages) - except Exception as ex: - LOG.exception('Failure in publishing messages to kafka') - raise ex - finally: - self._after_publish(sent_counter, num_of_msgs) - - def _transform_message(self, message): - """Transforms message into JSON. - - Method executes transformation operation for - single element. Operation is set of following - operations: - - * checking if message is valid - (:py:func:`.LogPublisher._is_message_valid`) - * truncating message if necessary - (:py:func:`.LogPublisher._truncate`) - - :param model.Envelope message: instance of message - :return: serialized message - :rtype: str - """ - if not self._is_message_valid(message): - raise InvalidMessageException() - truncated = self._truncate(message) - return encodeutils.safe_encode(truncated, incoming='utf-8') - - def _truncate(self, envelope): - """Truncates the message if needed. - - Each message send to kafka is verified. - Method checks if message serialized to json - exceeds maximum allowed size that can be posted to kafka - queue. If so, method truncates message property of the log - by difference between message and allowed size. - - :param Envelope envelope: original envelope - :return: serialized message - :rtype: str - """ - - msg_str = model.serialize_envelope(envelope) - envelope_size = ((len(bytearray(msg_str, 'utf-8', 'replace')) + - _TIMESTAMP_KEY_SIZE + - _KAFKA_META_DATA_SIZE) - if msg_str is not None else -1) - - diff_size = ((envelope_size - self.max_message_size) + - _TRUNCATION_SAFE_OFFSET) - - if diff_size > 1: - truncated_by = diff_size + _TRUNCATED_PROPERTY_SIZE - - LOG.warning(('Detected message that exceeds %d bytes,' - 'message will be truncated by %d bytes'), - self.max_message_size, - truncated_by) - - log_msg = envelope['log']['message'] - truncated_log_msg = log_msg[:-truncated_by] - - envelope['log']['truncated'] = True - envelope['log']['message'] = truncated_log_msg - if CONF.monitoring.enable: - self._logs_truncated_gauge.send(name=None, value=truncated_by) - - msg_str = rest_utils.as_json(envelope) - else: - if CONF.monitoring.enable: - self._logs_truncated_gauge.send(name=None, value=0) - - return msg_str - - def _publish(self, messages): - """Publishes messages to kafka. - - :param list messages: list of messages - """ - num_of_msg = len(messages) - - LOG.debug('Publishing %d messages', num_of_msg) - - try: - for topic in self._topics: - self._kafka_publisher.publish( - topic, - messages - ) - LOG.debug('Sent %d messages to topic %s', num_of_msg, topic) - except Exception as ex: - raise falcon.HTTPServiceUnavailable('Service unavailable', - str(ex), 60) - - @staticmethod - def _is_message_valid(message): - """Validates message before sending. - - Methods checks if message is :py:class:`model.Envelope`. - By being instance of this class it is ensured that all required - keys are found and they will have their values. - - """ - return message and isinstance(message, model.Envelope) - - def _after_publish(self, send_count, to_send_count): - """Executed after publishing to sent metrics. - - :param int send_count: how many messages have been sent - :param int to_send_count: how many messages should be sent - - """ - - failed_to_send = to_send_count - send_count - - if failed_to_send == 0: - LOG.debug('Successfully published all [%d] messages', - send_count) - else: - error_str = ('Failed to send all messages, %d ' - 'messages out of %d have not been published') - LOG.error(error_str, failed_to_send, to_send_count) - if CONF.monitoring.enable: - self._logs_published_counter.increment(value=send_count) - self._logs_lost_counter.increment(value=failed_to_send) diff --git a/monasca_log_api/app/base/model.py b/monasca_log_api/app/base/model.py deleted file mode 100644 index 0acc78ac..00000000 --- a/monasca_log_api/app/base/model.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_utils import timeutils -import six - -from monasca_log_api.common.rest import utils as rest_utils - - -def serialize_envelope(envelope): - """Returns json representation of an envelope. - - :return: json object of envelope - :rtype: six.text_type - - """ - json = rest_utils.as_json(envelope, ensure_ascii=False) - - if six.PY2: - raw = six.text_type(json.replace(r'\\', r'\\\\'), encoding='utf-8', - errors='replace') - else: - raw = json - - return raw - - -class LogEnvelopeException(Exception): - pass - - -class Envelope(dict): - def __init__(self, log, meta): - if not log: - error_msg = 'Envelope cannot be created without log' - raise LogEnvelopeException(error_msg) - if 'tenantId' not in meta or not meta.get('tenantId'): - error_msg = 'Envelope cannot be created without tenant' - raise LogEnvelopeException(error_msg) - - creation_time = self._get_creation_time() - super(Envelope, self).__init__( - log=log, - creation_time=creation_time, - meta=meta - ) - - @staticmethod - def _get_creation_time(): - return timeutils.utcnow_ts() - - @classmethod - def new_envelope(cls, log, tenant_id, region, dimensions=None): - """Creates new log envelope - - Log envelope is combined ouf of following properties - - * log - dict - * creation_time - timestamp - * meta - meta block - - Example output json would like this: - - .. code-block:: json - - { - "log": { - "message": "Some message", - "dimensions": { - "hostname": "devstack" - } - }, - "creation_time": 1447834886, - "meta": { - "tenantId": "e4bd29509eda473092d32aadfee3e7b1", - "region": "pl" - } - } - - :param dict log: original log element (containing message and other - params - :param str tenant_id: tenant id to be put in meta field - :param str region: region to be put in meta field - :param dict dimensions: additional dimensions to be appended to log - object dimensions - - """ - if dimensions: - log['dimensions'].update(dimensions) - - log_meta = { - 'region': region, - 'tenantId': tenant_id - } - - return cls(log, log_meta) - - @property - def log(self): - return self.get('log', None) - - @property - def creation_time(self): - return self.get('creation_time', None) - - @property - def meta(self): - return self.get('meta', None) diff --git a/monasca_log_api/app/base/request.py b/monasca_log_api/app/base/request.py deleted file mode 100644 index 11d6a97d..00000000 --- a/monasca_log_api/app/base/request.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon - -from monasca_common.policy import policy_engine as policy - -from monasca_log_api.app.base import request_context -from monasca_log_api.app.base import validation -from monasca_log_api import policies - -policy.POLICIES = policies - - -_TENANT_ID_PARAM = 'tenant_id' -"""Name of the query-param pointing at project-id (tenant-id)""" - - -class Request(falcon.Request): - """Variation of falcon.Request with context - - Following class enhances :py:class:`falcon.Request` with - :py:class:`context.RequestContext`. - - """ - - def __init__(self, env, options=None): - super(Request, self).__init__(env, options) - self.context = request_context.RequestContext.from_environ(self.env) - - def validate(self, content_types): - """Performs common request validation - - Validation checklist (in that order): - - * :py:func:`validation.validate_content_type` - * :py:func:`validation.validate_payload_size` - * :py:func:`validation.validate_cross_tenant` - - :param content_types: allowed content-types handler supports - :type content_types: list - :raises Exception: if any of the validation fails - - """ - validation.validate_content_type(self, content_types) - validation.validate_payload_size(self) - validation.validate_cross_tenant( - tenant_id=self.project_id, - roles=self.roles, - cross_tenant_id=self.cross_project_id - ) - - @property - def project_id(self): - """Returns project-id (tenant-id) - - :return: project-id - :rtype: str - - """ - return self.context.project_id - - @property - def cross_project_id(self): - """Returns project-id (tenant-id) found in query params. - - This particular project-id is later on identified as - cross-project-id - - :return: project-id - :rtype: str - - """ - return self.get_param(_TENANT_ID_PARAM, required=False) - - @property - def user_id(self): - """Returns user-id - - :return: user-id - :rtype: str - - """ - return self.context.user - - @property - def roles(self): - """Returns roles associated with user - - :return: user's roles - :rtype: list - - """ - return self.context.roles - - def can(self, action, target=None): - return self.context.can(action, target) - - def __repr__(self): - return '%s, context=%s' % (self.path, self.context) diff --git a/monasca_log_api/app/base/request_context.py b/monasca_log_api/app/base/request_context.py deleted file mode 100644 index 5805c40d..00000000 --- a/monasca_log_api/app/base/request_context.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# Copyright 2018 OP5 AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_common.policy import policy_engine as policy -from oslo_context import context - -from monasca_log_api import policies - -policy.POLICIES = policies - - -class RequestContext(context.RequestContext): - """RequestContext. - - RequestContext is customized version of - :py:class:oslo_context.context.RequestContext. - """ - - def can(self, action, target=None): - if target is None: - target = {'project_id': self.project_id, - 'user_id': self.user_id} - - return policy.authorize(self, action=action, target=target) diff --git a/monasca_log_api/app/base/validation.py b/monasca_log_api/app/base/validation.py deleted file mode 100644 index 872625e6..00000000 --- a/monasca_log_api/app/base/validation.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import re - -import falcon -from oslo_log import log -import six - -from monasca_log_api.app.base import exceptions -from monasca_log_api import conf - -LOG = log.getLogger(__name__) -CONF = conf.CONF - -APPLICATION_TYPE_CONSTRAINTS = { - 'MAX_LENGTH': 255, - 'PATTERN': re.compile('^[a-zA-Z0-9_.\\-]+$') -} -"""Application type constraint used in validation. - -See :py:func:`Validations.validate_application_type` -""" -DIMENSION_NAME_CONSTRAINTS = { - 'MAX_LENGTH': 255, - 'PATTERN': re.compile('[^><={}(), \'";&]+$') -} -"""Constraint for name of single dimension. - -See :py:func:`Validations.validate_dimensions` -""" -DIMENSION_VALUE_CONSTRAINTS = { - 'MAX_LENGTH': 255 -} -"""Constraint for value of single dimension. - -See :py:func:`Validations.validate_dimensions` -""" - - -def validate_application_type(application_type=None): - """Validates application type. - - Validation won't take place if application_type is None. - For details see: :py:data:`APPLICATION_TYPE_CONSTRAINTS` - - :param str application_type: application type - """ - - def validate_length(): - if (len(application_type) > - APPLICATION_TYPE_CONSTRAINTS['MAX_LENGTH']): - msg = ('Application type {type} must be ' - '{length} characters or less') - raise exceptions.HTTPUnprocessableEntity( - msg.format( - type=application_type, - length=APPLICATION_TYPE_CONSTRAINTS[ - 'MAX_LENGTH'] - ) - ) - - def validate_match(): - if (not APPLICATION_TYPE_CONSTRAINTS['PATTERN'] - .match(application_type)): - raise exceptions.HTTPUnprocessableEntity( - 'Application type %s may only contain: "a-z A-Z 0-9 _ - ."' - % application_type - ) - - if application_type: - validate_length() - validate_match() - - -def _validate_dimension_name(name): - try: - if len(name) > DIMENSION_NAME_CONSTRAINTS['MAX_LENGTH']: - raise exceptions.HTTPUnprocessableEntity( - 'Dimension name %s must be 255 characters or less' % - name - ) - if name[0] == '_': - raise exceptions.HTTPUnprocessableEntity( - 'Dimension name %s cannot start with underscore (_)' % - name - ) - if not DIMENSION_NAME_CONSTRAINTS['PATTERN'].match(name): - raise exceptions.HTTPUnprocessableEntity( - 'Dimension name %s may not contain: %s' % - (name, '> < = { } ( ) \' " , ; &') - ) - except (TypeError, IndexError): - raise exceptions.HTTPUnprocessableEntity( - 'Dimension name cannot be empty' - ) - - -def _validate_dimension_value(value): - try: - value[0] - if len(value) > DIMENSION_VALUE_CONSTRAINTS['MAX_LENGTH']: - raise exceptions.HTTPUnprocessableEntity( - 'Dimension value %s must be 255 characters or less' % - value - ) - except (TypeError, IndexError): - raise exceptions.HTTPUnprocessableEntity( - 'Dimension value cannot be empty' - ) - - -def validate_dimensions(dimensions): - """Validates dimensions type. - - Empty dimensions are not being validated. - For details see: - - :param dict dimensions: dimensions to validate - - * :py:data:`DIMENSION_NAME_CONSTRAINTS` - * :py:data:`DIMENSION_VALUE_CONSTRAINTS` - """ - try: - for dim_name, dim_value in dimensions.items(): - _validate_dimension_name(dim_name) - _validate_dimension_value(dim_value) - except AttributeError: - raise exceptions.HTTPUnprocessableEntity( - 'Dimensions %s must be a dictionary (map)' % dimensions) - - -def validate_content_type(req, allowed): - """Validates content type. - - Method validates request against correct - content type. - - If content-type cannot be established (i.e. header is missing), - :py:class:`falcon.HTTPMissingHeader` is thrown. - If content-type is not **application/json** or **text/plain**, - :py:class:`falcon.HTTPUnsupportedMediaType` is thrown. - - - :param falcon.Request req: current request - :param iterable allowed: allowed content type - - :exception: :py:class:`falcon.HTTPMissingHeader` - :exception: :py:class:`falcon.HTTPUnsupportedMediaType` - """ - content_type = req.content_type - - LOG.debug('Content-Type is %s', content_type) - - if content_type is None or len(content_type) == 0: - raise falcon.HTTPMissingHeader('Content-Type') - - if content_type not in allowed: - sup_types = ', '.join(allowed) - details = ('Only [%s] are accepted as logs representations' - % str(sup_types)) - raise falcon.HTTPUnsupportedMediaType(description=details) - - -def validate_payload_size(req): - """Validates payload size. - - Method validates sent payload size. - It expects that http header **Content-Length** is present. - If it does not, method raises :py:class:`falcon.HTTPLengthRequired`. - Otherwise values is being compared with :: - - [service] - max_log_size = 1048576 - - **max_log_size** refers to the maximum allowed content length. - If it is exceeded :py:class:`falcon.HTTPRequestEntityTooLarge` is - thrown. - - :param falcon.Request req: current request - - :exception: :py:class:`falcon.HTTPLengthRequired` - :exception: :py:class:`falcon.HTTPRequestEntityTooLarge` - - """ - payload_size = req.content_length - max_size = CONF.service.max_log_size - - LOG.debug('Payload (content-length) is %s', str(payload_size)) - - if payload_size is None: - raise falcon.HTTPLengthRequired( - title='Content length header is missing', - description='Content length is required to estimate if ' - 'payload can be processed' - ) - - if payload_size >= max_size: - raise falcon.HTTPPayloadTooLarge( - title='Log payload size exceeded', - description='Maximum allowed size is %d bytes' % max_size - ) - - -def validate_is_delegate(roles): - delegate_roles = CONF.roles_middleware.delegate_roles - if roles and delegate_roles: - roles = roles.split(',') if isinstance(roles, six.string_types) \ - else roles - return any(x in set(delegate_roles) for x in roles) - return False - - -def validate_cross_tenant(tenant_id, cross_tenant_id, roles): - - if not validate_is_delegate(roles): - if cross_tenant_id: - raise falcon.HTTPForbidden( - 'Permission denied', - 'Projects %s cannot POST cross tenant logs' % tenant_id - ) - - -def validate_log_message(log_object): - """Validates log property. - - Log property should have message property. - - Args: - log_object (dict): log property - """ - if 'message' not in log_object: - raise exceptions.HTTPUnprocessableEntity( - 'Log property should have message' - ) - - -def validate_authorization(http_request, authorized_rules_list): - """Validates whether is authorized according to provided policy rules list. - - If authorization fails, 401 is thrown with appropriate description. - Additionally response specifies 'WWW-Authenticate' header with 'Token' - value challenging the client to use different token (the one with - different set of roles which can access the service). - """ - challenge = 'Token' - for rule in authorized_rules_list: - try: - http_request.can(rule) - return - except Exception as ex: - LOG.debug(ex) - - raise falcon.HTTPUnauthorized('Forbidden', - 'The request does not have access to this service', - challenge) diff --git a/monasca_log_api/app/controller/__init__.py b/monasca_log_api/app/controller/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/app/controller/api/__init__.py b/monasca_log_api/app/controller/api/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/app/controller/api/headers.py b/monasca_log_api/app/controller/api/headers.py deleted file mode 100644 index 1a6734c8..00000000 --- a/monasca_log_api/app/controller/api/headers.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2015 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections - -LogApiHeader = collections.namedtuple('LogApiHeader', ['name', 'is_required']) -"""Tuple describing a header.""" - -X_TENANT_ID = LogApiHeader(name='X-Tenant-Id', is_required=False) -X_ROLES = LogApiHeader(name='X-Roles', is_required=False) -X_APPLICATION_TYPE = LogApiHeader(name='X-Application-Type', is_required=False) -X_DIMENSIONS = LogApiHeader(name='X_Dimensions', is_required=False) diff --git a/monasca_log_api/app/controller/api/healthcheck_api.py b/monasca_log_api/app/controller/api/healthcheck_api.py deleted file mode 100644 index a1eda841..00000000 --- a/monasca_log_api/app/controller/api/healthcheck_api.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections - -import falcon -from oslo_log import log - -LOG = log.getLogger(__name__) - -HealthCheckResult = collections.namedtuple('HealthCheckResult', - ['status', 'details']) - - -# TODO(feature) monasca-common candidate -class HealthChecksApi(object): - """HealthChecks Api - - HealthChecksApi server information regarding health of the API. - - """ - - def __init__(self): - super(HealthChecksApi, self).__init__() - LOG.info('Initializing HealthChecksApi!') - - def on_get(self, req, res): - """Complex healthcheck report on GET. - - Returns complex report regarding API well being - and all dependent services. - - :param falcon.Request req: current request - :param falcon.Response res: current response - """ - res.status = falcon.HTTP_501 - - def on_head(self, req, res): - """Simple healthcheck report on HEAD. - - In opposite to :py:meth:`.HealthChecksApi.on_get`, this - method is supposed to execute ASAP to inform user that - API is up and running. - - :param falcon.Request req: current request - :param falcon.Response res: current response - - """ - res.status = falcon.HTTP_501 diff --git a/monasca_log_api/app/controller/api/logs_api.py b/monasca_log_api/app/controller/api/logs_api.py deleted file mode 100644 index 8d2f8b88..00000000 --- a/monasca_log_api/app/controller/api/logs_api.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon -from oslo_log import log - -from monasca_log_api import conf -from monasca_log_api.monitoring import client -from monasca_log_api.monitoring import metrics - -CONF = conf.CONF -LOG = log.getLogger(__name__) - - -class LogsApi(object): - """Logs API. - - Logs API acts as RESTful endpoint accepting - messages contains collected log entries from the system. - Works as gateway for any further processing for accepted data. - - """ - def __init__(self): - super(LogsApi, self).__init__() - if CONF.monitoring.enable: - self._statsd = client.get_client() - - # create_common counters, gauges etc. - self._metrics_dimensions = dimensions = {'version': self.version} - - self._logs_in_counter = self._statsd.get_counter( - name=metrics.LOGS_RECEIVED_METRIC, - dimensions=dimensions - ) - self._logs_size_gauge = self._statsd.get_gauge( - name=metrics.LOGS_RECEIVED_BYTE_SIZE_METRICS, - dimensions=dimensions - ) - self._logs_rejected_counter = self._statsd.get_counter( - name=metrics.LOGS_REJECTED_METRIC, - dimensions=dimensions - ) - self._logs_processing_time = self._statsd.get_timer( - name=metrics.LOGS_PROCESSING_TIME_METRIC, - dimensions=dimensions - ) - - LOG.info('Initializing LogsApi %s!' % self.version) - - def on_post(self, req, res): - """Accepts sent logs as text or json. - - Accepts logs sent to resource which should - be sent to kafka queue. - - :param req: current request - :param res: current response - - """ - res.status = falcon.HTTP_501 # pragma: no cover - - def on_get(self, req, res): - """Queries logs matching specified dimension values. - - Performs queries on the underlying log storage - against a time range and set of dimension values. - - :param req: current request - :param res: current response - - """ - res.status = falcon.HTTP_501 # pragma: no cover - - @property - def version(self): - return getattr(self, 'VERSION') diff --git a/monasca_log_api/app/controller/api/versions_api.py b/monasca_log_api/app/controller/api/versions_api.py deleted file mode 100644 index bdc078f8..00000000 --- a/monasca_log_api/app/controller/api/versions_api.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2015 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon -from oslo_log import log - -LOG = log.getLogger(__name__) - - -class VersionsAPI(object): - """Versions API - - VersionsAPI returns information about API itself. - - """ - - def __init__(self): - super(VersionsAPI, self).__init__() - LOG.info('Initializing VersionsAPI!') - - def on_get(self, req, res, version_id): - res.status = falcon.HTTP_501 diff --git a/monasca_log_api/app/controller/healthchecks.py b/monasca_log_api/app/controller/healthchecks.py deleted file mode 100644 index 93ba85fb..00000000 --- a/monasca_log_api/app/controller/healthchecks.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon -from monasca_log_api.common.rest import utils as rest_utils - -from monasca_log_api.app.base.validation import validate_authorization -from monasca_log_api.app.controller.api import healthcheck_api -from monasca_log_api.healthcheck import kafka_check - - -class HealthChecks(healthcheck_api.HealthChecksApi): - # response configuration - CACHE_CONTROL = ['must-revalidate', 'no-cache', 'no-store'] - - # response codes - HEALTHY_CODE_GET = falcon.HTTP_OK - HEALTHY_CODE_HEAD = falcon.HTTP_NO_CONTENT - NOT_HEALTHY_CODE = falcon.HTTP_SERVICE_UNAVAILABLE - - def __init__(self): - self._kafka_check = kafka_check.KafkaHealthCheck() - super(HealthChecks, self).__init__() - - def on_head(self, req, res): - validate_authorization(req, ['log_api:healthcheck:head']) - res.status = self.HEALTHY_CODE_HEAD - res.cache_control = self.CACHE_CONTROL - - def on_get(self, req, res): - # at this point we know API is alive, so - # keep up good work and verify kafka status - validate_authorization(req, ['log_api:healthcheck:get']) - kafka_result = self._kafka_check.healthcheck() - - # in case it'd be unhealthy, - # message will contain error string - status_data = { - 'kafka': kafka_result.message - } - - # Really simple approach, ideally that should be - # part of monasca-common with some sort of registration of - # healthchecks concept - - res.status = (self.HEALTHY_CODE_GET - if kafka_result.healthy else self.NOT_HEALTHY_CODE) - res.cache_control = self.CACHE_CONTROL - res.body = rest_utils.as_json(status_data) diff --git a/monasca_log_api/app/controller/v2/__init__.py b/monasca_log_api/app/controller/v2/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/app/controller/v2/aid/__init__.py b/monasca_log_api/app/controller/v2/aid/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/app/controller/v2/aid/service.py b/monasca_log_api/app/controller/v2/aid/service.py deleted file mode 100644 index 29fd7786..00000000 --- a/monasca_log_api/app/controller/v2/aid/service.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime - -from oslo_config import cfg -from oslo_log import log - -from monasca_log_api.app.base import exceptions -from monasca_log_api.app.base import model -from monasca_log_api.app.base import validation -from monasca_log_api import conf - -LOG = log.getLogger(__name__) -CONF = conf.CONF - -EPOCH_START = datetime.datetime(1970, 1, 1) - - -class LogCreator(object): - """Transforms logs, - - Takes care of transforming information received via - HTTP requests into log and log envelopes objects. - - For more details see following: - - * :py:func:`LogCreator.new_log` - * :py:func:`LogCreator.new_log_envelope` - - """ - - def __init__(self): - self._log = log.getLogger('service.LogCreator') - self._log.info('Initializing LogCreator') - - @staticmethod - def _create_meta_info(tenant_id): - """Creates meta block for log envelope. - - Additionally method accesses oslo configuration, - looking for *service.region* configuration property. - - For more details see :py:data:`service_opts` - - :param tenant_id: ID of the tenant - :type tenant_id: str - :return: meta block - :rtype: dict - - """ - return { - 'tenantId': tenant_id, - 'region': cfg.CONF.service.region - } - - def new_log(self, - application_type, - dimensions, - payload, - content_type='application/json', - validate=True): - """Creates new log object. - - :param str application_type: origin of the log - :param dict dimensions: dictionary of dimensions (any data sent to api) - :param stream payload: stream to read log entry from - :param str content_type: actual content type used to send data to - server - :param bool validate: by default True, marks if log should be validated - :return: log object - :rtype: dict - - :keyword: log_object - """ - - if not payload: - return None - - # normalize_yet_again - application_type = parse_application_type(application_type) - dimensions = parse_dimensions(dimensions) - - if validate: - self._log.debug('Validation enabled, proceeding with validation') - validation.validate_application_type(application_type) - validation.validate_dimensions(dimensions) - - self._log.debug( - 'application_type=%s,dimensions=%s' % ( - application_type, dimensions) - ) - - log_object = {} - if content_type == 'application/json': - log_object.update(payload) - else: - log_object.update({'message': payload}) - - validation.validate_log_message(log_object) - - dimensions['component'] = application_type - log_object.update({'dimensions': dimensions}) - - return log_object - - def new_log_envelope(self, log_object, tenant_id): - return model.Envelope( - log=log_object, - meta=self._create_meta_info(tenant_id) - ) - - -def parse_application_type(app_type): - if app_type: - app_type = app_type.strip() - return app_type if app_type else None - - -def parse_dimensions(dimensions): - if not dimensions: - raise exceptions.HTTPUnprocessableEntity('Dimension are required') - - new_dimensions = {} - dimensions = map(str.strip, dimensions.split(',')) - - for dim in dimensions: - if not dim: - raise exceptions.HTTPUnprocessableEntity( - 'Dimension cannot be empty') - elif ':' not in dim: - raise exceptions.HTTPUnprocessableEntity( - '%s is not a valid dimension' % dim) - - dim = dim.split(':') - name = str(dim[0].strip()) if dim[0] else None - value = str(dim[1].strip()) if dim[1] else None - if name and value: - new_dimensions.update({name: value}) - - return new_dimensions diff --git a/monasca_log_api/app/controller/v2/logs.py b/monasca_log_api/app/controller/v2/logs.py deleted file mode 100644 index 5dc59186..00000000 --- a/monasca_log_api/app/controller/v2/logs.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon -import six - - -from monasca_log_api.app.base import log_publisher -from monasca_log_api.app.base.validation import validate_authorization -from monasca_log_api.app.controller.api import headers -from monasca_log_api.app.controller.api import logs_api -from monasca_log_api.app.controller.v2.aid import service -from monasca_log_api import conf - -CONF = conf.CONF -_DEPRECATED_INFO = ('/v2.0/log/single has been deprecated. ' - 'Please use /v3.0/logs') - - -class Logs(logs_api.LogsApi): - """Logs Api V2.""" - - VERSION = 'v2.0' - SUPPORTED_CONTENT_TYPES = {'application/json', 'text/plain'} - - def __init__(self): - self._log_creator = service.LogCreator() - self._kafka_publisher = log_publisher.LogPublisher() - super(Logs, self).__init__() - - @falcon.deprecated(_DEPRECATED_INFO) - def on_post(self, req, res): - validate_authorization(req, ['log_api:logs:post']) - if CONF.monitoring.enable: - with self._logs_processing_time.time(name=None): - self.process_on_post_request(req, res) - else: - self.process_on_post_request(req, res) - - def process_on_post_request(self, req, res): - try: - req.validate(self.SUPPORTED_CONTENT_TYPES) - tenant_id = (req.project_id if req.project_id - else req.cross_project_id) - - log = self.get_log(request=req) - envelope = self.get_envelope( - log=log, - tenant_id=tenant_id - ) - if CONF.monitoring.enable: - self._logs_size_gauge.send(name=None, - value=int(req.content_length)) - self._logs_in_counter.increment() - except Exception: - # any validation that failed means - # log is invalid and rejected - if CONF.monitoring.enable: - self._logs_rejected_counter.increment() - raise - - self._kafka_publisher.send_message(envelope) - - res.status = falcon.HTTP_204 - res.add_link( - target=str(_get_v3_link(req)), - rel='current', # [RFC5005] - title='V3 Logs', - type_hint='application/json' - ) - res.append_header('DEPRECATED', 'true') - - def get_envelope(self, log, tenant_id): - return self._log_creator.new_log_envelope( - log_object=log, - tenant_id=tenant_id - ) - - def get_log(self, request): - return self._log_creator.new_log( - application_type=request.get_header(*headers.X_APPLICATION_TYPE), - dimensions=request.get_header(*headers.X_DIMENSIONS), - payload=request.media, - content_type=request.content_type - ) - - -def _get_v3_link(req): - self_uri = req.uri - if six.PY2: - self_uri = self_uri.decode('UTF-8') - base_uri = self_uri.replace(req.relative_uri, '') - return '%s/v3.0/logs' % base_uri diff --git a/monasca_log_api/app/controller/v3/__init__.py b/monasca_log_api/app/controller/v3/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/app/controller/v3/aid/__init__.py b/monasca_log_api/app/controller/v3/aid/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/app/controller/v3/aid/bulk_processor.py b/monasca_log_api/app/controller/v3/aid/bulk_processor.py deleted file mode 100644 index 8a431930..00000000 --- a/monasca_log_api/app/controller/v3/aid/bulk_processor.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_log import log - -from monasca_log_api.app.base import log_publisher -from monasca_log_api.app.base import model -from monasca_log_api.app.base import validation -from monasca_log_api import conf - -LOG = log.getLogger(__name__) -CONF = conf.CONF - - -class BulkProcessor(log_publisher.LogPublisher): - """BulkProcessor for effective log processing and publishing. - - BulkProcessor is customized version of - :py:class:`monasca_log_api.app.base.log_publisher.LogPublisher` - that utilizes processing of bulk request inside single loop. - - """ - - def __init__(self, logs_in_counter=None, logs_rejected_counter=None): - """Initializes BulkProcessor. - - :param logs_in_counter: V3 received logs counter - :param logs_rejected_counter: V3 rejected logs counter - """ - super(BulkProcessor, self).__init__() - - if CONF.monitoring.enable: - assert logs_in_counter is not None - assert logs_rejected_counter is not None - self._logs_in_counter = logs_in_counter - self._logs_rejected_counter = logs_rejected_counter - - self.service_region = CONF.service.region - - def send_message(self, logs, global_dimensions=None, log_tenant_id=None): - """Sends bulk package to kafka - - :param list logs: received logs - :param dict global_dimensions: global dimensions for each log - :param str log_tenant_id: tenant who sent logs - """ - - num_of_msgs = len(logs) if logs else 0 - sent_count = 0 - to_send_msgs = [] - - LOG.debug('Bulk package ', - num_of_msgs, global_dimensions, log_tenant_id) - - try: - for log_el in logs: - t_el = self._transform_message(log_el, - global_dimensions, - log_tenant_id) - if t_el: - to_send_msgs.append(t_el) - if CONF.monitoring.enable: - with self._publish_time_ms.time(name=None): - self._publish(to_send_msgs) - else: - self._publish(to_send_msgs) - - sent_count = len(to_send_msgs) - - except Exception as ex: - LOG.error('Failed to send bulk package ', - num_of_msgs, global_dimensions) - LOG.exception(ex) - raise ex - finally: - if CONF.monitoring.enable: - self._update_counters(len(to_send_msgs), num_of_msgs) - self._after_publish(sent_count, len(to_send_msgs)) - - def _update_counters(self, in_counter, to_send_counter): - rejected_counter = to_send_counter - in_counter - - self._logs_in_counter.increment(value=in_counter) - self._logs_rejected_counter.increment(value=rejected_counter) - - def _transform_message(self, log_element, *args): - try: - validation.validate_log_message(log_element) - - log_envelope = model.Envelope.new_envelope( - log=log_element, - tenant_id=args[1], - region=self.service_region, - dimensions=self._get_dimensions(log_element, - global_dims=args[0]) - ) - - msg_payload = (super(BulkProcessor, self) - ._transform_message(log_envelope)) - - return msg_payload - except Exception as ex: - LOG.error('Log transformation failed, rejecting log') - LOG.exception(ex) - - return None - - def _create_envelope(self, log_element, tenant_id, dimensions=None): - """Create a log envelope. - - :param dict log_element: raw log element - :param str tenant_id: tenant who sent logs - :param dict dimensions: log dimensions - :return: log envelope - :rtype: model.Envelope - - """ - return - - def _get_dimensions(self, log_element, global_dims=None): - """Get the dimensions of log element. - - If global dimensions are specified and passed to this method, - both instances are merged with each other. - - If neither is specified empty dictionary is returned. - - If only local dimensions are specified they are returned without any - additional operations. The last statement applies also - to global dimensions. - - :param dict log_element: raw log instance - :param dict global_dims: global dimensions or None - :return: local dimensions merged with global dimensions - :rtype: dict - """ - local_dims = log_element.get('dimensions', {}) - - if not global_dims: - global_dims = {} - if local_dims: - validation.validate_dimensions(local_dims) - - dimensions = global_dims.copy() - dimensions.update(local_dims) - - return dimensions diff --git a/monasca_log_api/app/controller/v3/aid/helpers.py b/monasca_log_api/app/controller/v3/aid/helpers.py deleted file mode 100644 index c5ad1872..00000000 --- a/monasca_log_api/app/controller/v3/aid/helpers.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2014 Hewlett-Packard -# Copyright 2015 Cray Inc. All Rights Reserved. -# Copyright 2016 Hewlett Packard Enterprise Development Company LP -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon -from monasca_log_api.common.rest import utils as rest_utils -from oslo_log import log - -from monasca_log_api.app.base import exceptions -from monasca_log_api.app.base import validation - -LOG = log.getLogger(__name__) - - -def read_json_msg_body(req): - """Read the json_msg from the http request body and return them as JSON. - - :param req: HTTP request object. - :return: Returns the metrics as a JSON object. - :raises falcon.HTTPBadRequest: - """ - try: - body = req.media - - if body is not None: - return body - else: - raise falcon.HTTPBadRequest('Bad request', - 'Request body is Empty') - - except rest_utils.exceptions.DataConversionException as ex: - LOG.debug(ex) - raise falcon.HTTPBadRequest('Bad request', - 'Request body is not valid JSON') - except ValueError as ex: - LOG.debug(ex) - raise falcon.HTTPBadRequest('Bad request', - 'Request body is not valid JSON') - - -def get_global_dimensions(request_body): - """Get the top level dimensions in the HTTP request body.""" - global_dims = request_body.get('dimensions', {}) - validation.validate_dimensions(global_dims) - return global_dims - - -def get_logs(request_body): - """Get the logs in the HTTP request body.""" - if 'logs' not in request_body: - raise exceptions.HTTPUnprocessableEntity( - 'Unprocessable Entity Logs not found') - return request_body['logs'] diff --git a/monasca_log_api/app/controller/v3/logs.py b/monasca_log_api/app/controller/v3/logs.py deleted file mode 100644 index 349382b9..00000000 --- a/monasca_log_api/app/controller/v3/logs.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P. -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon -from oslo_log import log - -from monasca_log_api.app.base import exceptions -from monasca_log_api.app.base import validation -from monasca_log_api.app.controller.api import logs_api -from monasca_log_api.app.controller.v3.aid import bulk_processor -from monasca_log_api.app.controller.v3.aid import helpers -from monasca_log_api import conf -from monasca_log_api.monitoring import metrics - - -CONF = conf.CONF -LOG = log.getLogger(__name__) -_LOG_API_DEPRECATED = ('This API has been deprecated. Please use ' - 'monasca-api/logs') - - -class Logs(logs_api.LogsApi): - - VERSION = 'v3.0' - SUPPORTED_CONTENT_TYPES = {'application/json'} - - def __init__(self): - super(Logs, self).__init__() - - if CONF.monitoring.enable: - self._processor = bulk_processor.BulkProcessor( - logs_in_counter=self._logs_in_counter, - logs_rejected_counter=self._logs_rejected_counter - ) - self._bulks_rejected_counter = self._statsd.get_counter( - name=metrics.LOGS_BULKS_REJECTED_METRIC, - dimensions=self._metrics_dimensions - ) - else: - self._processor = bulk_processor.BulkProcessor() - - @falcon.deprecated(_LOG_API_DEPRECATED) - def on_post(self, req, res): - validation.validate_authorization(req, ['log_api:logs:post']) - if CONF.monitoring.enable: - with self._logs_processing_time.time(name=None): - self.process_on_post_request(req, res) - else: - self.process_on_post_request(req, res) - - def process_on_post_request(self, req, res): - try: - req.validate(self.SUPPORTED_CONTENT_TYPES) - - request_body = helpers.read_json_msg_body(req) - - log_list = self._get_logs(request_body) - global_dimensions = self._get_global_dimensions(request_body) - - except Exception as ex: - LOG.error('Entire bulk package has been rejected') - LOG.exception(ex) - if CONF.monitoring.enable: - self._bulks_rejected_counter.increment(value=1) - - raise ex - - if CONF.monitoring.enable: - self._bulks_rejected_counter.increment(value=0) - self._logs_size_gauge.send(name=None, - value=int(req.content_length)) - - tenant_id = (req.cross_project_id if req.cross_project_id - else req.project_id) - - try: - self._processor.send_message( - logs=log_list, - global_dimensions=global_dimensions, - log_tenant_id=tenant_id - ) - except Exception as ex: - res.status = getattr(ex, 'status', falcon.HTTP_500) - return - - res.status = falcon.HTTP_204 - - @staticmethod - def _get_global_dimensions(request_body): - """Get the top level dimensions in the HTTP request body.""" - global_dims = request_body.get('dimensions', {}) - validation.validate_dimensions(global_dims) - return global_dims - - @staticmethod - def _get_logs(request_body): - """Get the logs in the HTTP request body.""" - if 'logs' not in request_body: - raise exceptions.HTTPUnprocessableEntity( - 'Unprocessable Entity Logs not found') - return request_body['logs'] diff --git a/monasca_log_api/app/controller/versions.py b/monasca_log_api/app/controller/versions.py deleted file mode 100644 index 972d15e0..00000000 --- a/monasca_log_api/app/controller/versions.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon -import six - -from monasca_log_api.common.rest import utils as rest_utils - -from monasca_log_api.app.base.validation import validate_authorization -from monasca_log_api.app.controller.api import versions_api - -_VERSIONS_TPL_DICT = { - 'v2.0': { - 'id': 'v2.0', - 'links': [ - { - 'rel': 'logs', - 'href': '/log/single' - } - ], - 'status': 'DEPRECATED', - 'updated': "2015-09-01T00:00:00Z" - }, - 'v3.0': { - 'id': 'v3.0', - 'links': [ - { - 'rel': 'logs', - 'href': '/logs' - } - ], - 'status': 'CURRENT', - 'updated': "2016-03-01T00:00:00Z" - } -} - - -class Versions(versions_api.VersionsAPI): - """Versions Api""" - - @staticmethod - def handle_none_version_id(req, res, result): - for version in _VERSIONS_TPL_DICT: - selected_version = _parse_version(version, req) - result['elements'].append(selected_version) - res.body = rest_utils.as_json(result, sort_keys=True) - res.status = falcon.HTTP_200 - - @staticmethod - def handle_version_id(req, res, result, version_id): - if version_id in _VERSIONS_TPL_DICT: - result['elements'].append(_parse_version(version_id, req)) - res.body = rest_utils.as_json(result, sort_keys=True) - res.status = falcon.HTTP_200 - else: - error_body = {'message': '%s is not valid version' % version_id} - res.body = rest_utils.as_json(error_body) - res.status = falcon.HTTP_400 - - def on_get(self, req, res, version_id=None): - validate_authorization(req, ['log_api:versions:get']) - result = { - 'links': _get_common_links(req), - 'elements': [] - } - if version_id is None: - self.handle_none_version_id(req, res, result) - else: - self.handle_version_id(req, res, result, version_id) - - -def _get_common_links(req): - self_uri = req.uri - if six.PY2: - self_uri = self_uri.decode(rest_utils.ENCODING) - base_uri = self_uri.rpartition(req.path)[0] - return [ - { - 'rel': 'self', - 'href': self_uri - }, - { - 'rel': 'version', - 'href': '%s/version' % base_uri - }, - { - 'rel': 'healthcheck', - 'href': '%s/healthcheck' % base_uri - } - ] - - -def _parse_version(version_id, req): - self_uri = req.uri - if six.PY2: - self_uri = self_uri.decode(rest_utils.ENCODING) - base_uri = self_uri.rpartition(req.path)[0] - - # need to get template dict, consecutive calls - # needs to operate on unmodified instance - - selected_version = _VERSIONS_TPL_DICT[version_id].copy() - raw_links = selected_version['links'] - links = [] - - for link in raw_links: - raw_link_href = link.get('href') - raw_link_rel = link.get('rel') - link_href = base_uri + '/' + version_id + raw_link_href - links.append({ - 'href': link_href, - 'rel': raw_link_rel - }) - selected_version['links'] = links - - return selected_version diff --git a/monasca_log_api/app/main.py b/monasca_log_api/app/main.py deleted file mode 100644 index 16e7a8c7..00000000 --- a/monasca_log_api/app/main.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Allows to run monasca-log-api from within local [dev] environment. -Primarily used for development. -""" - -import sys - -from paste import deploy -from paste import httpserver - -from monasca_log_api import version - - -def get_wsgi_app(): - config_dir = 'etc/monasca' - - return deploy.loadapp( - 'config:%s/log-api-paste.ini' % config_dir, - relative_to='./', - name='main' - ) - - -def main(): - wsgi_app = get_wsgi_app() - server_version = 'log-api/%s' % version.version_str - - server = httpserver.serve(application=wsgi_app, host='127.0.0.1', - port=5607, server_version=server_version) - return server - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/monasca_log_api/app/wsgi.py b/monasca_log_api/app/wsgi.py deleted file mode 100644 index 61cd25fc..00000000 --- a/monasca_log_api/app/wsgi.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Use this file for deploying the API under mod_wsgi. -""" - -from paste import deploy - -application = None - - -def main(): - base_dir = '/etc/monasca/' - conf = '%slog-api-paste.ini' % base_dir - app = deploy.loadapp('config:%s' % conf) - return app - - -if __name__ == '__main__' or __name__.startswith('_mod_wsgi'): - application = main() diff --git a/monasca_log_api/common/rest/__init__.py b/monasca_log_api/common/rest/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/common/rest/exceptions.py b/monasca_log_api/common/rest/exceptions.py deleted file mode 100644 index 7054ff97..00000000 --- a/monasca_log_api/common/rest/exceptions.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2015 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -class UnsupportedContentTypeException(Exception): - """Exception thrown if content type is not supported.""" - pass - - -class UnreadableContentError(IOError): - """Exception thrown if reading data fails - - :py:class`.UnreadableContentError` may be thrown - if data was impossible to read from input - - """ - pass - - -class DataConversionException(Exception): - """Exception thrown if data transformation fails - - :py:class`.DataConversionException` may be thrown - if data was impossible to transform into target - representation according to content_type classifier. - - """ - pass diff --git a/monasca_log_api/common/rest/utils.py b/monasca_log_api/common/rest/utils.py deleted file mode 100644 index abda2b65..00000000 --- a/monasca_log_api/common/rest/utils.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2015 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import simplejson as json -import six - -from monasca_log_api.common.rest import exceptions - -ENCODING = 'utf8' - -TEXT_CONTENT_TYPE = 'text/plain' -JSON_CONTENT_TYPE = 'application/json' - - -def _try_catch(fun): - - @six.wraps(fun) - def wrapper(*args, **kwargs): - try: - return fun(*args, **kwargs) - except Exception as ex: - raise exceptions.DataConversionException(str(ex)) - - return wrapper - - -@_try_catch -def as_json(data, **kwargs): - """Writes data as json. - - :param dict data: data to convert to json - :param kwargs kwargs: kwargs for json dumps - :return: json string - :rtype: str - """ - - if 'sort_keys' not in kwargs: - kwargs['sort_keys'] = False - if 'ensure_ascii' not in kwargs: - kwargs['ensure_ascii'] = False - - data = json.dumps(data, **kwargs) - - return data - - -@_try_catch -def from_json(data, **kwargs): - """Reads data from json str. - - :param str data: data to read - :param kwargs kwargs: kwargs for json loads - :return: read data - :rtype: dict - """ - return json.loads(data, **kwargs) - - -_READABLE_CONTENT_TYPES = { - TEXT_CONTENT_TYPE: lambda content: content, - JSON_CONTENT_TYPE: from_json -} - - -def read_body(payload, content_type=JSON_CONTENT_TYPE): - """Reads HTTP payload according to given content_type. - - Function is capable of reading from payload stream. - Read data is then processed according to content_type. - - Note: - Content-Type is validated. It means that if read_body - body is not capable of reading data in requested type, - it will throw an exception. - - If read data was empty method will return false boolean - value to indicate that. - - Note: - There is no transformation if content type is equal to - 'text/plain'. What has been read is returned. - - :param stream payload: payload to read, payload should have read method - :param str content_type: payload content type, default to application/json - :return: read data, returned type depends on content_type or False - if empty - - :exception: :py:class:`.UnreadableBody` - in case of any failure when - reading data - - """ - if content_type not in _READABLE_CONTENT_TYPES: - msg = ('Cannot read %s, not in %s' % - (content_type, _READABLE_CONTENT_TYPES)) - raise exceptions.UnsupportedContentTypeException(msg) - - try: - content = payload.read() - if not content: - return None - except Exception as ex: - raise exceptions.UnreadableContentError(str(ex)) - - return _READABLE_CONTENT_TYPES[content_type](content) diff --git a/monasca_log_api/conf/__init__.py b/monasca_log_api/conf/__init__.py deleted file mode 100644 index 7d4dfe8a..00000000 --- a/monasca_log_api/conf/__init__.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import pkgutil - -from oslo_config import cfg -from oslo_log import log -from oslo_utils import importutils - -CONF = cfg.CONF -LOG = log.getLogger(__name__) - - -def load_conf_modules(): - """Loads all modules that contain configuration - - Method iterates over modules of :py:module:`monasca_log_api.conf` - and imports only those that contain following methods: - - - list_opts (required by oslo_config.genconfig) - - register_opts (required by :py:currentmodule:) - - """ - for modname in _list_module_names(): - mod = importutils.import_module('monasca_log_api.conf.' + modname) - required_funcs = ['register_opts', 'list_opts'] - for func in required_funcs: - if hasattr(mod, func): - yield mod - - -def _list_module_names(): - package_path = os.path.dirname(os.path.abspath(__file__)) - for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]): - if not (modname == "opts" and ispkg): - yield modname - - -def register_opts(): - """Registers all conf modules opts - - This method allows different modules to register - opts according to their needs. - - """ - for mod in load_conf_modules(): - mod.register_opts(CONF) - - -def list_opts(): - """Lists all conf modules opts. - - Goes through all conf modules and yields their opts - - """ - for mod in load_conf_modules(): - mod_opts = mod.list_opts() - yield mod_opts[0], mod_opts[1] diff --git a/monasca_log_api/conf/healthcheck.py b/monasca_log_api/conf/healthcheck.py deleted file mode 100644 index 00cf215b..00000000 --- a/monasca_log_api/conf/healthcheck.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg - -kafka_check_opts = [ - cfg.StrOpt('kafka_url', - required=True, - help='Url to kafka server'), - cfg.ListOpt('kafka_topics', - required=True, - default=['logs'], - help='Verify existence of configured topics') -] -kafka_check_group = cfg.OptGroup(name='kafka_healthcheck', - title='kafka_healthcheck') - - -def register_opts(conf): - conf.register_group(kafka_check_group) - conf.register_opts(kafka_check_opts, kafka_check_group) - - -def list_opts(): - return kafka_check_group, kafka_check_opts diff --git a/monasca_log_api/conf/log_publisher.py b/monasca_log_api/conf/log_publisher.py deleted file mode 100644 index 5a770729..00000000 --- a/monasca_log_api/conf/log_publisher.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg - -_MAX_MESSAGE_SIZE = 1048576 - -log_publisher_opts = [ - cfg.StrOpt('kafka_url', - required=True, - help='Url to kafka server'), - cfg.MultiStrOpt('topics', - default=['logs'], - help='Consumer topics'), - cfg.IntOpt('max_message_size', - default=_MAX_MESSAGE_SIZE, - required=True, - help=('Message max size that can be sent ' - 'to kafka, default to %d bytes' % _MAX_MESSAGE_SIZE)) -] -log_publisher_group = cfg.OptGroup(name='log_publisher', title='log_publisher') - - -def register_opts(conf): - conf.register_group(log_publisher_group) - conf.register_opts(log_publisher_opts, log_publisher_group) - - -def list_opts(): - return log_publisher_group, log_publisher_opts diff --git a/monasca_log_api/conf/monitoring.py b/monasca_log_api/conf/monitoring.py deleted file mode 100644 index 5567a687..00000000 --- a/monasca_log_api/conf/monitoring.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg - -_DEFAULT_HOST = '127.0.0.1' -_DEFAULT_PORT = 8125 -_DEFAULT_BUFFER_SIZE = 50 - -monitoring_opts = [ - cfg.BoolOpt('enable', - default=True, - help='Determine if self monitoring is enabled'), - cfg.HostAddressOpt('statsd_host', - default=_DEFAULT_HOST, - help=('IP address or host domain name of statsd server, default to %s' - % _DEFAULT_HOST)), - cfg.PortOpt('statsd_port', - default=_DEFAULT_PORT, - help='Port of statsd server, default to %d' % _DEFAULT_PORT), - cfg.IntOpt('statsd_buffer', - default=_DEFAULT_BUFFER_SIZE, - required=True, - help=('Maximum number of metric to buffer before sending, ' - 'default to %d' % _DEFAULT_BUFFER_SIZE)), - cfg.DictOpt('dimensions', default={}, - required=False, help='Additional dimensions that can be set') -] -monitoring_group = cfg.OptGroup(name='monitoring', title='monitoring') - - -def register_opts(conf): - conf.register_group(monitoring_group) - conf.register_opts(monitoring_opts, monitoring_group) - - -def list_opts(): - return monitoring_group, monitoring_opts diff --git a/monasca_log_api/conf/role_middleware.py b/monasca_log_api/conf/role_middleware.py deleted file mode 100644 index bb460f47..00000000 --- a/monasca_log_api/conf/role_middleware.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg - -role_m_opts = [ - cfg.ListOpt(name='path', - default='/', - help='List of paths where middleware applies to'), - cfg.ListOpt(name='default_roles', - default=['monasca-user'], - help='List of roles allowed to enter api'), - cfg.ListOpt(name='agent_roles', - default=None, - help=('List of roles, that if set, mean that request ' - 'comes from agent, thus is authorized in the same ' - 'time')), - cfg.ListOpt(name='delegate_roles', - default=['admin'], - help=('Roles that are allowed to POST logs on ' - 'behalf of another tenant (project)')), - cfg.ListOpt(name='check_roles', - default=['@'], - help=('Roles that are allowed to do check ' - 'version and health')) -] -role_m_group = cfg.OptGroup(name='roles_middleware', title='roles_middleware') - - -def register_opts(conf): - conf.register_group(role_m_group) - conf.register_opts(role_m_opts, role_m_group) - - -def list_opts(): - return role_m_group, role_m_opts diff --git a/monasca_log_api/conf/service.py b/monasca_log_api/conf/service.py deleted file mode 100644 index 0ce2496d..00000000 --- a/monasca_log_api/conf/service.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg - -_DEFAULT_MAX_LOG_SIZE = 1024 * 1024 - -service_opts = [ - cfg.StrOpt('region', - default=None, - help='Region'), - cfg.IntOpt('max_log_size', - default=_DEFAULT_MAX_LOG_SIZE, - help=('Refers to payload/envelope size. If either is exceeded' - 'API will throw an error')) -] -service_group = cfg.OptGroup(name='service', title='service') - - -def register_opts(conf): - conf.register_group(service_group) - conf.register_opts(service_opts, service_group) - - -def list_opts(): - return service_group, service_opts diff --git a/monasca_log_api/config.py b/monasca_log_api/config.py deleted file mode 100644 index 441a6bae..00000000 --- a/monasca_log_api/config.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sys - -from oslo_config import cfg -from oslo_log import log -from oslo_policy import opts as policy_opts - -from monasca_log_api import conf -from monasca_log_api import version - -CONF = conf.CONF -LOG = log.getLogger(__name__) - -_CONF_LOADED = False -_GUNICORN_MARKER = 'gunicorn' - - -def _is_running_under_gunicorn(): - """Evaluates if api runs under gunicorn""" - content = filter(lambda x: x != sys.executable and _GUNICORN_MARKER in x, - sys.argv or []) - return len(list(content) if not isinstance(content, list) else content) > 0 - - -def get_config_files(): - """Get the possible configuration files accepted by oslo.config - - This also includes the deprecated ones - """ - # default files - conf_files = cfg.find_config_files(project='monasca', - prog='monasca-log-api') - # deprecated config files (only used if standard config files are not there) - if len(conf_files) == 0: - old_conf_files = cfg.find_config_files(project='monasca', - prog='log-api') - if len(old_conf_files) > 0: - LOG.warning('Found deprecated old location "{}" ' - 'of main configuration file'.format(old_conf_files)) - conf_files += old_conf_files - return conf_files - - -def parse_args(argv=None): - global _CONF_LOADED - if _CONF_LOADED: - LOG.debug('Configuration has been already loaded') - return - - log.set_defaults() - log.register_options(CONF) - - argv = (argv if argv is not None else sys.argv[1:]) - args = ([] if _is_running_under_gunicorn() else argv or []) - - CONF(args=args, - prog=sys.argv[1:], - project='monasca', - version=version.version_str, - default_config_files=get_config_files(), - description='RESTful API to collect log files') - - log.setup(CONF, - product_name='monasca-log-api', - version=version.version_str) - - conf.register_opts() - policy_opts.set_defaults(CONF) - - _CONF_LOADED = True diff --git a/monasca_log_api/db/__init__.py b/monasca_log_api/db/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/db/common/__init__.py b/monasca_log_api/db/common/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/db/common/model.py b/monasca_log_api/db/common/model.py deleted file mode 100644 index 60c0cabd..00000000 --- a/monasca_log_api/db/common/model.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2017 StackHPC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections - - -class Dimension(collections.namedtuple('Dimension', 'name values')): - """Representation of dimension names and optional values list. - - Named-tuple type to represent the pairing of a dimension name and an - optional list of values. - - :ivar name: Name of the dimension to reference. - :ivar values: Optional list of values associated with the dimension. - - :vartype name: str - :vartype values: None or list[str] - """ - - -class SortBy(collections.namedtuple('SortBy', 'field direction')): - """Representation of an individual sorting directive. - - Named-tuple type to represent a directive for indicating how a result set - should be sorted. - - :ivar field: Name of the field which is provides the values to sort by. - :ivar direction: Either 'asc' or 'desc' specifying the order of values. - - :vartype name: str - :vartype values: str - """ diff --git a/monasca_log_api/db/repo/__init__.py b/monasca_log_api/db/repo/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/db/repo/logs_repository.py b/monasca_log_api/db/repo/logs_repository.py deleted file mode 100644 index 64e9936f..00000000 --- a/monasca_log_api/db/repo/logs_repository.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2017 StackHPC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc -import six - - -@six.add_metaclass(abc.ABCMeta) -class LogsRepository(object): - - def __init__(self): - super(LogsRepository, self).__init__() - - @abc.abstractmethod - def list_logs(self, tenant_id, dimensions, start_time, end_time, offset, - limit, sort_by): - """Obtain log listing based on simple criteria of dimension values. - - Performs queries on the underlying log storage against a time range and - set of dimension values. Additionally, it is possible to optionally - sort results by timestamp. - - :param tenant_id: - Tenant/project id for which to obtain logs (required). - :param dimensions: - List of Dimension tuples containing pairs of dimension names and - optional lists of dimension values. These will be used to filter - the logs returned. If no dimensions are specified, then no - filtering is performed. When multiple values are given, the - dimension must match any of the given values. If None is given, - logs with any value for the dimension will be returned. - :param start_time: - Optional starting time in UNIX time (seconds, inclusive). - :param end_time: - Optional ending time in UNIX time (seconds, inclusive). - :param offset: - Number of matching results to skip past, if specified. - :param limit: - Number of matching results to return (required). - :param sort_by: - List of SortBy tuples specifying fields to sort by and the - direction to sort the result set by. e.g. ('timestamp','asc'). The - direction is specified by either the string 'asc' for ascending - direction, or 'desc' for descending. If not specified, no order - must be enforced and the implementation is free to choose the most - efficient method to return the results. - - :type tenant_id: str - :type dimensions: None or list[Dimension[str, list[str] or None]] - :type start_time: None or int - :type end_time: None or int - :type offset: None or int - :type limit: int - :type sort_by: None or list[SortBy[str, str]] - - :return: - Log messages matching the given criteria. The dict representing - each message entry will contain attributes extracted from the - underlying structure; 'message', 'timestamp' and 'dimensions'. - - :rtype: list[dict] - """ - pass diff --git a/monasca_log_api/healthcheck/__init__.py b/monasca_log_api/healthcheck/__init__.py deleted file mode 100644 index 72fb7750..00000000 --- a/monasca_log_api/healthcheck/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Base package for monasca-log-api healthcheck""" diff --git a/monasca_log_api/healthcheck/kafka_check.py b/monasca_log_api/healthcheck/kafka_check.py deleted file mode 100644 index 71a23fc7..00000000 --- a/monasca_log_api/healthcheck/kafka_check.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2015-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections - -from monasca_common.kafka_lib import client -from oslo_log import log -from six import PY3 - -from monasca_log_api import conf - -LOG = log.getLogger(__name__) -CONF = conf.CONF - - -CheckResult = collections.namedtuple('CheckResult', ['healthy', 'message']) -"""Result from the healthcheck, contains healthy(boolean) and message""" - - -# TODO(feature) monasca-common candidate -class KafkaHealthCheck(object): - """Evaluates kafka health - - Healthcheck verifies if: - - * kafka server is up and running - * there is a configured topic in kafka - - If following conditions are met healthcheck returns healthy status. - Otherwise unhealthy status is returned with explanation. - - Example of middleware configuration: - - .. code-block:: ini - - [kafka_healthcheck] - kafka_url = localhost:8900 - kafka_topics = log - - Note: - It is possible to specify multiple topics if necessary. - Just separate them with , - - """ - - def healthcheck(self): - url = CONF.kafka_healthcheck.kafka_url - - try: - kafka_client = client.KafkaClient(hosts=url) - except client.KafkaUnavailableError as ex: - LOG.error(repr(ex)) - error_str = 'Could not connect to kafka at %s' % url - return CheckResult(healthy=False, message=error_str) - - result = self._verify_topics(kafka_client) - self._disconnect_gracefully(kafka_client) - - return result - - # noinspection PyMethodMayBeStatic - def _verify_topics(self, kafka_client): - topics = CONF.kafka_healthcheck.kafka_topics - - if PY3: - topics = tuple(topic.encode('utf-8') for topic in topics) - - for t in topics: - # kafka client loads metadata for topics as fast - # as possible (happens in __init__), therefore this - # topic_partitions is sure to be filled - for_topic = t in kafka_client.topic_partitions - if not for_topic: - error_str = 'Kafka: Topic %s not found' % t - LOG.error(error_str) - return CheckResult(healthy=False, message=error_str) - - return CheckResult(healthy=True, message='OK') - - # noinspection PyMethodMayBeStatic - def _disconnect_gracefully(self, kafka_client): - # at this point, client is connected so it must be closed - # regardless of topic existence - try: - kafka_client.close() - except Exception as ex: - # log that something went wrong and move on - LOG.error(repr(ex)) diff --git a/monasca_log_api/middleware/__init__.py b/monasca_log_api/middleware/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/middleware/role_middleware.py b/monasca_log_api/middleware/role_middleware.py deleted file mode 100644 index cac851a8..00000000 --- a/monasca_log_api/middleware/role_middleware.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2015-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_log import log -from oslo_middleware import base as om -from webob import response - -from monasca_log_api import conf - -CONF = conf.CONF -LOG = log.getLogger(__name__) - -_X_IDENTITY_STATUS = 'X-Identity-Status' -_X_ROLES = 'X-Roles' -_X_MONASCA_LOG_AGENT = 'X-MONASCA-LOG-AGENT' -_CONFIRMED_STATUS = 'Confirmed' - - -def _ensure_lower_roles(roles): - if not roles: - return [] - return [role.strip().lower() for role in roles] - - -def _intersect(a, b): - return list(set(a) & set(b)) - - -class RoleMiddleware(om.ConfigurableMiddleware): - """Authorization middleware for X-Roles header. - - RoleMiddleware is responsible for authorizing user's - access against **X-Roles** header. Middleware - expects authentication to be completed (i.e. keystone middleware - has been already called). - - If tenant is authenticated and authorized middleware - exits silently (that is considered a success). Otherwise - middleware produces JSON response according to following schema - - .. code-block:: javascript - - { - 'title': u'Unauthorized', - 'message': explanation (str) - } - - Configuration example - - .. code-block:: cfg - - [roles_middleware] - path = /v2.0/log - default_roles = monasca-user - agent_roles = monasca-log-agent - delegate_roles = admin - - Configuration explained: - - * path (list) - path (or list of paths) middleware should be applied - * agent_roles (list) - list of roles that identifies tenant as an agent - * default_roles (list) - list of roles that should be authorized - * delegate_roles (list) - list of roles that are allowed to POST logs on - behalf of another tenant (project) - - Note: - Being an agent means that tenant is automatically authorized. - Note: - Middleware works only for configured paths and for all - requests apart from HTTP method **OPTIONS**. - - """ - - def __init__(self, application, conf=None): - super(RoleMiddleware, self).__init__(application, conf) - middleware = CONF.roles_middleware - - self._path = middleware.path - self._default_roles = _ensure_lower_roles(middleware.default_roles) - self._agent_roles = _ensure_lower_roles(middleware.agent_roles) - - LOG.debug('RolesMiddleware initialized for paths=%s', self._path) - - def process_request(self, req): - if not self._can_apply_middleware(req): - LOG.debug('%s skipped in role middleware', req.path) - return None - - is_authenticated = self._is_authenticated(req) - is_agent = self._is_agent(req) - tenant_id = req.headers.get('X-Tenant-Id') - - req.environ[_X_MONASCA_LOG_AGENT] = is_agent - - LOG.debug('%s is authenticated=%s, log_agent=%s', - tenant_id, is_authenticated, is_agent) - - if is_authenticated: - LOG.debug('%s has been authenticated', tenant_id) - return # do return nothing to enter API internal - - explanation = u'Failed to authenticate request for %s' % tenant_id - LOG.error(explanation) - json_body = {u'title': u'Unauthorized', u'message': explanation} - return response.Response(status=401, - json_body=json_body, - content_type='application/json') - - def _is_agent(self, req): - headers = req.headers - roles = headers.get(_X_ROLES) - - if not roles: - LOG.warning('Couldn\'t locate %s header,or it was empty', _X_ROLES) - return False - else: - roles = _ensure_lower_roles(roles.split(',')) - - is_agent = len(_intersect(roles, self._agent_roles)) > 0 - - return is_agent - - def _is_authenticated(self, req): - headers = req.headers - if _X_IDENTITY_STATUS in headers: - status = req.headers.get(_X_IDENTITY_STATUS) - return _CONFIRMED_STATUS == status - return False - - def _can_apply_middleware(self, req): - path = req.path - method = req.method - - if method == 'OPTIONS': - return False - - if self._path: - for p in self._path: - if path.startswith(p): - return True - return False # if no configured paths, or nothing matches diff --git a/monasca_log_api/monitoring/__init__.py b/monasca_log_api/monitoring/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/monitoring/client.py b/monasca_log_api/monitoring/client.py deleted file mode 100644 index 0a9174b1..00000000 --- a/monasca_log_api/monitoring/client.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import monascastatsd - -from oslo_log import log - -from monasca_log_api import conf - -LOG = log.getLogger(__name__) -CONF = conf.CONF - -_CLIENT_NAME = 'monasca' -_DEFAULT_DIMENSIONS = { - 'service': 'monitoring', - 'component': 'monasca-log-api' -} - - -def get_client(dimensions=None): - """Creates statsd client - - Creates monasca-statsd client using configuration from - config file and supplied dimensions. - - Configuration is composed out of :: - - [monitoring] - statsd_host = 192.168.10.4 - statsd_port = 8125 - statsd_buffer = 50 - - Dimensions are appended to following dictionary :: - - { - 'service': 'monitoring', - 'component': 'monasca-log-api' - } - - Note: - Passed dimensions do not override those specified in - dictionary above - - :param dict dimensions: Optional dimensions - :return: statsd client - :rtype: monascastatsd.Client - """ - dims = _DEFAULT_DIMENSIONS.copy() - if dimensions: - for key, val in dimensions.items(): - if key not in _DEFAULT_DIMENSIONS: - dims[key] = val - else: - LOG.warning('Cannot override fixed dimension %s=%s', key, - _DEFAULT_DIMENSIONS[key]) - - connection = monascastatsd.Connection( - host=CONF.monitoring.statsd_host, - port=CONF.monitoring.statsd_port, - max_buffer_size=CONF.monitoring.statsd_buffer - ) - client = monascastatsd.Client(name=_CLIENT_NAME, - connection=connection, - dimensions=dims) - - LOG.debug('Created statsd client %s[%s] = %s:%d', _CLIENT_NAME, dims, - CONF.monitoring.statsd_host, CONF.monitoring.statsd_port) - - return client diff --git a/monasca_log_api/monitoring/metrics.py b/monasca_log_api/monitoring/metrics.py deleted file mode 100644 index 97e3677b..00000000 --- a/monasca_log_api/monitoring/metrics.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -LOGS_RECEIVED_METRIC = 'log.in_logs' -"""Metrics sent with amount of logs (not requests) API receives""" - -LOGS_REJECTED_METRIC = 'log.in_logs_rejected' -"""Metric sent with amount of logs that were rejected -(i.e. invalid dimension)""" - -LOGS_BULKS_REJECTED_METRIC = 'log.in_bulks_rejected' -"""Metric sent with amount of bulk packages that were rejected due -to early stage validation (content-length, content-type). -Only valid for v3.0. -""" - -LOGS_RECEIVED_BYTE_SIZE_METRICS = 'log.in_logs_bytes' -"""Metric sent with size of payloads(a.k.a. Content-Length) - (in bytes) API receives""" - -LOGS_PROCESSING_TIME_METRIC = 'log.processing_time_ms' -"""Metric sent with time that log-api needed to process each received log. -Metric does not include time needed to authorize requests.""" - -LOGS_PUBLISHED_METRIC = 'log.out_logs' -"""Metric sent with amount of logs published to kafka""" - -LOGS_PUBLISHED_LOST_METRIC = 'log.out_logs_lost' -"""Metric sent with amount of logs that were lost due to critical error in -publish phase.""" - -LOGS_PUBLISH_TIME_METRIC = 'log.publish_time_ms' -"""Metric sent with time that publishing took""" - -LOGS_TRUNCATED_METRIC = 'log.out_logs_truncated_bytes' -"""Metric sent with amount of truncated bytes from log message""" diff --git a/monasca_log_api/policies/__init__.py b/monasca_log_api/policies/__init__.py deleted file mode 100644 index f892a22a..00000000 --- a/monasca_log_api/policies/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# Copyright 2018 OP5 AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import pkgutil - -from oslo_config import cfg -from oslo_log import log -from oslo_utils import importutils - -from monasca_log_api.conf import role_middleware - -LOG = log.getLogger(__name__) -_BASE_MOD_PATH = 'monasca_log_api.policies.' -CONF = cfg.CONF - - -def roles_list_to_check_str(roles_list): - if roles_list: - converted_roles_list = ["role:" + role if role != '@' else role for role in roles_list] - return ' or '.join(converted_roles_list) - else: - return None - - -role_middleware.register_opts(CONF) - - -def load_policy_modules(): - """Load all modules that contain policies. - - Method iterates over modules of :py:mod:`monasca_events_api.policies` - and imports only those that contain following methods: - - - list_rules - - """ - for modname in _list_module_names(): - mod = importutils.import_module(_BASE_MOD_PATH + modname) - if hasattr(mod, 'list_rules'): - yield mod - - -def _list_module_names(): - package_path = os.path.dirname(os.path.abspath(__file__)) - for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]): - if not (modname == "opts" and ispkg): - yield modname - - -def list_rules(): - """List all policy modules rules. - - Goes through all policy modules and yields their rules - - """ - all_rules = [] - for mod in load_policy_modules(): - rules = mod.list_rules() - all_rules.extend(rules) - return all_rules diff --git a/monasca_log_api/policies/healthchecks.py b/monasca_log_api/policies/healthchecks.py deleted file mode 100644 index c5233d4b..00000000 --- a/monasca_log_api/policies/healthchecks.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2018 OP5 AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg -from oslo_policy import policy - -from monasca_log_api import policies - -CHECK_AUTHORIZED_ROLES = policies.roles_list_to_check_str( - cfg.CONF.roles_middleware.check_roles) - -rules = [ - policy.DocumentedRuleDefault( - name='log_api:healthcheck:head', - check_str=CHECK_AUTHORIZED_ROLES, - description='Healthcheck head rule', - operations=[ - {'path': '/healthcheck', 'method': 'HEAD'} - ] - ), - policy.DocumentedRuleDefault( - name='log_api:healthcheck:get', - check_str=CHECK_AUTHORIZED_ROLES, - description='Healthcheck get rule', - operations=[ - {'path': '/healthcheck', 'method': 'GET'} - ] - ), -] - - -def list_rules(): - return rules diff --git a/monasca_log_api/policies/logs.py b/monasca_log_api/policies/logs.py deleted file mode 100644 index a7332e38..00000000 --- a/monasca_log_api/policies/logs.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2018 OP5 AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg -from oslo_policy import policy - -from monasca_log_api import policies - -DEFAULT_AUTHORIZED_ROLES = policies.roles_list_to_check_str( - cfg.CONF.roles_middleware.default_roles) -AGENT_AUTHORIZED_ROLES = policies.roles_list_to_check_str( - cfg.CONF.roles_middleware.agent_roles) -DELEGATE_AUTHORIZED_ROLES = policies.roles_list_to_check_str( - cfg.CONF.roles_middleware.delegate_roles) - -rules = [ - policy.DocumentedRuleDefault( - name='log_api:logs:post', - check_str=' or '.join(filter(None, [AGENT_AUTHORIZED_ROLES, - DEFAULT_AUTHORIZED_ROLES, - DELEGATE_AUTHORIZED_ROLES])), - description='Logs post rule', - operations=[ - {'path': '/logs', 'method': 'POST'}, - {'path': '/log/single', 'method': 'POST'} - ] - ) -] - - -def list_rules(): - return rules diff --git a/monasca_log_api/policies/versions.py b/monasca_log_api/policies/versions.py deleted file mode 100644 index d0e7c9da..00000000 --- a/monasca_log_api/policies/versions.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2018 OP5 AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_config import cfg -from oslo_policy import policy - -from monasca_log_api import policies - -CHECK_AUTHORIZED_ROLES = policies.roles_list_to_check_str( - cfg.CONF.roles_middleware.check_roles) - -rules = [ - policy.DocumentedRuleDefault( - name='log_api:versions:get', - check_str=CHECK_AUTHORIZED_ROLES, - description='Versions get rule', - operations=[ - {'path': '/', 'method': 'GET'}, - {'path': '/version', 'method': 'GET'}, - {'path': '/version/{version_id}', 'method': 'GET'} - ] - ) -] - - -def list_rules(): - return rules diff --git a/monasca_log_api/tests/__init__.py b/monasca_log_api/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/monasca_log_api/tests/base.py b/monasca_log_api/tests/base.py deleted file mode 100644 index d348eafd..00000000 --- a/monasca_log_api/tests/base.py +++ /dev/null @@ -1,208 +0,0 @@ -# coding=utf-8 -# Copyright 2015 kornicameister@gmail.com -# Copyright 2015-2017 FUJITSU LIMITED -# Copyright 2018 OP5 AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import codecs -import os -import random -import string -from unittest import mock - -import falcon -from falcon import testing -import fixtures -from monasca_common.policy import policy_engine as policy -from oslo_config import fixture as oo_cfg -from oslo_context import fixture as oo_ctx -from oslo_serialization import jsonutils -from oslotest import base as oslotest_base -import six - -from monasca_log_api.app.base import request -from monasca_log_api import conf -from monasca_log_api import config -from monasca_log_api import policies - -policy.POLICIES = policies - - -def generate_unique_message(size): - letters = string.ascii_letters - - def rand(amount, space=True): - space = ' ' if space else '' - return ''.join((random.choice(letters + space) for _ in range(amount))) - - return rand(size) - - -def _hex_to_unicode(hex_raw): - hex_raw = six.b(hex_raw.replace(' ', '')) - hex_str_raw = codecs.getdecoder('hex')(hex_raw)[0] - hex_str = hex_str_raw.decode('utf-8', 'replace') - return hex_str - - -# NOTE(trebskit) => http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt -UNICODE_MESSAGES = [ - # Unicode is evil... - {'case': 'arabic', 'input': 'يونيكود هو الشر'}, - {'case': 'polish', 'input': 'Unicode to zło'}, - {'case': 'greek', 'input': 'Unicode είναι κακό'}, - {'case': 'portuguese', 'input': 'Unicode é malvado'}, - {'case': 'lao', 'input': 'unicode ເປັນຄວາມຊົ່ວຮ້າຍ'}, - {'case': 'german', 'input': 'Unicode ist böse'}, - {'case': 'japanese', 'input': 'ユニコードは悪です'}, - {'case': 'russian', 'input': 'Unicode - зло'}, - {'case': 'urdu', 'input': 'یونیسیڈ برائی ہے'}, - {'case': 'weird', 'input': '🆄🅽🅸🅲🅾🅳🅴 🅸🆂 🅴🆅🅸🅻...'}, # funky, huh ? - # conditions from link above - # 2.3 Other boundary conditions - {'case': 'stress_2_3_1', 'input': _hex_to_unicode('ed 9f bf')}, - {'case': 'stress_2_3_2', 'input': _hex_to_unicode('ee 80 80')}, - {'case': 'stress_2_3_3', 'input': _hex_to_unicode('ef bf bd')}, - {'case': 'stress_2_3_4', 'input': _hex_to_unicode('f4 8f bf bf')}, - {'case': 'stress_2_3_5', 'input': _hex_to_unicode('f4 90 80 80')}, - # 3.5 Impossible byes - {'case': 'stress_3_5_1', 'input': _hex_to_unicode('fe')}, - {'case': 'stress_3_5_2', 'input': _hex_to_unicode('ff')}, - {'case': 'stress_3_5_3', 'input': _hex_to_unicode('fe fe ff ff')}, - # 4.1 Examples of an overlong ASCII character - {'case': 'stress_4_1_1', 'input': _hex_to_unicode('c0 af')}, - {'case': 'stress_4_1_2', 'input': _hex_to_unicode('e0 80 af')}, - {'case': 'stress_4_1_3', 'input': _hex_to_unicode('f0 80 80 af')}, - {'case': 'stress_4_1_4', 'input': _hex_to_unicode('f8 80 80 80 af')}, - {'case': 'stress_4_1_5', 'input': _hex_to_unicode('fc 80 80 80 80 af')}, - # 4.2 Maximum overlong sequences - {'case': 'stress_4_2_1', 'input': _hex_to_unicode('c1 bf')}, - {'case': 'stress_4_2_2', 'input': _hex_to_unicode('e0 9f bf')}, - {'case': 'stress_4_2_3', 'input': _hex_to_unicode('f0 8f bf bf')}, - {'case': 'stress_4_2_4', 'input': _hex_to_unicode('f8 87 bf bf bf')}, - {'case': 'stress_4_2_5', 'input': _hex_to_unicode('fc 83 bf bf bf bf')}, - # 4.3 Overlong representation of the NUL character - {'case': 'stress_4_3_1', 'input': _hex_to_unicode('c0 80')}, - {'case': 'stress_4_3_2', 'input': _hex_to_unicode('e0 80 80')}, - {'case': 'stress_4_3_3', 'input': _hex_to_unicode('f0 80 80 80')}, - {'case': 'stress_4_3_4', 'input': _hex_to_unicode('f8 80 80 80 80')}, - {'case': 'stress_4_3_5', 'input': _hex_to_unicode('fc 80 80 80 80 80')}, - # and some cheesy example from polish novel 'Pan Tadeusz' - {'case': 'mr_t', 'input': 'Hajże na Soplicę!'}, - # it won't be complete without that one - {'case': 'mr_b', 'input': 'Grzegorz Brzęczyszczykiewicz, ' - 'Chrząszczyżewoszyce, powiat Łękołody'}, - # great success, christmas time - {'case': 'olaf', 'input': '☃'} -] - - -class DisableStatsdFixture(fixtures.Fixture): - - def setUp(self): - super(DisableStatsdFixture, self).setUp() - statsd_patch = mock.patch('monascastatsd.Connection') - statsd_patch.start() - self.addCleanup(statsd_patch.stop) - - -class ConfigFixture(oo_cfg.Config): - """Mocks configuration""" - - def __init__(self): - super(ConfigFixture, self).__init__(config.CONF) - - def setUp(self): - super(ConfigFixture, self).setUp() - self.addCleanup(self._clean_config_loaded_flag) - conf.register_opts() - self._set_defaults() - config.parse_args(argv=[]) # prevent oslo from parsing test args - - @staticmethod - def _clean_config_loaded_flag(): - config._CONF_LOADED = False - - def _set_defaults(self): - self.conf.set_default('kafka_url', '127.0.0.1', 'kafka_healthcheck') - self.conf.set_default('kafka_url', '127.0.0.1', 'log_publisher') - - -class PolicyFixture(fixtures.Fixture): - - """Override the policy with a completely new policy file. - - This overrides the policy with a completely fake and synthetic - policy file. - - """ - - def setUp(self): - super(PolicyFixture, self).setUp() - self._prepare_policy() - policy.reset() - policy.init() - - def _prepare_policy(self): - policy_dir = self.useFixture(fixtures.TempDir()) - policy_file = os.path.join(policy_dir.path, 'policy.yaml') - # load the fake_policy data and add the missing default rules. - policy_rules = jsonutils.loads('{}') - self.add_missing_default_rules(policy_rules) - with open(policy_file, 'w') as f: - jsonutils.dump(policy_rules, f) - - BaseTestCase.conf_override(policy_file=policy_file, group='oslo_policy') - BaseTestCase.conf_override(policy_dirs=[], group='oslo_policy') - - @staticmethod - def add_missing_default_rules(rules): - for rule in policies.list_rules(): - if rule.name not in rules: - rules[rule.name] = rule.check_str - - -class BaseTestCase(oslotest_base.BaseTestCase): - - def setUp(self): - super(BaseTestCase, self).setUp() - self.useFixture(ConfigFixture()) - self.useFixture(DisableStatsdFixture()) - self.useFixture(oo_ctx.ClearRequestContext()) - self.useFixture(PolicyFixture()) - - @staticmethod - def conf_override(**kw): - """Override flag variables for a test.""" - group = kw.pop('group', None) - for k, v in kw.items(): - config.CONF.set_override(k, v, group) - - @staticmethod - def conf_default(**kw): - """Override flag variables for a test.""" - group = kw.pop('group', None) - for k, v in kw.items(): - config.CONF.set_default(k, v, group) - - -class BaseApiTestCase(BaseTestCase, testing.TestCase): - - def setUp(self): - super(BaseApiTestCase, self).setUp() - self.app = falcon.API(request_type=request.Request) - # NOTE(czarneckia): Falcon 2.0.0 switches the default for this from True - # to False so we explicitly set it here to prevent the behaviour - # changing between versions. - self.app.req_options.strip_url_path_trailing_slash = True diff --git a/monasca_log_api/tests/test_config.py b/monasca_log_api/tests/test_config.py deleted file mode 100644 index 9868b66d..00000000 --- a/monasca_log_api/tests/test_config.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from unittest import mock - -from monasca_log_api import config -from monasca_log_api.tests import base - - -class TestConfig(base.BaseTestCase): - - @mock.patch('monasca_log_api.config.sys') - def test_should_return_true_if_runs_under_gunicorn(self, sys_patch): - sys_patch.argv = [ - '/bin/gunicorn', - '--capture-output', - '--paste', - 'etc/monasca/log-api-paste.ini', - '--workers', - '1' - ] - sys_patch.executable = '/bin/python' - self.assertTrue(config._is_running_under_gunicorn()) - - @mock.patch('monasca_log_api.config.sys') - def test_should_return_false_if_runs_without_gunicorn(self, sys_patch): - sys_patch.argv = ['/bin/monasca-log-api'] - sys_patch.executable = '/bin/python' - self.assertFalse(config._is_running_under_gunicorn()) diff --git a/monasca_log_api/tests/test_healthchecks.py b/monasca_log_api/tests/test_healthchecks.py deleted file mode 100644 index 04e8819f..00000000 --- a/monasca_log_api/tests/test_healthchecks.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon -from unittest import mock - -from monasca_log_api.app.controller import healthchecks -from monasca_log_api.healthcheck import kafka_check as healthcheck -from monasca_log_api.tests import base - -ENDPOINT = '/healthcheck' - - -class TestApiHealthChecks(base.BaseApiTestCase): - - def setUp(self): - super(TestApiHealthChecks, self).setUp() - self.resource = healthchecks.HealthChecks() - self.app.add_route( - ENDPOINT, - self.resource - ) - - def test_should_return_200_for_head(self): - res = self.simulate_request( - path=ENDPOINT, - method='HEAD') - self.assertEqual(falcon.HTTP_NO_CONTENT, res.status) - - @mock.patch('monasca_log_api.healthcheck.kafka_check.KafkaHealthCheck') - def test_should_report_healthy_if_kafka_healthy(self, kafka_check): - kafka_check.healthcheck.return_value = healthcheck.CheckResult(True, - 'OK') - self.resource._kafka_check = kafka_check - - res = self.simulate_request(path=ENDPOINT, - headers={ - 'Content-Type': 'application/json' - }, - method='GET') - self.assertEqual(falcon.HTTP_OK, res.status) - - res = res.json - self.assertIn('kafka', res) - self.assertEqual('OK', res.get('kafka')) - - @mock.patch('monasca_log_api.healthcheck.kafka_check.KafkaHealthCheck') - def test_should_report_unhealthy_if_kafka_healthy(self, kafka_check): - url = 'localhost:8200' - err_str = 'Could not connect to kafka at %s' % url - kafka_check.healthcheck.return_value = healthcheck.CheckResult(False, - err_str) - self.resource._kafka_check = kafka_check - - res = self.simulate_request(path=ENDPOINT, - headers={ - 'Content-Type': 'application/json' - }, - method='GET') - self.assertEqual(falcon.HTTP_SERVICE_UNAVAILABLE, res.status) - - res = res.json - self.assertIn('kafka', res) - self.assertEqual(err_str, res.get('kafka')) diff --git a/monasca_log_api/tests/test_kafka_check.py b/monasca_log_api/tests/test_kafka_check.py deleted file mode 100644 index 2d42868b..00000000 --- a/monasca_log_api/tests/test_kafka_check.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2015-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from unittest import mock - -from monasca_common.kafka_lib import client - -from monasca_log_api.healthcheck import kafka_check as kc -from monasca_log_api.tests import base - - -class KafkaCheckLogicTest(base.BaseTestCase): - - mock_kafka_url = 'localhost:1234' - mocked_topics = ['test_1', 'test_2'] - mock_config = { - 'kafka_url': mock_kafka_url, - 'kafka_topics': mocked_topics - } - - def setUp(self): - super(KafkaCheckLogicTest, self).setUp() - self.conf_default(group='kafka_healthcheck', **self.mock_config) - - @mock.patch('monasca_log_api.healthcheck.kafka_check.client.KafkaClient') - def test_should_fail_kafka_unavailable(self, kafka_client): - kafka_client.side_effect = client.KafkaUnavailableError() - kafka_health = kc.KafkaHealthCheck() - result = kafka_health.healthcheck() - - self.assertFalse(result.healthy) - - @mock.patch('monasca_log_api.healthcheck.kafka_check.client.KafkaClient') - def test_should_fail_topic_missing(self, kafka_client): - kafka = mock.Mock() - kafka.topic_partitions = [self.mocked_topics[0]] - kafka_client.return_value = kafka - - kafka_health = kc.KafkaHealthCheck() - result = kafka_health.healthcheck() - - # verify result - self.assertFalse(result.healthy) - - # ensure client was closed - self.assertTrue(kafka.close.called) - - @mock.patch('monasca_log_api.healthcheck.kafka_check.client.KafkaClient') - def test_should_pass(self, kafka_client): - kafka = mock.Mock() - kafka.topic_partitions = self.mocked_topics - kafka_client.return_value = kafka - - kafka_health = kc.KafkaHealthCheck() - result = kafka_health.healthcheck() - - self.assertTrue(result) - - # ensure client was closed - self.assertTrue(kafka.close.called) diff --git a/monasca_log_api/tests/test_log_publisher.py b/monasca_log_api/tests/test_log_publisher.py deleted file mode 100644 index f605d737..00000000 --- a/monasca_log_api/tests/test_log_publisher.py +++ /dev/null @@ -1,293 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import copy -import datetime -import random -import simplejson as json -import unittest -from unittest import mock - -from oslo_config import cfg -from oslo_log import log -import six - -from monasca_log_api.app.base import log_publisher -from monasca_log_api.app.base import model -from monasca_log_api.tests import base - -LOG = log.getLogger(__name__) -EPOCH_START = datetime.datetime(1970, 1, 1) - - -class TestSendMessage(base.BaseTestCase): - - @mock.patch('monasca_log_api.app.base.log_publisher.producer' - '.KafkaProducer') - def test_should_not_send_empty_message(self, _): - instance = log_publisher.LogPublisher() - - instance._kafka_publisher = mock.Mock() - instance.send_message({}) - - self.assertFalse(instance._kafka_publisher.publish.called) - - @unittest.expectedFailure - def test_should_not_send_message_not_dict(self): - instance = log_publisher.LogPublisher() - not_dict_value = 123 - instance.send_message(not_dict_value) - - @mock.patch('monasca_log_api.app.base.log_publisher.producer' - '.KafkaProducer') - def test_should_not_send_message_missing_keys(self, _): - # checks every combination of missing keys - # test does not rely on those keys having a value or not, - # it simply assumes that values are set but important - # message (i.e. envelope) properties are missing entirely - # that's why there are two loops instead of three - - instance = log_publisher.LogPublisher() - keys = ['log', 'creation_time', 'meta'] - - for key_1 in keys: - diff = keys[:] - diff.remove(key_1) - for key_2 in diff: - message = { - key_1: random.randint(10, 20), - key_2: random.randint(30, 50) - } - self.assertRaises(log_publisher.InvalidMessageException, - instance.send_message, - message) - - @mock.patch('monasca_log_api.app.base.log_publisher.producer' - '.KafkaProducer') - def test_should_not_send_message_missing_values(self, _): - # original message assumes that every property has value - # test modify each property one by one by removing that value - # (i.e. creating false-like value) - instance = log_publisher.LogPublisher() - message = { - 'log': { - 'message': '11' - }, - 'creation_time': 123456, - 'meta': { - 'region': 'pl' - } - } - - for key in message: - tmp_message = message - tmp_message[key] = None - self.assertRaises(log_publisher.InvalidMessageException, - instance.send_message, - tmp_message) - - @mock.patch('monasca_log_api.app.base.log_publisher.producer' - '.KafkaProducer') - def test_should_send_message(self, kafka_producer): - instance = log_publisher.LogPublisher() - instance._kafka_publisher = kafka_producer - instance.send_message({}) - - creation_time = ((datetime.datetime.utcnow() - EPOCH_START) - .total_seconds()) - application_type = 'monasca-log-api' - dimension_1_name = 'disk_usage' - dimension_1_value = '50' - dimension_2_name = 'cpu_time' - dimension_2_value = '60' - - msg = model.Envelope( - log={ - 'message': '1', - 'application_type': application_type, - 'dimensions': { - dimension_1_name: dimension_1_value, - dimension_2_name: dimension_2_value - } - }, - meta={ - 'tenantId': '1' - } - ) - msg['creation_time'] = creation_time - instance.send_message(msg) - - instance._kafka_publisher.publish.assert_called_once_with( - cfg.CONF.log_publisher.topics[0], - [json.dumps(msg, ensure_ascii=False).encode('utf-8')]) - - @mock.patch('monasca_log_api.app.base.log_publisher.producer' - '.KafkaProducer') - def test_should_send_message_multiple_topics(self, _): - topics = ['logs', 'analyzer', 'tester'] - self.conf_override(topics=topics, - max_message_size=5000, - group='log_publisher') - - instance = log_publisher.LogPublisher() - instance._kafka_publisher = mock.Mock() - instance.send_message({}) - - creation_time = ((datetime.datetime.utcnow() - EPOCH_START) - .total_seconds()) - dimension_1_name = 'disk_usage' - dimension_1_value = '50' - dimension_2_name = 'cpu_time' - dimension_2_value = '60' - application_type = 'monasca-log-api' - msg = model.Envelope( - log={ - 'message': '1', - 'application_type': application_type, - 'dimensions': { - dimension_1_name: dimension_1_value, - dimension_2_name: dimension_2_value - } - }, - meta={ - 'tenantId': '1' - } - ) - msg['creation_time'] = creation_time - json_msg = json.dumps(msg, ensure_ascii=False) - - instance.send_message(msg) - - self.assertEqual(len(topics), - instance._kafka_publisher.publish.call_count) - for topic in topics: - instance._kafka_publisher.publish.assert_any_call( - topic, - [json_msg.encode('utf-8')]) - - @mock.patch('monasca_log_api.app.base.log_publisher.producer' - '.KafkaProducer') - def test_should_send_unicode_message(self, kp): - instance = log_publisher.LogPublisher() - instance._kafka_publisher = kp - - for um in base.UNICODE_MESSAGES: - case, msg = um.values() - try: - envelope = model.Envelope( - log={ - 'message': msg, - 'application_type': 'test', - 'dimensions': { - 'test': 'test_log_publisher', - 'case': 'test_should_send_unicode_message' - } - }, - meta={ - 'tenantId': 1 - } - ) - instance.send_message(envelope) - - expected_message = json.dumps(envelope, ensure_ascii=False) - - if six.PY3: - expected_message = expected_message.encode('utf-8') - - instance._kafka_publisher.publish.assert_called_with( - cfg.CONF.log_publisher.topics[0], - [expected_message] - ) - except Exception: - LOG.exception('Failed to evaluate unicode case %s', case) - raise - - -@mock.patch( - 'monasca_log_api.app.base.log_publisher.producer' - '.KafkaProducer') -class TestTruncation(base.BaseTestCase): - EXTRA_CHARS_SIZE = len(bytearray(json.dumps({ - 'log': { - 'message': None - } - }), 'utf8')) - 2 - - def test_should_not_truncate_message_if_size_is_smaller(self, _): - diff_size = random.randint(1, 100) - self._run_truncate_test(log_size_factor=-diff_size, - truncate_by=0) - - def test_should_not_truncate_message_if_size_equal_to_max(self, _): - self._run_truncate_test(log_size_factor=0, - truncate_by=0) - - def test_should_truncate_too_big_message(self, _): - diff_size = random.randint(1, 100) - max_size = 1000 - truncate_by = ((max_size - - (max_size - log_publisher._TRUNCATED_PROPERTY_SIZE)) + - log_publisher._TRUNCATION_SAFE_OFFSET + diff_size) - self._run_truncate_test(max_message_size=1000, - log_size_factor=diff_size, - truncate_by=truncate_by) - - def _run_truncate_test(self, - max_message_size=1000, - log_size_factor=0, - truncate_by=0, - gen_fn=base.generate_unique_message): - - log_size = (max_message_size - - TestTruncation.EXTRA_CHARS_SIZE - - log_publisher._KAFKA_META_DATA_SIZE - - log_publisher._TIMESTAMP_KEY_SIZE + - log_size_factor) - - expected_log_message_size = log_size - truncate_by - - self.conf_override( - group='log_publisher', - max_message_size=max_message_size - ) - - log_msg = gen_fn(log_size) - envelope = { - 'log': { - 'message': log_msg - } - } - - instance = log_publisher.LogPublisher() - instance._logs_truncated_gauge.send = meter = mock.Mock() - - envelope_copy = copy.deepcopy(envelope) - json_envelope = instance._truncate(envelope_copy) - - parsed_envelope = json.loads(json_envelope) - - parsed_log_message = parsed_envelope['log']['message'] - parsed_log_message_len = len(parsed_log_message) - - if truncate_by > 0: - self.assertNotEqual(envelope['log']['message'], - parsed_log_message) - else: - self.assertEqual(envelope['log']['message'], - parsed_log_message) - - self.assertEqual(expected_log_message_size, parsed_log_message_len) - self.assertEqual(1, meter.call_count) - self.assertEqual(truncate_by, meter.mock_calls[0][2]['value']) diff --git a/monasca_log_api/tests/test_logs.py b/monasca_log_api/tests/test_logs.py deleted file mode 100644 index 4b38c436..00000000 --- a/monasca_log_api/tests/test_logs.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import simplejson as json -from unittest import mock - -import falcon - -from monasca_log_api.app.base import exceptions as log_api_exceptions -from monasca_log_api.app.controller.api import headers -from monasca_log_api.app.controller.v2 import logs -from monasca_log_api.tests import base - -ROLES = 'admin' - - -def _init_resource(test): - resource = logs.Logs() - test.app.add_route('/log/single', resource) - return resource - - -class TestApiLogsVersion(base.BaseApiTestCase): - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - def test_should_return_v2_as_version(self, _, __): - logs_resource = logs.Logs() - self.assertEqual('v2.0', logs_resource.version) - - -class TestApiLogs(base.BaseApiTestCase): - - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - def test_should_contain_deprecated_details_in_successful_response(self, - _, - __): - _init_resource(self) - - res = self.simulate_request( - path='/log/single', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_DIMENSIONS.name: 'a:1', - 'Content-Type': 'application/json', - 'Content-Length': '0' - } - ) - - self.assertEqual(falcon.HTTP_204, res.status) - self.assertIn('deprecated', res.headers) - self.assertIn('link', res.headers) - - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - def test_should_fail_not_delegate_ok_cross_tenant_id(self, _, __): - _init_resource(self) - res = self.simulate_request( - path='/log/single', - method='POST', - query_string='tenant_id=1', - headers={ - 'Content-Type': 'application/json', - 'Content-Length': '0' - } - ) - self.assertEqual(falcon.HTTP_401, res.status) - - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - def test_should_pass_empty_cross_tenant_id_wrong_role(self, - log_creator, - kafka_publisher): - logs_resource = _init_resource(self) - logs_resource._log_creator = log_creator - logs_resource._kafka_publisher = kafka_publisher - - res = self.simulate_request( - path='/log/single', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_DIMENSIONS.name: 'a:1', - 'Content-Type': 'application/json', - 'Content-Length': '0' - } - ) - self.assertEqual(falcon.HTTP_204, res.status) - - self.assertEqual(1, kafka_publisher.send_message.call_count) - self.assertEqual(1, log_creator.new_log.call_count) - self.assertEqual(1, log_creator.new_log_envelope.call_count) - - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - def test_should_pass_empty_cross_tenant_id_ok_role(self, - log_creator, - kafka_publisher): - logs_resource = _init_resource(self) - logs_resource._log_creator = log_creator - logs_resource._kafka_publisher = kafka_publisher - - res = self.simulate_request( - path='/log/single', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_DIMENSIONS.name: 'a:1', - 'Content-Type': 'application/json', - 'Content-Length': '0' - } - ) - self.assertEqual(falcon.HTTP_204, res.status) - - self.assertEqual(1, kafka_publisher.send_message.call_count) - self.assertEqual(1, log_creator.new_log.call_count) - self.assertEqual(1, log_creator.new_log_envelope.call_count) - - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - def test_should_pass_delegate_cross_tenant_id_ok_role(self, - log_creator, - log_publisher): - resource = _init_resource(self) - resource._log_creator = log_creator - resource._kafka_publisher = log_publisher - - res = self.simulate_request( - path='/log/single', - method='POST', - query_string='tenant_id=1', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_DIMENSIONS.name: 'a:1', - 'Content-Type': 'application/json', - 'Content-Length': '0' - } - ) - self.assertEqual(falcon.HTTP_204, res.status) - - self.assertEqual(1, log_publisher.send_message.call_count) - self.assertEqual(1, log_creator.new_log.call_count) - self.assertEqual(1, log_creator.new_log_envelope.call_count) - - @mock.patch('monasca_log_api.common.rest.utils') - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - def test_should_fail_empty_dimensions_delegate(self, _, rest_utils): - _init_resource(self) - rest_utils.read_body.return_value = True - - res = self.simulate_request( - path='/log/single', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_DIMENSIONS.name: '', - 'Content-Type': 'application/json', - }, - body='{"message":"test"}' - ) - self.assertEqual(log_api_exceptions.HTTP_422, res.status) - - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - def test_should_fail_for_invalid_content_type(self, _, __): - _init_resource(self) - - res = self.simulate_request( - path='/log/single', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_DIMENSIONS.name: '', - 'Content-Type': 'video/3gpp', - 'Content-Length': '0' - } - ) - self.assertEqual(falcon.HTTP_415, res.status) - - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - def test_should_pass_payload_size_not_exceeded(self, _, __): - _init_resource(self) - - max_log_size = 1000 - body = json.dumps({ - 'message': 't' * (max_log_size - 100) - }) - - content_length = len(body) - self.conf_override(max_log_size=max_log_size, group='service') - - res = self.simulate_request( - path='/log/single', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_DIMENSIONS.name: '', - 'Content-Type': 'application/json', - 'Content-Length': str(content_length) - }, - body=body - ) - self.assertEqual(falcon.HTTP_204, res.status) - - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - def test_should_fail_payload_size_exceeded(self, _, __): - _init_resource(self) - - max_log_size = 1000 - content_length = max_log_size + 100 - self.conf_override(max_log_size=max_log_size, group='service') - - res = self.simulate_request( - path='/log/single', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_DIMENSIONS.name: '', - 'Content-Type': 'application/json', - 'Content-Length': str(content_length) - } - ) - self.assertEqual(falcon.HTTP_413, res.status) - - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - def test_should_fail_payload_size_equal(self, _, __): - _init_resource(self) - - max_log_size = 1000 - content_length = max_log_size - self.conf_override(max_log_size=max_log_size, group='service') - - res = self.simulate_request( - path='/log/single', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_DIMENSIONS.name: '', - 'Content-Type': 'application/json', - 'Content-Length': str(content_length) - } - ) - self.assertEqual(falcon.HTTP_413, res.status) - - @mock.patch('monasca_log_api.app.controller.v2.aid.service.LogCreator') - @mock.patch('monasca_log_api.app.base.log_publisher.LogPublisher') - def test_should_fail_content_length(self, _, __): - _init_resource(self) - - res = self.simulate_request( - path='/log/single', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_DIMENSIONS.name: '', - 'Content-Type': 'application/json' - } - ) - self.assertEqual(falcon.HTTP_411, res.status) diff --git a/monasca_log_api/tests/test_logs_v3.py b/monasca_log_api/tests/test_logs_v3.py deleted file mode 100644 index 710aa5c0..00000000 --- a/monasca_log_api/tests/test_logs_v3.py +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon -import simplejson as json -from unittest import mock - -from six import PY3 - -from monasca_log_api.app.base import exceptions as log_api_exceptions -from monasca_log_api.app.controller.api import headers -from monasca_log_api.app.controller.v3 import logs -from monasca_log_api.tests import base - -ENDPOINT = '/logs' -TENANT_ID = 'bob' -ROLES = 'admin' - - -def _init_resource(test): - resource = logs.Logs() - test.app.add_route(ENDPOINT, resource) - return resource - - -def _generate_v3_payload(log_count=None, messages=None): - if not log_count and messages: - log_count = len(messages) - v3_logs = [{ - 'message': messages[it], - 'dimensions': { - 'hostname': 'host_%d' % it, - 'component': 'component_%d' % it, - 'service': 'service_%d' % it - } - } for it in range(log_count)] - else: - v3_logs = [{ - 'message': base.generate_unique_message(100), - 'dimensions': { - 'hostname': 'host_%d' % it, - 'component': 'component_%d' % it, - 'service': 'service_%d' % it - } - } for it in range(log_count)] - v3_body = { - 'dimensions': { - 'origin': __name__ - }, - 'logs': v3_logs - } - - return v3_body, v3_logs - - -class TestApiLogsVersion(base.BaseApiTestCase): - - @mock.patch('monasca_log_api.app.controller.v3.aid' - '.bulk_processor.BulkProcessor') - def test_should_return_v3_as_version(self, _): - logs_resource = logs.Logs() - self.assertEqual('v3.0', logs_resource.version) - - -@mock.patch('monasca_log_api.app.base.log_publisher.producer.KafkaProducer') -@mock.patch('monasca_log_api.monitoring.client.monascastatsd.Connection') -class TestApiLogsMonitoring(base.BaseApiTestCase): - - def test_monitor_bulk_rejected(self, __, _): - res = _init_resource(self) - - in_counter = res._logs_in_counter.increment = mock.Mock() - bulk_counter = res._bulks_rejected_counter.increment = mock.Mock() - rejected_counter = res._logs_rejected_counter.increment = mock.Mock() - size_gauge = res._logs_size_gauge.send = mock.Mock() - - res._get_logs = mock.Mock( - side_effect=log_api_exceptions.HTTPUnprocessableEntity('')) - - log_count = 1 - v3_body, _ = _generate_v3_payload(log_count) - payload = json.dumps(v3_body) - content_length = len(payload) - - self.simulate_request( - path=ENDPOINT, - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_TENANT_ID.name: TENANT_ID, - 'Content-Type': 'application/json', - 'Content-Length': str(content_length) - }, - body=payload - ) - - self.assertEqual(1, bulk_counter.call_count) - self.assertEqual(0, in_counter.call_count) - self.assertEqual(0, rejected_counter.call_count) - self.assertEqual(0, size_gauge.call_count) - - def test_monitor_not_all_logs_ok(self, __, _): - res = _init_resource(self) - - in_counter = res._logs_in_counter.increment = mock.Mock() - bulk_counter = res._bulks_rejected_counter.increment = mock.Mock() - rejected_counter = res._logs_rejected_counter.increment = mock.Mock() - size_gauge = res._logs_size_gauge.send = mock.Mock() - - log_count = 5 - reject_logs = 1 - v3_body, _ = _generate_v3_payload(log_count) - payload = json.dumps(v3_body) - content_length = len(payload) - - side_effects = [{} for ___ in range(log_count - reject_logs)] - side_effects.append(log_api_exceptions.HTTPUnprocessableEntity('')) - - res._processor._get_dimensions = mock.Mock(side_effect=side_effects) - - self.simulate_request( - path=ENDPOINT, - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_TENANT_ID.name: TENANT_ID, - 'Content-Type': 'application/json', - 'Content-Length': str(content_length) - }, - body=payload - ) - - self.assertEqual(1, bulk_counter.call_count) - self.assertEqual(0, - bulk_counter.mock_calls[0][2]['value']) - - self.assertEqual(1, in_counter.call_count) - self.assertEqual(log_count - reject_logs, - in_counter.mock_calls[0][2]['value']) - - self.assertEqual(1, rejected_counter.call_count) - self.assertEqual(reject_logs, - rejected_counter.mock_calls[0][2]['value']) - - self.assertEqual(1, size_gauge.call_count) - self.assertEqual(content_length, - size_gauge.mock_calls[0][2]['value']) - - def test_monitor_all_logs_ok(self, __, _): - res = _init_resource(self) - - in_counter = res._logs_in_counter.increment = mock.Mock() - bulk_counter = res._bulks_rejected_counter.increment = mock.Mock() - rejected_counter = res._logs_rejected_counter.increment = mock.Mock() - size_gauge = res._logs_size_gauge.send = mock.Mock() - - res._send_logs = mock.Mock() - - log_count = 10 - - v3_body, _ = _generate_v3_payload(log_count) - - payload = json.dumps(v3_body) - content_length = len(payload) - self.simulate_request( - path=ENDPOINT, - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - headers.X_TENANT_ID.name: TENANT_ID, - 'Content-Type': 'application/json', - 'Content-Length': str(content_length) - }, - body=payload - ) - - self.assertEqual(1, bulk_counter.call_count) - self.assertEqual(0, - bulk_counter.mock_calls[0][2]['value']) - - self.assertEqual(1, in_counter.call_count) - self.assertEqual(log_count, - in_counter.mock_calls[0][2]['value']) - - self.assertEqual(1, rejected_counter.call_count) - self.assertEqual(0, - rejected_counter.mock_calls[0][2]['value']) - - self.assertEqual(1, size_gauge.call_count) - self.assertEqual(content_length, - size_gauge.mock_calls[0][2]['value']) - - -class TestApiLogs(base.BaseApiTestCase): - - @mock.patch('monasca_log_api.app.controller.v3.aid.bulk_processor.' - 'BulkProcessor') - def test_should_pass_cross_tenant_id(self, bulk_processor): - logs_resource = _init_resource(self) - logs_resource._processor = bulk_processor - - v3_body, v3_logs = _generate_v3_payload(1) - payload = json.dumps(v3_body) - content_length = len(payload) - res = self.simulate_request( - path='/logs', - method='POST', - query_string='tenant_id=1', - headers={ - headers.X_ROLES.name: ROLES, - 'Content-Type': 'application/json', - 'Content-Length': str(content_length) - }, - body=payload - ) - self.assertEqual(falcon.HTTP_204, res.status) - logs_resource._processor.send_message.assert_called_with( - logs=v3_logs, - global_dimensions=v3_body['dimensions'], - log_tenant_id='1') - - @mock.patch('monasca_log_api.app.controller.v3.aid.bulk_processor.' - 'BulkProcessor') - def test_should_fail_not_delegate_ok_cross_tenant_id(self, _): - _init_resource(self) - res = self.simulate_request( - path='/logs', - method='POST', - query_string='tenant_id=1', - headers={ - headers.X_ROLES.name: ROLES, - 'Content-Type': 'application/json', - 'Content-Length': '0' - } - ) - self.assertEqual(falcon.HTTP_400, res.status) - - @mock.patch('monasca_log_api.app.controller.v3.aid.bulk_processor.' - 'BulkProcessor') - def test_should_pass_empty_cross_tenant_id_wrong_role(self, - bulk_processor): - logs_resource = _init_resource(self) - logs_resource._processor = bulk_processor - - v3_body, _ = _generate_v3_payload(1) - payload = json.dumps(v3_body) - content_length = len(payload) - res = self.simulate_request( - path='/logs', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - 'Content-Type': 'application/json', - 'Content-Length': str(content_length) - }, - body=payload - ) - self.assertEqual(falcon.HTTP_204, res.status) - self.assertEqual(1, bulk_processor.send_message.call_count) - - @mock.patch('monasca_log_api.app.controller.v3.aid.bulk_processor.' - 'BulkProcessor') - def test_should_pass_empty_cross_tenant_id_ok_role(self, - bulk_processor): - logs_resource = _init_resource(self) - logs_resource._processor = bulk_processor - - v3_body, _ = _generate_v3_payload(1) - payload = json.dumps(v3_body) - content_length = len(payload) - res = self.simulate_request( - path='/logs', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - 'Content-Type': 'application/json', - 'Content-Length': str(content_length) - }, - body=payload - ) - self.assertEqual(falcon.HTTP_204, res.status) - self.assertEqual(1, bulk_processor.send_message.call_count) - - -class TestUnicodeLogs(base.BaseApiTestCase): - - @mock.patch('monasca_log_api.app.base.log_publisher.producer.' - 'KafkaProducer') - def test_should_send_unicode_messages(self, _): - _init_resource(self) - - messages = [m['input'] for m in base.UNICODE_MESSAGES] - v3_body, _ = _generate_v3_payload(messages=messages) - payload = json.dumps(v3_body, ensure_ascii=False) - content_length = len(payload.encode('utf8') if PY3 else payload) - res = self.simulate_request( - path='/logs', - method='POST', - headers={ - headers.X_ROLES.name: ROLES, - 'Content-Type': 'application/json', - 'Content-Length': str(content_length) - }, - body=payload - ) - self.assertEqual(falcon.HTTP_204, res.status) diff --git a/monasca_log_api/tests/test_monitoring.py b/monasca_log_api/tests/test_monitoring.py deleted file mode 100644 index 7ddef652..00000000 --- a/monasca_log_api/tests/test_monitoring.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from unittest import mock - -from monasca_log_api.monitoring import client -from monasca_log_api.tests import base - - -class TestMonitoring(base.BaseTestCase): - - @mock.patch('monasca_log_api.monitoring.client.monascastatsd') - def test_should_use_default_dimensions_if_none_specified(self, - monascastatsd): - client.get_client() - - statsd_client = monascastatsd.Client - - expected_dimensions = client._DEFAULT_DIMENSIONS - actual_dimensions = statsd_client.call_args[1]['dimensions'] - - self.assertEqual(1, statsd_client.call_count) - self.assertEqual(expected_dimensions, actual_dimensions) - - @mock.patch('monasca_log_api.monitoring.client.monascastatsd') - def test_should_not_override_fixed_dimensions(self, - monascastatsd): - dims = { - 'service': 'foo', - 'component': 'bar' - } - - client.get_client(dims) - - statsd_client = monascastatsd.Client - - expected_dimensions = client._DEFAULT_DIMENSIONS - actual_dimensions = statsd_client.call_args[1]['dimensions'] - - self.assertEqual(1, statsd_client.call_count) - self.assertEqual(expected_dimensions, actual_dimensions) diff --git a/monasca_log_api/tests/test_policy.py b/monasca_log_api/tests/test_policy.py deleted file mode 100644 index 37d5e4e1..00000000 --- a/monasca_log_api/tests/test_policy.py +++ /dev/null @@ -1,214 +0,0 @@ -# Copyright 2016-2017 FUJITSU LIMITED -# Copyright 2018 OP5 AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from falcon import testing - -from monasca_common.policy import policy_engine as policy -from oslo_context import context -from oslo_policy import policy as os_policy - -from monasca_log_api.app.base import request -from monasca_log_api.policies import roles_list_to_check_str -from monasca_log_api.tests import base - - -class TestPolicyFileCase(base.BaseTestCase): - def setUp(self): - super(TestPolicyFileCase, self).setUp() - self.context = context.RequestContext(user='fake', - tenant='fake', - roles=['fake']) - self.target = {'tenant_id': 'fake'} - - def test_modified_policy_reloads(self): - tmp_file = \ - self.create_tempfiles(files=[('policies', '{}')], ext='.yaml')[0] - base.BaseTestCase.conf_override(policy_file=tmp_file, - group='oslo_policy') - - policy.reset() - policy.init() - action = 'example:test' - rule = os_policy.RuleDefault(action, '') - policy._ENFORCER.register_defaults([rule]) - - with open(tmp_file, 'w') as policy_file: - policy_file.write('{"example:test": ""}') - policy.authorize(self.context, action, self.target) - - with open(tmp_file, 'w') as policy_file: - policy_file.write('{"example:test": "!"}') - policy._ENFORCER.load_rules(True) - self.assertRaises(os_policy.PolicyNotAuthorized, policy.authorize, - self.context, action, self.target) - - -class TestPolicyCase(base.BaseTestCase): - def setUp(self): - super(TestPolicyCase, self).setUp() - rules = [ - os_policy.RuleDefault("true", "@"), - os_policy.RuleDefault("example:allowed", "@"), - os_policy.RuleDefault("example:denied", "!"), - os_policy.RuleDefault("example:lowercase_monasca_user", - "role:monasca_user or role:sysadmin"), - os_policy.RuleDefault("example:uppercase_monasca_user", - "role:MONASCA_USER or role:sysadmin"), - ] - policy.reset() - policy.init() - policy._ENFORCER.register_defaults(rules) - - def test_authorize_nonexist_action_throws(self): - action = "example:noexist" - ctx = request.Request( - testing.create_environ( - path="/", - headers={ - "X_USER_ID": "fake", - "X_PROJECT_ID": "fake", - "X_ROLES": "member" - } - ) - ) - self.assertRaises(os_policy.PolicyNotRegistered, policy.authorize, - ctx.context, action, {}) - - def test_authorize_bad_action_throws(self): - action = "example:denied" - ctx = request.Request( - testing.create_environ( - path="/", - headers={ - "X_USER_ID": "fake", - "X_PROJECT_ID": "fake", - "X_ROLES": "member" - } - ) - ) - self.assertRaises(os_policy.PolicyNotAuthorized, policy.authorize, - ctx.context, action, {}) - - def test_authorize_bad_action_no_exception(self): - action = "example:denied" - ctx = request.Request( - testing.create_environ( - path="/", - headers={ - "X_USER_ID": "fake", - "X_PROJECT_ID": "fake", - "X_ROLES": "member" - } - ) - ) - result = policy.authorize(ctx.context, action, {}, False) - self.assertFalse(result) - - def test_authorize_good_action(self): - action = "example:allowed" - ctx = request.Request( - testing.create_environ( - path="/", - headers={ - "X_USER_ID": "fake", - "X_PROJECT_ID": "fake", - "X_ROLES": "member" - } - ) - ) - result = policy.authorize(ctx.context, action, {}, False) - self.assertTrue(result) - - def test_ignore_case_role_check(self): - lowercase_action = "example:lowercase_monasca_user" - uppercase_action = "example:uppercase_monasca_user" - - monasca_user_context = request.Request( - testing.create_environ( - path="/", - headers={ - "X_USER_ID": "monasca_user", - "X_PROJECT_ID": "fake", - "X_ROLES": "MONASCA_user" - } - ) - ) - self.assertTrue(policy.authorize(monasca_user_context.context, - lowercase_action, - {})) - self.assertTrue(policy.authorize(monasca_user_context.context, - uppercase_action, - {})) - - -class RegisteredPoliciesTestCase(base.BaseTestCase): - def __init__(self, *args, **kwds): - super(RegisteredPoliciesTestCase, self).__init__(*args, **kwds) - self.default_roles = ['monasca-user', 'admin'] - - def test_healthchecks_policies_roles(self): - healthcheck_policies = { - 'log_api:healthcheck:head': ['any_role'], - 'log_api:healthcheck:get': ['any_role'] - } - - self._assert_rules(healthcheck_policies) - - def test_versions_policies_roles(self): - versions_policies = { - 'log_api:versions:get': ['any_role'] - } - - self._assert_rules(versions_policies) - - def test_logs_policies_roles(self): - - logs_policies = { - 'log_api:logs:post': self.default_roles - } - - self._assert_rules(logs_policies) - - def _assert_rules(self, policies_list): - for policy_name in policies_list: - registered_rule = policy.get_rules()[policy_name] - if hasattr(registered_rule, 'rules'): - self.assertEqual(len(registered_rule.rules), - len(policies_list[policy_name])) - for role in policies_list[policy_name]: - ctx = self._get_request_context(role) - self.assertTrue(policy.authorize(ctx.context, - policy_name, - {}) - ) - - @staticmethod - def _get_request_context(role): - return request.Request( - testing.create_environ( - path='/', - headers={'X_ROLES': role} - ) - ) - - -class PolicyUtilsTestCase(base.BaseTestCase): - def test_roles_list_to_check_str(self): - self.assertEqual(roles_list_to_check_str(['test_role']), 'role:test_role') - self.assertEqual(roles_list_to_check_str(['role1', 'role2', 'role3']), - 'role:role1 or role:role2 or role:role3') - self.assertEqual(roles_list_to_check_str(['@']), '@') - self.assertEqual(roles_list_to_check_str(['role1', '@', 'role2']), - 'role:role1 or @ or role:role2') - self.assertIsNone(roles_list_to_check_str(None)) diff --git a/monasca_log_api/tests/test_request.py b/monasca_log_api/tests/test_request.py deleted file mode 100644 index 0467fee9..00000000 --- a/monasca_log_api/tests/test_request.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from unittest import mock - -from falcon import testing - -from monasca_log_api.app.base import request -from monasca_log_api.app.base import validation -from monasca_log_api.tests import base - - -class TestRequest(base.BaseTestCase): - - def test_use_context_from_request(self): - req = request.Request( - testing.create_environ( - path='/', - headers={ - 'X_AUTH_TOKEN': '111', - 'X_USER_ID': '222', - 'X_PROJECT_ID': '333', - 'X_ROLES': 'terminator,predator' - } - ) - ) - - self.assertEqual('111', req.context.auth_token) - self.assertEqual('222', req.user_id) - self.assertEqual('333', req.project_id) - self.assertEqual(['terminator', 'predator'], req.roles) - - def test_validate_context_type(self): - with mock.patch.object(validation, - 'validate_content_type') as vc_type, \ - mock.patch.object(validation, - 'validate_payload_size') as vp_size, \ - mock.patch.object(validation, - 'validate_cross_tenant') as vc_tenant: - req = request.Request(testing.create_environ()) - vc_type.side_effect = Exception() - - try: - req.validate(['test']) - except Exception as ex: - self.assertEqual(1, vc_type.call_count) - self.assertEqual(0, vp_size.call_count) - self.assertEqual(0, vc_tenant.call_count) - - self.assertIsInstance(ex, Exception) - - def test_validate_payload_size(self): - with mock.patch.object(validation, - 'validate_content_type') as vc_type, \ - mock.patch.object(validation, - 'validate_payload_size') as vp_size, \ - mock.patch.object(validation, - 'validate_cross_tenant') as vc_tenant: - - req = request.Request(testing.create_environ()) - vp_size.side_effect = Exception() - - try: - req.validate(['test']) - except Exception as ex: - self.assertEqual(1, vc_type.call_count) - self.assertEqual(1, vp_size.call_count) - self.assertEqual(0, vc_tenant.call_count) - - self.assertIsInstance(ex, Exception) - - def test_validate_cross_tenant(self): - with mock.patch.object(validation, - 'validate_content_type') as vc_type, \ - mock.patch.object(validation, - 'validate_payload_size') as vp_size, \ - mock.patch.object(validation, - 'validate_cross_tenant') as vc_tenant: - - req = request.Request(testing.create_environ()) - vc_tenant.side_effect = Exception() - - try: - req.validate(['test']) - except Exception as ex: - self.assertEqual(1, vc_type.call_count) - self.assertEqual(1, vp_size.call_count) - self.assertEqual(1, vc_tenant.call_count) - - self.assertIsInstance(ex, Exception) diff --git a/monasca_log_api/tests/test_role_middleware.py b/monasca_log_api/tests/test_role_middleware.py deleted file mode 100644 index 59e54f28..00000000 --- a/monasca_log_api/tests/test_role_middleware.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2015-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from unittest import mock - -from webob import response - -from monasca_log_api.middleware import role_middleware as rm -from monasca_log_api.tests import base - - -class SideLogicTestEnsureLowerRoles(base.BaseTestCase): - - def test_should_ensure_lower_roles(self): - roles = ['CMM-Admin', ' CmM-User '] - expected = ['cmm-admin', 'cmm-user'] - self.assertItemsEqual(expected, rm._ensure_lower_roles(roles)) - - def test_should_return_empty_array_for_falsy_input_1(self): - roles = [] - expected = [] - self.assertItemsEqual(expected, rm._ensure_lower_roles(roles)) - - def test_should_return_empty_array_for_falsy_input_2(self): - roles = None - expected = [] - self.assertItemsEqual(expected, rm._ensure_lower_roles(roles)) - - -class SideLogicTestIntersect(base.BaseTestCase): - - def test_should_intersect_seqs(self): - seq_1 = [1, 2, 3] - seq_2 = [2] - - expected = [2] - - self.assertItemsEqual(expected, rm._intersect(seq_1, seq_2)) - self.assertItemsEqual(expected, rm._intersect(seq_2, seq_1)) - - def test_should_intersect_empty(self): - seq_1 = [] - seq_2 = [] - - expected = [] - - self.assertItemsEqual(expected, rm._intersect(seq_1, seq_2)) - self.assertItemsEqual(expected, rm._intersect(seq_2, seq_1)) - - def test_should_not_intersect_without_common_elements(self): - seq_1 = [1, 2, 3] - seq_2 = [4, 5, 6] - - expected = [] - - self.assertItemsEqual(expected, rm._intersect(seq_1, seq_2)) - self.assertItemsEqual(expected, rm._intersect(seq_2, seq_1)) - - -class RolesMiddlewareSideLogicTest(base.BaseTestCase): - - def test_should_apply_middleware_for_valid_path(self): - paths = ['/', '/v2.0/', '/v2.0/log/'] - - instance = rm.RoleMiddleware(None) - instance._path = paths - - for p in paths: - req = mock.Mock() - req.method = 'GET' - req.path = p - self.assertTrue(instance._can_apply_middleware(req)) - - def test_should_apply_middleware_for_invalid_path(self): - paths = ['/v2.0/', '/v2.0/log/'] - - instance = rm.RoleMiddleware(None) - instance._path = paths - - for p in paths: - pp = 'test/%s' % p - req = mock.Mock() - req.method = 'GET' - req.path = pp - self.assertFalse(instance._can_apply_middleware(req)) - - def test_should_reject_OPTIONS_request(self): - instance = rm.RoleMiddleware(None) - req = mock.Mock() - req.method = 'OPTIONS' - req.path = '/' - self.assertFalse(instance._can_apply_middleware(req)) - - def test_should_return_true_if_authenticated(self): - instance = rm.RoleMiddleware(None) - - req = mock.Mock() - req.headers = {rm._X_IDENTITY_STATUS: rm._CONFIRMED_STATUS} - - self.assertTrue(instance._is_authenticated(req)) - - def test_should_return_false_if_not_authenticated(self): - instance = rm.RoleMiddleware(None) - - req = mock.Mock() - req.headers = {rm._X_IDENTITY_STATUS: 'Some_Other_Status'} - - self.assertFalse(instance._is_authenticated(req)) - - def test_should_return_false_if_identity_status_not_found(self): - instance = rm.RoleMiddleware(None) - - req = mock.Mock() - req.headers = {} - - self.assertFalse(instance._is_authenticated(req)) - - def test_should_return_true_if_is_agent(self): - roles = 'cmm-admin,cmm-user' - roles_array = roles.split(',') - - default_roles = [roles_array[0]] - admin_roles = [roles_array[1]] - - instance = rm.RoleMiddleware(None) - instance._default_roles = default_roles - instance._agent_roles = admin_roles - - req = mock.Mock() - req.headers = {rm._X_ROLES: roles} - - is_agent = instance._is_agent(req) - - self.assertTrue(is_agent) - - -class RolesMiddlewareLogicTest(base.BaseTestCase): - - def test_not_process_further_if_cannot_apply_path(self): - roles = 'cmm-admin,cmm-user' - roles_array = roles.split(',') - - default_roles = [roles_array[0]] - admin_roles = [roles_array[1]] - - instance = rm.RoleMiddleware(None) - instance._default_roles = default_roles - instance._agent_roles = admin_roles - instance._path = ['/test'] - - # spying - instance._is_authenticated = mock.Mock() - instance._is_agent = mock.Mock() - - req = mock.Mock() - req.headers = {rm._X_ROLES: roles} - req.path = '/different/test' - - instance.process_request(req=req) - - self.assertFalse(instance._is_authenticated.called) - self.assertFalse(instance._is_agent.called) - - def test_not_process_further_if_cannot_apply_method(self): - roles = 'cmm-admin,cmm-user' - roles_array = roles.split(',') - - default_roles = [roles_array[0]] - admin_roles = [roles_array[1]] - - instance = rm.RoleMiddleware(None) - instance._default_roles = default_roles - instance._agent_roles = admin_roles - instance._path = ['/test'] - - # spying - instance._is_authenticated = mock.Mock() - instance._is_agent = mock.Mock() - - req = mock.Mock() - req.headers = {rm._X_ROLES: roles} - req.path = '/test' - req.method = 'OPTIONS' - - instance.process_request(req=req) - - self.assertFalse(instance._is_authenticated.called) - self.assertFalse(instance._is_agent.called) - - def test_should_produce_json_response_if_not_authenticated( - self): - instance = rm.RoleMiddleware(None) - is_agent = True - is_authenticated = False - - instance._can_apply_middleware = mock.Mock(return_value=True) - instance._is_agent = mock.Mock(return_value=is_agent) - instance._is_authenticated = mock.Mock(return_value=is_authenticated) - - req = mock.Mock() - req.environ = {} - req.headers = { - 'X-Tenant-Id': '11111111' - } - - result = instance.process_request(req=req) - - self.assertIsNotNone(result) - self.assertIsInstance(result, response.Response) - - status = result.status_code - json_body = result.json_body - message = json_body.get('message') - - self.assertIn('Failed to authenticate request for', message) - self.assertEqual(401, status) diff --git a/monasca_log_api/tests/test_service.py b/monasca_log_api/tests/test_service.py deleted file mode 100644 index 54b1d4c0..00000000 --- a/monasca_log_api/tests/test_service.py +++ /dev/null @@ -1,479 +0,0 @@ -# Copyright 2015 kornicameister@gmail.com -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime -import unittest -from unittest import mock - -from falcon import errors -from falcon import testing - -from monasca_log_api.app.base import exceptions -from monasca_log_api.app.base import validation -from monasca_log_api.app.controller.v2.aid import service as aid_service -from monasca_log_api.tests import base - - -class IsDelegate(base.BaseTestCase): - - def __init__(self, *args, **kwargs): - super(IsDelegate, self).__init__(*args, **kwargs) - self._roles = ['admin'] - - def test_is_delegate_ok_role(self): - self.assertTrue(validation.validate_is_delegate(self._roles)) - - def test_is_delegate_ok_role_in_roles(self): - self._roles.extend(['a_role', 'b_role']) - self.assertTrue(validation.validate_is_delegate(self._roles)) - - def test_is_delegate_not_ok_role(self): - roles = ['a_role', 'b_role'] - self.assertFalse(validation.validate_is_delegate(roles)) - - -class ParseDimensions(base.BaseTestCase): - def test_should_fail_for_empty_dimensions(self): - self.assertRaises(exceptions.HTTPUnprocessableEntity, - aid_service.parse_dimensions, - '') - self.assertRaises(exceptions.HTTPUnprocessableEntity, - aid_service.parse_dimensions, - None) - - def test_should_fail_for_empty_dim_in_dimensions(self): - err = self.assertRaises(exceptions.HTTPUnprocessableEntity, - aid_service.parse_dimensions, - ',') - self.assertEqual(err.description, 'Dimension cannot be empty') - - def test_should_fail_for_invalid_dim_in_dimensions(self): - invalid_dim = 'a' - err = self.assertRaises(exceptions.HTTPUnprocessableEntity, - aid_service.parse_dimensions, - invalid_dim) - self.assertEqual(err.description, '%s is not a valid dimension' - % invalid_dim) - - def test_should_pass_for_valid_dimensions(self): - dimensions = 'a:1,b:2' - expected = { - 'a': '1', - 'b': '2' - } - - self.assertDictEqual(expected, - aid_service.parse_dimensions(dimensions)) - - -class ParseApplicationType(base.BaseTestCase): - def test_should_return_none_for_none(self): - self.assertIsNone(aid_service.parse_application_type(None)) - - def test_should_return_none_for_empty(self): - self.assertIsNone(aid_service.parse_application_type('')) - - def test_should_return_none_for_whitespace_filled(self): - self.assertIsNone(aid_service.parse_application_type(' ')) - - def test_should_return_value_for_ok_value(self): - app_type = 'monasca' - self.assertEqual(app_type, - aid_service.parse_application_type(app_type)) - - def test_should_return_value_for_ok_value_with_spaces(self): - app_type = ' monasca ' - expected = 'monasca' - self.assertEqual(expected, - aid_service.parse_application_type(app_type)) - - -class ApplicationTypeValidations(base.BaseTestCase): - def test_should_pass_for_empty_app_type(self): - validation.validate_application_type() - validation.validate_application_type('') - - def test_should_fail_for_invalid_length(self): - r_app_type = testing.rand_string(300, 600) - err = self.assertRaises(exceptions.HTTPUnprocessableEntity, - validation.validate_application_type, - r_app_type) - - length = validation.APPLICATION_TYPE_CONSTRAINTS['MAX_LENGTH'] - msg = ('Application type {type} must be ' - '{length} characters or less'.format(type=r_app_type, - length=length)) - - self.assertEqual(err.description, msg) - - def test_should_fail_for_invalid_content(self): - r_app_type = '%#$@!' - - err = self.assertRaises(exceptions.HTTPUnprocessableEntity, - validation.validate_application_type, - r_app_type) - msg = ('Application type %s may only contain: "a-z A-Z 0-9 _ - ."' % - r_app_type) - self.assertEqual(err.description, msg) - - def test_should_pass_for_ok_app_type(self): - r_app_type = 'monasca' - validation.validate_application_type(r_app_type) - - -class DimensionsValidations(base.BaseTestCase): - @unittest.expectedFailure - def test_should_fail_for_none_dimensions(self): - validation.validate_dimensions(None) - - @unittest.expectedFailure - def test_should_fail_pass_for_non_iterable_dimensions_str(self): - validation.validate_dimensions('') - - @unittest.expectedFailure - def test_should_fail_pass_for_non_iterable_dimensions_number(self): - validation.validate_dimensions(1) - - def test_should_pass_for_empty_dimensions_array(self): - validation.validate_dimensions({}) - - def test_should_fail_too_empty_name(self): - dimensions = {'': 1} - - err = self.assertRaises(exceptions.HTTPUnprocessableEntity, - validation.validate_dimensions, - dimensions) - msg = 'Dimension name cannot be empty' - self.assertEqual(err.description, msg) - - def test_should_fail_too_long_name(self): - name = testing.rand_string(256, 260) - dimensions = {name: 1} - - err = self.assertRaises(exceptions.HTTPUnprocessableEntity, - validation.validate_dimensions, - dimensions) - msg = 'Dimension name %s must be 255 characters or less' % name - self.assertEqual(err.description, msg) - - def test_should_fail_underscore_at_begin(self): - name = '_aDim' - dimensions = {name: 1} - - err = self.assertRaises(exceptions.HTTPUnprocessableEntity, - validation.validate_dimensions, - dimensions) - msg = 'Dimension name %s cannot start with underscore (_)' % name - self.assertEqual(err.description, msg) - - def test_should_fail_invalid_chars(self): - name = '<>' - dimensions = {name: 1} - - err = self.assertRaises(exceptions.HTTPUnprocessableEntity, - validation.validate_dimensions, - dimensions) - invalid_chars = '> < = { } ( ) \' " , ; &' - msg = 'Dimension name %s may not contain: %s' % (name, invalid_chars) - self.assertEqual(err.description, msg) - - def test_should_fail_ok_name_empty_value(self): - name = 'monasca' - dimensions = {name: ''} - - err = self.assertRaises(exceptions.HTTPUnprocessableEntity, - validation.validate_dimensions, - dimensions) - msg = 'Dimension value cannot be empty' - self.assertEqual(err.description, msg) - - def test_should_fail_ok_name_too_long_value(self): - name = 'monasca' - value = testing.rand_string(256, 300) - dimensions = {name: value} - - err = self.assertRaises(exceptions.HTTPUnprocessableEntity, - validation.validate_dimensions, - dimensions) - msg = 'Dimension value %s must be 255 characters or less' % value - self.assertEqual(err.description, msg) - - def test_should_pass_ok_name_ok_value_empty_service(self): - name = 'monasca' - value = '1' - dimensions = {name: value} - validation.validate_dimensions(dimensions) - - def test_should_pass_ok_name_ok_value_service_SERVICE_DIMENSIONS_as_name( - self): - name = 'some_name' - value = '1' - dimensions = {name: value} - validation.validate_dimensions(dimensions) - - -class ContentTypeValidations(base.BaseTestCase): - def test_should_pass_text_plain(self): - content_type = 'text/plain' - allowed_types = ['text/plain'] - - req = mock.Mock() - req.content_type = content_type - validation.validate_content_type(req, allowed_types) - - def test_should_pass_application_json(self): - content_type = 'application/json' - allowed_types = ['application/json'] - - req = mock.Mock() - req.content_type = content_type - - validation.validate_content_type(req, allowed_types) - - def test_should_fail_invalid_content_type(self): - content_type = 'no/such/type' - allowed_types = ['application/json'] - - req = mock.Mock() - req.content_type = content_type - - self.assertRaises( - errors.HTTPUnsupportedMediaType, - validation.validate_content_type, - req, - allowed_types - ) - - def test_should_fail_missing_header(self): - content_type = None - allowed_types = ['application/json'] - - req = mock.Mock() - req.content_type = content_type - - self.assertRaises( - errors.HTTPMissingHeader, - validation.validate_content_type, - req, - allowed_types - ) - - -class PayloadSizeValidations(base.BaseTestCase): - - def test_should_fail_missing_header(self): - content_length = None - req = mock.Mock() - req.content_length = content_length - self.assertRaises( - errors.HTTPLengthRequired, - validation.validate_payload_size, - req - ) - - def test_should_pass_limit_not_exceeded(self): - content_length = 120 - max_log_size = 240 - self.conf_override(max_log_size=max_log_size, - group='service') - - req = mock.Mock() - req.content_length = content_length - - validation.validate_payload_size(req) - - def test_should_fail_limit_exceeded(self): - content_length = 120 - max_log_size = 60 - self.conf_override(max_log_size=max_log_size, - group='service') - - req = mock.Mock() - req.content_length = content_length - - self.assertRaises( - errors.HTTPPayloadTooLarge, - validation.validate_payload_size, - req - ) - - def test_should_fail_limit_equal(self): - content_length = 120 - max_log_size = 120 - self.conf_override(max_log_size=max_log_size, - group='service') - - req = mock.Mock() - req.content_length = content_length - - self.assertRaises( - errors.HTTPPayloadTooLarge, - validation.validate_payload_size, - req - ) - - -class LogMessageValidations(base.BaseTestCase): - def test_should_pass_message_in_log_property(self): - log_object = { - 'message': 'some messages', - 'application_type': 'monasca-log-api', - 'dimensions': { - 'hostname': 'devstack' - } - } - validation.validate_log_message(log_object) - - @unittest.expectedFailure - def test_should_fail_pass_for_non_message_in_log_property(self): - log_object = { - 'massage': 'some messages', - 'application_type': 'monasca-log-api', - 'dimensions': { - 'hostname': 'devstack' - } - } - validation.validate_log_message(log_object) - - def test_should_fail_with_empty_message(self): - self.assertRaises(exceptions.HTTPUnprocessableEntity, - validation.validate_log_message, {}) - - -class LogsCreatorNewLog(base.BaseTestCase): - def setUp(self): - super(LogsCreatorNewLog, self).setUp() - self.instance = aid_service.LogCreator() - - def test_should_create_log_from_json(self): - msg = u'Hello World' - path = u'/var/log/messages' - payload = {"path": path, - "message": msg} - app_type = 'monasca' - dimensions = 'cpu_time:30' - - expected_log = { - 'message': msg, - 'dimensions': { - 'component': app_type, - 'cpu_time': '30' - }, - 'path': path - } - - self.assertEqual(expected_log, self.instance.new_log( - application_type=app_type, - dimensions=dimensions, - payload=payload - )) - - def test_should_create_log_from_text(self): - msg = u'Hello World' - app_type = 'monasca' - dimension_name = 'cpu_time' - dimension_value = 30 - dimensions = '%s:%s' % (dimension_name, str(dimension_value)) - - expected_log = { - 'message': msg, - 'dimensions': { - 'component': app_type, - dimension_name: str(dimension_value) - } - } - - self.assertEqual(expected_log, self.instance.new_log( - application_type=app_type, - dimensions=dimensions, - payload=msg, - content_type='text/plain' - )) - - -class LogCreatorNewEnvelope(base.BaseTestCase): - def setUp(self): - super(LogCreatorNewEnvelope, self).setUp() - self.instance = aid_service.LogCreator() - - def test_should_create_envelope(self): - msg = u'Hello World' - path = u'/var/log/messages' - app_type = 'monasca' - dimension_name = 'cpu_time' - dimension_value = 30 - expected_log = { - 'message': msg, - 'application_type': app_type, - 'dimensions': { - dimension_name: str(dimension_value) - }, - 'path': path - } - tenant_id = 'a_tenant' - none = None - meta = {'tenantId': tenant_id, 'region': none} - timestamp = (datetime.datetime.utcnow() - - datetime.datetime(1970, 1, 1)).total_seconds() - expected_envelope = { - 'log': expected_log, - 'creation_time': timestamp, - 'meta': meta - } - - with mock.patch.object(self.instance, '_create_meta_info', - return_value=meta): - actual_envelope = self.instance.new_log_envelope(expected_log, - tenant_id) - - self.assertEqual(expected_envelope.get('log'), - actual_envelope.get('log')) - self.assertEqual(expected_envelope.get('meta'), - actual_envelope.get('meta')) - self.assertDictEqual( - expected_envelope.get('log').get('dimensions'), - actual_envelope.get('log').get('dimensions')) - - @unittest.expectedFailure - def test_should_not_create_log_none(self): - log_object = None - tenant_id = 'a_tenant' - - self.instance.new_log_envelope(log_object, tenant_id) - - @unittest.expectedFailure - def test_should_not_create_log_empty(self): - log_object = {} - tenant_id = 'a_tenant' - - self.instance.new_log_envelope(log_object, tenant_id) - - @unittest.expectedFailure - def test_should_not_create_tenant_none(self): - log_object = { - 'message': '' - } - tenant_id = None - - self.instance.new_log_envelope(log_object, tenant_id) - - @unittest.expectedFailure - def test_should_not_create_tenant_empty(self): - log_object = { - 'message': '' - } - tenant_id = '' - - self.instance.new_log_envelope(log_object, tenant_id) diff --git a/monasca_log_api/tests/test_v2_v3_compare.py b/monasca_log_api/tests/test_v2_v3_compare.py deleted file mode 100644 index 57c78ca1..00000000 --- a/monasca_log_api/tests/test_v2_v3_compare.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import simplejson as json -from unittest import mock - -from monasca_log_api.app.controller.api import headers -from monasca_log_api.app.controller.v2 import logs as v2_logs -from monasca_log_api.app.controller.v3 import logs as v3_logs -from monasca_log_api.tests import base - - -class TestApiSameV2V3Output(base.BaseApiTestCase): - - # noinspection PyProtectedMember - @mock.patch('monasca_log_api.app.base.log_publisher.' - 'producer.KafkaProducer') - def test_send_identical_messages(self, _): - # mocks only log publisher, so the last component that actually - # sends data to kafka - # case is to verify if publisher was called with same arguments - # for both cases - - v2 = v2_logs.Logs() - v3 = v3_logs.Logs() - - publish_mock = mock.Mock() - - v2._kafka_publisher._kafka_publisher.publish = publish_mock - v3._processor._kafka_publisher.publish = publish_mock - - component = 'monasca-log-api' - service = 'laas' - hostname = 'kornik' - tenant_id = 'ironMan' - roles = 'admin' - - v2_dimensions = 'hostname:%s,service:%s' % (hostname, service) - v3_dimensions = { - 'hostname': hostname, - 'component': component, - 'service': service - } - - v2_body = { - 'message': 'test' - } - - v3_body = { - 'logs': [ - { - 'message': 'test', - 'dimensions': v3_dimensions - } - ] - } - - self.app.add_route('/v2.0', v2) - self.app.add_route('/v3.0', v3) - - self.simulate_request( - path='/v2.0', - method='POST', - headers={ - headers.X_ROLES.name: roles, - headers.X_DIMENSIONS.name: v2_dimensions, - headers.X_APPLICATION_TYPE.name: component, - headers.X_TENANT_ID.name: tenant_id, - 'Content-Type': 'application/json', - }, - body=json.dumps(v2_body) - ) - - self.simulate_request( - path='/v3.0', - method='POST', - headers={ - headers.X_ROLES.name: roles, - headers.X_TENANT_ID.name: tenant_id, - 'Content-Type': 'application/json', - }, - body=json.dumps(v3_body) - ) - - self.assertEqual(2, publish_mock.call_count) - - # in v2 send_messages is called with single envelope - v2_send_msg_arg = publish_mock.mock_calls[0][1][1] - - # in v3 it is always called with list of envelopes - v3_send_msg_arg = publish_mock.mock_calls[1][1][1] - - self.maxDiff = None - - # at this point we know that both args should be identical - self.assertEqual(type(v2_send_msg_arg), type(v3_send_msg_arg)) - self.assertIsInstance(v3_send_msg_arg, list) - - self.assertEqual(len(v2_send_msg_arg), len(v3_send_msg_arg)) - self.assertEqual(1, len(v2_send_msg_arg)) - - v2_msg_as_dict = json.loads(v2_send_msg_arg[0]) - v3_msg_as_dict = json.loads(v3_send_msg_arg[0]) - - self.assertDictEqual(v2_msg_as_dict, v3_msg_as_dict) diff --git a/monasca_log_api/tests/test_version.py b/monasca_log_api/tests/test_version.py deleted file mode 100644 index 4d7605b4..00000000 --- a/monasca_log_api/tests/test_version.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from monasca_log_api.tests import base -from monasca_log_api import version - - -class TestAppVersion(base.BaseTestCase): - def test_should_report_version(self): - self.assertIsNotNone(version.version_str) diff --git a/monasca_log_api/tests/test_versions.py b/monasca_log_api/tests/test_versions.py deleted file mode 100644 index 8a450705..00000000 --- a/monasca_log_api/tests/test_versions.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2016 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import falcon - -from monasca_log_api.app.controller import versions -from monasca_log_api.tests import base - - -def _get_versioned_url(version_id): - return '/version/%s' % version_id - - -class TestApiVersions(base.BaseApiTestCase): - - def setUp(self): - super(TestApiVersions, self).setUp() - self.versions = versions.Versions() - self.app.add_route("/", self.versions) - self.app.add_route("/version/", self.versions) - self.app.add_route("/version/{version_id}", self.versions) - - def test_should_fail_for_unsupported_version(self): - unsupported_version = 'v5.0' - uri = _get_versioned_url(unsupported_version) - - res = self.simulate_request( - path=uri, - method='GET', - headers={ - 'Content-Type': 'application/json' - } - ) - - self.assertEqual(falcon.HTTP_400, res.status) - - def test_should_return_all_supported_versions(self): - - def _check_elements(): - self.assertIn('elements', response) - elements = response.get('elements') - self.assertIsInstance(elements, list) - - for el in elements: - # do checkup by expected keys - self.assertIn('id', el) - self.assertItemsEqual([ - u'id', - u'links', - u'status', - u'updated' - ], el.keys()) - - ver = el.get('id') - self.assertIn(ver, expected_versions) - - def _check_global_links(): - self.assertIn('links', response) - links = response.get('links') - self.assertIsInstance(links, list) - - for link in links: - self.assertIn('rel', link) - key = link.get('rel') - self.assertIn(key, expected_links_keys) - href = link.get('href') - self.assertTrue(href.startswith(expected_url)) - - expected_versions = 'v2.0', 'v3.0' - expected_links_keys = 'self', 'version', 'healthcheck' - expected_protocol = 'http' - expected_host = 'fakehost.com' - expected_url = '{}://{}'.format(expected_protocol, expected_host) - - for expected_path in ['/', '/version']: - res = self.simulate_request( - path=expected_path, - protocol=expected_protocol, - host=expected_host, - method='GET', - headers={ - 'Content-Type': 'application/json' - } - ) - self.assertEqual(falcon.HTTP_200, res.status) - - response = res.json - - _check_elements() - _check_global_links() - - def test_should_return_expected_version_id(self): - expected_versions = 'v2.0', 'v3.0' - for expected_version in expected_versions: - uri = _get_versioned_url(expected_version) - res = self.simulate_request( - path=uri, - method='GET', - headers={ - 'Content-Type': 'application/json' - }, - ) - self.assertEqual(falcon.HTTP_200, res.status) - - response = res.json - self.assertIn('elements', response) - self.assertIn('links', response) - - elements = response.get('elements') - self.assertIsInstance(elements, list) - self.assertEqual(1, len(elements)) - - el = elements[0] - ver = el.get('id') - self.assertEqual(expected_version, ver) diff --git a/monasca_log_api/version.py b/monasca_log_api/version.py deleted file mode 100644 index 7b20ca39..00000000 --- a/monasca_log_api/version.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import pbr.version - -version_info = pbr.version.VersionInfo('monasca-log-api') -version_str = version_info.version_string() diff --git a/playbooks/docker-publish.yml b/playbooks/docker-publish.yml deleted file mode 100644 index f9429f22..00000000 --- a/playbooks/docker-publish.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- hosts: all - tasks: - - name: Login to Dockerhub - command: "docker login -u {{ doker_hub_login_log_api.user }} -p {{ doker_hub_login_log_api.password }}" - no_log: true - - - name: List images - shell: "docker images --format '{% raw %}{{ .Repository }}:{{ .Tag }}{% endraw %}' | grep monasca" - - - name: Push to Docker Hub all tags - shell: "docker push monasca/log-api" diff --git a/releasenotes/notes/cli_args-6dc5e2d13337b871.yaml b/releasenotes/notes/cli_args-6dc5e2d13337b871.yaml deleted file mode 100644 index b25af046..00000000 --- a/releasenotes/notes/cli_args-6dc5e2d13337b871.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -other: - - | - Enabled possibility of specifying the CLI arguments, when launching - monasca-log-api, for cases where API is not deployed using Gunicorn - server. diff --git a/releasenotes/notes/deprecate-60181d946200dff7.yaml b/releasenotes/notes/deprecate-60181d946200dff7.yaml deleted file mode 100644 index 3477621b..00000000 --- a/releasenotes/notes/deprecate-60181d946200dff7.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -deprecations: - - | - This API is deprecated. Last maintained release is OpenStack Train. - Please use monasca-api for newer versions. This repository is kept only - for maintenance purposes. diff --git a/releasenotes/notes/drop-py-2-7-02f3af38da6f6634.yaml b/releasenotes/notes/drop-py-2-7-02f3af38da6f6634.yaml deleted file mode 100644 index 882a169e..00000000 --- a/releasenotes/notes/drop-py-2-7-02f3af38da6f6634.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -upgrade: - - | - Python 2.7 support has been dropped. Last release of monasca-log-api - to support python 2.7 is OpenStack Train. The minimum version of Python now - supported by monasca-log-api is Python 3.6. diff --git a/releasenotes/notes/os-docs-bf79803595ac884b.yaml b/releasenotes/notes/os-docs-bf79803595ac884b.yaml deleted file mode 100644 index 1ec3efc7..00000000 --- a/releasenotes/notes/os-docs-bf79803595ac884b.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- - -deprecations: - - Usage of Markdown based documentation has been deprecated. - It will not be maintained and in future removed from the project - -upgrade: - - Documentation handling of monasca-log-api has been migrated - to match OpenStack process diff --git a/releasenotes/notes/oslo-policy-e142fa9243a8dcf6.yaml b/releasenotes/notes/oslo-policy-e142fa9243a8dcf6.yaml deleted file mode 100644 index ab3d57bb..00000000 --- a/releasenotes/notes/oslo-policy-e142fa9243a8dcf6.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -features: - - | - Use of oslo mechanisms for defining and enforcing policy. - A command line entry point that allows the user to generate a sample policy file. diff --git a/releasenotes/notes/osloconfiggen-d8a0f0a8d1acb961.yaml b/releasenotes/notes/osloconfiggen-d8a0f0a8d1acb961.yaml deleted file mode 100644 index 8415aef8..00000000 --- a/releasenotes/notes/osloconfiggen-d8a0f0a8d1acb961.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -prelude: > - Matching OpenStack projects in configuration handling. -other: - - Removed configuration file from project tree. It is now generated via - oslo-config-generator utility. - - Moved all the code where configuration options are registered into single - location to mimic the configuration file (i.e. provide single place as it - was with configuration file). - - Removed final place where configuration is duplicated. In other words - person providing and/or modyfying options could have been forced to include - them also in the configuration file held in the tree. Not the only place - where it is required is Python codebase. - - Devstack plugin is also using new utility to provide configuration for - monasca-log-api launched inside the VM or Gate environment. diff --git a/releasenotes/notes/project_tree-3a041cbffc83595a.yaml b/releasenotes/notes/project_tree-3a041cbffc83595a.yaml deleted file mode 100644 index 3d3b852a..00000000 --- a/releasenotes/notes/project_tree-3a041cbffc83595a.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -other: - - | - Refactored project tree into more predictable and organized - one. Old tree had everythin mixed up between different locations, - and it was very difficult to follow the code. diff --git a/releasenotes/notes/use-standard-config-path-f47c856009b6410d.yaml b/releasenotes/notes/use-standard-config-path-f47c856009b6410d.yaml deleted file mode 100644 index d5028a5c..00000000 --- a/releasenotes/notes/use-standard-config-path-f47c856009b6410d.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -deprecations: - - | - Configuration file path /etc/monasca/log-api.conf is deprecated. - Use the standard path /etc/monasca/monasca-log-api.conf or the - configuration dir (supported via oslo.config) - /etc/monasca/monasca-log-api.conf.d/any_config_name.conf diff --git a/releasenotes/notes/uwsgi-0bf04f0ecd9c7522.yaml b/releasenotes/notes/uwsgi-0bf04f0ecd9c7522.yaml deleted file mode 100644 index 0b7c14dd..00000000 --- a/releasenotes/notes/uwsgi-0bf04f0ecd9c7522.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -prelude: > - https://governance.openstack.org/tc/goals/pike/deploy-api-in-wsgi.html -features: - - | - According to Pike release goals, all APIs should have the possibility - to be deployed under WSGI. With this change, monasca-log-api is capable - of running on either of following: gunicorn, uwsgi or mod_wsgi. diff --git a/releasenotes/notes/zuul_v3-b4138cd73cb6117a.yaml b/releasenotes/notes/zuul_v3-b4138cd73cb6117a.yaml deleted file mode 100644 index ead2d83b..00000000 --- a/releasenotes/notes/zuul_v3-b4138cd73cb6117a.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -upgrade: - - Upgrade integrations tests setup for Zuul V3. diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py deleted file mode 100644 index f4166433..00000000 --- a/releasenotes/source/conf.py +++ /dev/null @@ -1,252 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.6' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'openstackdocstheme', - 'reno.sphinxext' -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -source_encoding = 'utf-8' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -openstackdocs_repo_name = u'openstack/monasca-log-api' -openstackdocs_auto_name = False -project = u'Monasca Log Release Notes' -openstackdocs_bug_project = u'monasca-log-api' -openstackdocs_bug_tag = u'releasenotes' -copyright = u'2014, OpenStack Foundation' - -# Release notes do not need a version number in the title, they -# cover multiple releases. -version = '' -release = '' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'native' - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'openstackdocs' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = [] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# html_extra_path = [] - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'MonascaLogApiReleaseNotesdoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # 'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'MonascaLogApiReleaseNotes.tex', - u'MonascaLogApi Release Notes Documentation', u'OpenStack Foundation', - 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'barbicanreleasenotes', u'MonascaLogApi Release Notes Documentation', - [u'OpenStack Foundation'], 1) -] - -# If true, show URL addresses after external links. -# man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'MonascaLogApiReleaseNotes', u'MonascaLogApi Release Notes Documentation', - u'OpenStack Foundation', 'MonascaLogApiReleaseNotes', - 'MonascaLogApi Release Notes Documentation.', 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -# texinfo_appendices = [] - -# If false, no module index is generated. -# texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -# texinfo_no_detailmenu = False - -# -- Options for Internationalization output ------------------------------ -locale_dirs = ['locale/'] diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst deleted file mode 100644 index c0c0fa72..00000000 --- a/releasenotes/source/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -=========================== -MonascaLogApi Release Notes -=========================== - -Contents: - -.. toctree:: - :maxdepth: 1 - - unreleased - train - stein - rocky - queens - pike diff --git a/releasenotes/source/locale/.gitkeep b/releasenotes/source/locale/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/releasenotes/source/pike.rst b/releasenotes/source/pike.rst deleted file mode 100644 index e43bfc0c..00000000 --- a/releasenotes/source/pike.rst +++ /dev/null @@ -1,6 +0,0 @@ -=================================== - Pike Series Release Notes -=================================== - -.. release-notes:: - :branch: stable/pike diff --git a/releasenotes/source/queens.rst b/releasenotes/source/queens.rst deleted file mode 100644 index 36ac6160..00000000 --- a/releasenotes/source/queens.rst +++ /dev/null @@ -1,6 +0,0 @@ -=================================== - Queens Series Release Notes -=================================== - -.. release-notes:: - :branch: stable/queens diff --git a/releasenotes/source/rocky.rst b/releasenotes/source/rocky.rst deleted file mode 100644 index 40dd517b..00000000 --- a/releasenotes/source/rocky.rst +++ /dev/null @@ -1,6 +0,0 @@ -=================================== - Rocky Series Release Notes -=================================== - -.. release-notes:: - :branch: stable/rocky diff --git a/releasenotes/source/stein.rst b/releasenotes/source/stein.rst deleted file mode 100644 index efaceb66..00000000 --- a/releasenotes/source/stein.rst +++ /dev/null @@ -1,6 +0,0 @@ -=================================== - Stein Series Release Notes -=================================== - -.. release-notes:: - :branch: stable/stein diff --git a/releasenotes/source/train.rst b/releasenotes/source/train.rst deleted file mode 100644 index 58390039..00000000 --- a/releasenotes/source/train.rst +++ /dev/null @@ -1,6 +0,0 @@ -========================== -Train Series Release Notes -========================== - -.. release-notes:: - :branch: stable/train diff --git a/releasenotes/source/unreleased.rst b/releasenotes/source/unreleased.rst deleted file mode 100644 index cd22aabc..00000000 --- a/releasenotes/source/unreleased.rst +++ /dev/null @@ -1,5 +0,0 @@ -============================== - Current Series Release Notes -============================== - -.. release-notes:: diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 0dd25c3d..00000000 --- a/requirements.txt +++ /dev/null @@ -1,19 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - -pbr!=2.1.0,>=2.0.0 # Apache-2.0 -Paste>=2.0.2 # MIT -falcon>=2.0.0 # Apache-2.0 -keystonemiddleware>=4.17.0 # Apache-2.0 -oslo.config>=5.2.0 # Apache-2.0 -oslo.context>=2.19.2 # Apache-2.0 -oslo.middleware>=3.31.0 # Apache-2.0 -oslo.log>=3.36.0 # Apache-2.0 -oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 -oslo.utils>=3.33.0 # Apache-2.0 -PasteDeploy>=1.5.0 # MIT -monasca-common>=2.7.0 # Apache-2.0 -eventlet!=0.18.3,!=0.20.1,!=0.21.0,>=0.18.2 # MIT -monasca-statsd>=1.1.0 # Apache-2.0 -simplejson>=3.8.1 # MIT diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 9cc9b567..00000000 --- a/setup.cfg +++ /dev/null @@ -1,59 +0,0 @@ -[metadata] -name = monasca-log-api -summary = Monasca API for sending log entries -description-file = - README.rst -author = OpenStack -author-email = openstack-discuss@lists.openstack.org -home-page = https://docs.openstack.org/monasca-log-api/latest/ -python-requires = >=3.6 -classifier = - Environment :: OpenStack - Intended Audience :: Information Technology - Intended Audience :: System Administrators - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: Implementation :: CPython - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - -[files] -packages = - monasca_log_api - -data_files = - etc/monasca = - etc/monasca/log-api-paste.ini - etc/monasca/log-api-logging.conf - -[entry_points] -console_scripts = - monasca-log-api = monasca_log_api.app.main:main - -wsgi_scripts = - monasca-log-api-wsgi = monasca_log_api.app.wsgi:main - -oslo.config.opts = - monasca_log_api = monasca_log_api.conf:list_opts - -oslo.policy.policies = - monasca_log_api = monasca_log_api.policies:list_rules - -[egg_info] -tag_build = -tag_date = 0 -tag_svn_revision = 0 - -[pbr] -autodoc_index_modules = True -autodoc_exclude_modules = - monasca_log_api.app.wsgi* - # NOTE(trebskit) we cannot document that because of side-effect - # of trying to load entire application and inability to find - # config files. - monasca_log_api.tests.* - monasca_log_api_tempest.* -api_doc_dir = contributor/api diff --git a/setup.py b/setup.py deleted file mode 100644 index cd35c3c3..00000000 --- a/setup.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import setuptools - -setuptools.setup( - setup_requires=['pbr>=2.0.0'], - pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index db747947..00000000 --- a/test-requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - -# Install bounded pep8/pyflakes first, then let flake8 install -hacking>=3.0,<3.1.0 # Apache-2.0 -bandit!=1.6.0,>=1.1.0 # Apache-2.0 -bashate>=0.5.1 # Apache-2.0 - -fixtures>=3.0.0 # Apache-2.0/BSD -coverage!=4.4,>=4.0 # Apache-2.0 -oslotest>=3.2.0 # Apache-2.0 -stestr>=1.0.0 # Apache-2.0 -simplejson>=3.8.1 # MIT - -# documentation -doc8>=0.6.0 # Apache-2.0 -sphinx>=2.0.0,!=2.1.0 # BSD -os-api-ref>=1.5.0 # Apache-2.0 -reno>=3.1.0 # Apache-2.0 -openstackdocstheme>=2.2.1 # Apache-2.0 diff --git a/tools/bashate.sh b/tools/bashate.sh deleted file mode 100644 index 33d5bebb..00000000 --- a/tools/bashate.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# Ignore too long lines error E006 from bashate and treat -# E005, E042 as errors. -SH_FILES=$(find $1 -type d -name files -prune -o -type f -name '*.sh' -print) -bashate -v -iE006 -eE005,E042 ${SH_FILES:-''} diff --git a/tox.ini b/tox.ini deleted file mode 100644 index cc43ce7f..00000000 --- a/tox.ini +++ /dev/null @@ -1,158 +0,0 @@ -[tox] -envlist = py37,pep8,cover -minversion = 3.1.1 -ignore_basepython_conflict = True -skipsdist = True - -[testenv] -basepython = python3 -usedevelop = True -setenv = VIRTUAL_ENV={envdir} - OS_TEST_PATH=monasca_log_api/tests -passenv = *_proxy - *_PROXY -whitelist_externals = bash - find - rm -install_command = pip install {opts} {packages} -deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} - -r{toxinidir}/test-requirements.txt - -r{toxinidir}/requirements.txt -commands = - find ./ -type f -name '*.pyc' -delete - stestr run {posargs} - -[testenv:cover] -description = Calculates code coverage -setenv = - PYTHON=coverage run --source monasca_log_api --parallel-mode -commands = - stestr run {posargs} - coverage combine - coverage html -d cover - coverage xml -o cover/coverage.xml - -[testenv:debug] -description = Allows to run unit-test with debug mode enabled -commands = - oslo_debug_helper -t {toxinidir}/monasca_log_api/tests {posargs} - -[testenv:bashate] -description = Validates (pep8-like) devstack plugins -skip_install = True -usedevelop = False -commands = bash {toxinidir}/tools/bashate.sh {toxinidir}/devstack - -[testenv:bandit] -description = Validates codebase with bandit -skip_install = True -usedevelop = False -commands = - # FIXME(dmllr); B101 needs to be fixed first - bandit -r monasca_log_api -n5 -s B101 -x monasca_log_api/tests - -[testenv:pep8] -description = Runs set of linters against codebase (flake8, bandit, bashate, checkniceness) -commands = - {[testenv:flake8]commands} - {[testenv:bandit]commands} - {[testenv:bashate]commands} - {[testenv:checkniceness]commands} - -[testenv:flake8] -description = Validates codebase with flake -commands = - flake8 monasca_log_api - -[testenv:genconfig] -description = Generates sample documentation file for monasca-log-api -commands = oslo-config-generator --config-file=config-generator/monasca-log-api.conf - -[testenv:genpolicy] -description = Generates sample policy.json file for monasca-log-api -commands = oslopolicy-sample-generator --config-file=config-generator/policy.conf - -[testenv:docs] -description = Builds main documention -commands = - {[testenv:devdocs]commands} - -[testenv:api-guide] -description = Called from CI scripts to test and publish the API Guide -commands = - rm -rf api-guide/build - {[testenv:checkjson]commands} - sphinx-build -W -b html -d api-guide/build/doctrees api-guide/source api-guide/build/html - -[testenv:api-ref] -description = Called from CI scripts to test and publish the API Ref -commands = - rm -rf api-ref/build - {[testenv:checkjson]commands} - sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html - -[testenv:releasenotes] -description = Called from CI script to test and publish the Release Notes -commands = - rm -rf releasenotes/build - sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html - -[testenv:all-docs] -description = Builds all docouments -commands = - {[testenv:devdocs]commands} - {[testenv:api-guide]commands} - {[testenv:api-ref]commands} - {[testenv:releasenotes]commands} - -[testenv:devdocs] -description = Builds developer documentation -commands = - rm -rf doc/build - rm -rf doc/source/contributor/api - {[testenv:checkjson]commands} - sphinx-build -a -E -d doc/build/doctrees -b html doc/source doc/build/html - -[testenv:checkniceness] -description = Validates (pep-like) documenation -skip_install = True -usedevelop = False -commands = - doc8 --file-encoding utf-8 {toxinidir}/doc - doc8 --file-encoding utf-8 {toxinidir}/api-ref - doc8 --file-encoding utf-8 {toxinidir}/api-guide - doc8 --file-encoding utf-8 {toxinidir}/releasenotes - -[testenv:checkjson] -description = Validates all json samples inside doc folder -skip_install = True -usedevelop = False -deps = -whitelist_externals = - bash - python - find - rm -commands = - bash -c "! find doc/ -type f -name *.json | xargs grep -U -n $'\r'" - bash -c '! find doc/ -type f -name *.json | xargs -t -n1 python -m json.tool 2>&1 > /dev/null | grep -B1 -v ^python' - -[testenv:venv] -commands = {posargs} - -[flake8] -max-line-length = 100 -exclude = .git,.tox,dist,doc,api-ref,api-guide,releasenotes,documentation,*.egg,build -show-source = True -enable-extensions = H203,H106 - -[hacking] -import_exceptions = - six.moves - -[testenv:lower-constraints] -deps = - -c{toxinidir}/lower-constraints.txt - -r{toxinidir}/test-requirements.txt - -r{toxinidir}/requirements.txt