From 977ca7c54872d331b7fdee6cf765849ac341db9e Mon Sep 17 00:00:00 2001 From: Saad Zaher Date: Wed, 7 Sep 2016 12:19:41 +0000 Subject: [PATCH] Implemented: Installation documentation and API Ref integrated freezer-api with openstackdoctheme lib and os-api-ref lib to build the installation documentation and api reference for freezer. Change-Id: Ib5fc5f8227484525a13c3cfff9acb13d2b384e1a Implements: blueprint implement-install-doc Implements: blueprint implement-api-ref-doc --- api-ref/source/conf.py | 206 ++++++++++ api-ref/source/index.rst | 20 + api-ref/source/parameters.yaml | 8 + .../samples/versions-list-response.json | 15 + api-ref/source/versions.inc | 38 ++ freezer_api/__init__.py | 1 + install-guide/source/actions.rst | 27 ++ install-guide/source/api_documents.rst | 12 + install-guide/source/api_routes.rst | 79 ++++ install-guide/source/client_structure.rst | 29 ++ install-guide/source/conf.py | 291 ++++++++++++++ install-guide/source/devstack_plugin.rst | 48 +++ install-guide/source/get_started.rst | 27 ++ install-guide/source/index.rst | 23 ++ install-guide/source/install.rst | 158 ++++++++ install-guide/source/jobs.rst | 357 ++++++++++++++++++ install-guide/source/known_issues.rst | 22 ++ install-guide/source/metadata_structure.rst | 55 +++ install-guide/source/sessions.rst | 104 +++++ test-requirements.txt | 1 + tox.ini | 8 + 21 files changed, 1529 insertions(+) create mode 100644 api-ref/source/conf.py create mode 100644 api-ref/source/index.rst create mode 100644 api-ref/source/parameters.yaml create mode 100644 api-ref/source/samples/versions-list-response.json create mode 100644 api-ref/source/versions.inc create mode 100644 install-guide/source/actions.rst create mode 100644 install-guide/source/api_documents.rst create mode 100644 install-guide/source/api_routes.rst create mode 100644 install-guide/source/client_structure.rst create mode 100644 install-guide/source/conf.py create mode 100644 install-guide/source/devstack_plugin.rst create mode 100644 install-guide/source/get_started.rst create mode 100644 install-guide/source/index.rst create mode 100644 install-guide/source/install.rst create mode 100644 install-guide/source/jobs.rst create mode 100644 install-guide/source/known_issues.rst create mode 100644 install-guide/source/metadata_structure.rst create mode 100644 install-guide/source/sessions.rst diff --git a/api-ref/source/conf.py b/api-ref/source/conf.py new file mode 100644 index 00000000..836ba8cc --- /dev/null +++ b/api-ref/source/conf.py @@ -0,0 +1,206 @@ +""" +(c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +import subprocess +import sys + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../../')) +sys.path.insert(0, os.path.abspath('../')) +sys.path.insert(0, os.path.abspath('./')) + +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. + +extensions = [ + 'os_api_ref', + 'oslosphinx', +] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# +# source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Backup and Disaster Recovery Service API Reference' +copyright = u'2010-present, OpenStack Foundation' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. + +from freezer_api import version_info +# The full version, including alpha/beta/rc tags. +release = version_info.release_string() +# The short X.Y version. +version = version_info.version_string() + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# The reST default role (used for this markup: `text`) to use +# for all documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = False + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# -- Options for man page output ---------------------------------------------- + +# Grouping the document tree for man pages. +# List of tuples 'sourcefile', 'target', u'title', u'Authors name', 'manual' + + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +# html_theme = '_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' +git_cmd = ["git", "log", "--pretty=format:'%ad, commit %h'", "--date=local", + "-n1"] +html_last_updated_fmt = subprocess.Popen( + git_cmd, stdout=subprocess.PIPE).communicate()[0] + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_use_modindex = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'freezerdoc' + + +# -- Options for LaTeX output ------------------------------------------------- + +# The paper size ('letter' or 'a4'). +# latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +# latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', 'Freezer.tex', u'OpenStack Backup and Disaster Recovery Service ' + u'API Documentation', u'OpenStack Foundation', + 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +# latex_preamble = '' + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_use_modindex = True diff --git a/api-ref/source/index.rst b/api-ref/source/index.rst new file mode 100644 index 00000000..578a0738 --- /dev/null +++ b/api-ref/source/index.rst @@ -0,0 +1,20 @@ +.. + (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +:tocdepth: 2 + + +========================== +Backup and DR Service APIs +========================== + +.. include:: versions.inc \ No newline at end of file diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml new file mode 100644 index 00000000..c32fb434 --- /dev/null +++ b/api-ref/source/parameters.yaml @@ -0,0 +1,8 @@ +############################### Response #################################### + +versions: + type: list + in: body + required: True + description: | + A list of supported major API versions. diff --git a/api-ref/source/samples/versions-list-response.json b/api-ref/source/samples/versions-list-response.json new file mode 100644 index 00000000..d649a519 --- /dev/null +++ b/api-ref/source/samples/versions-list-response.json @@ -0,0 +1,15 @@ +{ + "versions": [ + { + "status": "CURRENT", + "updated": "2015-03-23T13:45:00", + "id": "1", + "links": [ + { + "href": "/v1/", + "rel": "self" + } + ] + } + ] +} \ No newline at end of file diff --git a/api-ref/source/versions.inc b/api-ref/source/versions.inc new file mode 100644 index 00000000..f01da7da --- /dev/null +++ b/api-ref/source/versions.inc @@ -0,0 +1,38 @@ +============ +API Versions +============ + +The Freezer API only supports v1. + + +List major versions +=================== + +.. rest_method:: GET / + +Gets the home document. + +This operation gets the home document. + +The entire API is discoverable from a single starting point, the home document. To explore the entire API, you need to know only this one URI. This document is cacheable. + +The home document lets you write clients by using relational links, so clients do not have to construct their own URLs. You can click through and view the JSON doc in your browser. + +For more information about home documents, see `http://tools.ietf.org/html/draft-nottingham-json-home-02 `__. + + +Normal response codes: 300 +Error response codes: serviceUnavailable (503) + +Response Parameters +------------------- + +.. rest_parameters:: parameters.yaml + + - versions: versions + +Response Example +---------------- + +.. literalinclude:: samples/versions-list-response.json + :language: javascript \ No newline at end of file diff --git a/freezer_api/__init__.py b/freezer_api/__init__.py index 2f95ef0a..f7170d09 100644 --- a/freezer_api/__init__.py +++ b/freezer_api/__init__.py @@ -18,3 +18,4 @@ import pbr.version __version__ = pbr.version.VersionInfo('freezer-api').version_string() +version_info = pbr.version.VersionInfo('freezer-api') diff --git a/install-guide/source/actions.rst b/install-guide/source/actions.rst new file mode 100644 index 00000000..61262abd --- /dev/null +++ b/install-guide/source/actions.rst @@ -0,0 +1,27 @@ +.. _actions: + +Actions +======= + +Actions are stored only to facilitate the assembling of different actions into jobs in the web UI. +They are not directly used by the scheduler. +They are stored in this structure + +.. code-block:: json + + + { + + "freezer_action": { + "action": string, + "backup_name": string, + .... + }, + "mandatory": bool, + "max_retries": int, + "max_retries_interval": int + + "action_id": string, + "user_id": string + } + diff --git a/install-guide/source/api_documents.rst b/install-guide/source/api_documents.rst new file mode 100644 index 00000000..38a23e71 --- /dev/null +++ b/install-guide/source/api_documents.rst @@ -0,0 +1,12 @@ +.. _api_documents: + +API Documents +============= + +Freezer has different types of documents as follow: + +.. toctree:: + + jobs.rst + actions.rst + sessions.rst diff --git a/install-guide/source/api_routes.rst b/install-guide/source/api_routes.rst new file mode 100644 index 00000000..742c203c --- /dev/null +++ b/install-guide/source/api_routes.rst @@ -0,0 +1,79 @@ +.. _api_routes: + +API routes +========== + +General +------- + +.. code-block:: rest + + GET / List API version + GET /v1 JSON Home document, see http://tools.ietf.org/html/draft-nottingham-json-home-03 + +Backup metadata +--------------- + +.. code-block:: rest + + GET /v1/backups(?limit,offset) Lists backups + POST /v1/backups Creates backup entry + + GET /v1/backups/{backup_id} Get backup details + DELETE /v1/backups/{backup_id} Deletes the specified backup + +Freezer clients management +-------------------------- + +.. code-block:: rest + + GET /v1/clients(?limit,offset) Lists registered clients + POST /v1/clients Creates client entry + + GET /v1/clients/{freezerc_id} Get client details + UPDATE /v1/clients/{freezerc_id} Updates the specified client information + DELETE /v1/clients/{freezerc_id} Deletes the specified client information + +Freezer jobs management +----------------------- + +.. code-block:: rest + + GET /v1/jobs(?limit,offset) Lists registered jobs + POST /v1/jobs Creates job entry + + GET /v1/jobs/{jobs_id} Get job details + POST /v1/jobs/{jobs_id} creates or replaces a job entry using the specified job_id + DELETE /v1/jobs/{jobs_id} Deletes the specified job information + PATCH /v1/jobs/{jobs_id} Updates part of the document + +Freezer actions management +-------------------------- + +.. code-block:: rest + + GET /v1/actions(?limit,offset) Lists registered action + POST /v1/actions Creates action entry + + GET /v1/actions/{actions_id} Get action details + POST /v1/actions/{actions_id} creates or replaces a action entry using the specified action_id + DELETE /v1/actions/{actions_id} Deletes the specified action information + PATCH /v1/actions/{actions_id} Updates part of the action document + +Freezer sessions management +--------------------------- + +.. code-block:: rest + + GET /v1/sessions(?limit,offset) Lists registered session + POST /v1/sessions Creates session entry + + GET /v1/sessions/{sessions_id} Get session details + POST /v1/sessions/{sessions_id} creates or replaces a session entry using the specified session_id + DELETE /v1/sessions/{sessions_id} Deletes the specified session information + PATCH /v1/sessions/{sessions_id} Updates part of the session document + + POST /v1/sessions/{sessions_id}/action requests actions (e.g. start/end) upon a specific session + + PUT /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session + DELETE /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session diff --git a/install-guide/source/client_structure.rst b/install-guide/source/client_structure.rst new file mode 100644 index 00000000..64a686b9 --- /dev/null +++ b/install-guide/source/client_structure.rst @@ -0,0 +1,29 @@ +.. _client_structure: + +Freezer Client document structure +================================= + +Identifies a freezer client for the purpose of sending action + +client_info document contains information relevant for client identification + +.. code-block:: json + + client_info: + { + "client_id": string actually a concatenation "tenant-id_hostname" + "hostname": string + "description": string + "uuid": + } + + +client_type document embeds the client_info and adds user_id + +.. code-block:: json + + client_type : + { + "client" : client_info document, + "user_id": string, # owner of the information (OS X-User-Id, keystone provided, added by api) + } diff --git a/install-guide/source/conf.py b/install-guide/source/conf.py new file mode 100644 index 00000000..06bbd0e3 --- /dev/null +++ b/install-guide/source/conf.py @@ -0,0 +1,291 @@ +""" +(c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os +# import sys + + +import openstackdocstheme + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +# TODO(ajaeger): enable PDF building, for example add 'rst2pdf.pdfbuilder' +# extensions = + +# Add any paths that contain templates here, relative to this directory. +# templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Installation Guide for Messaging Service' +bug_tag = u'install-guide' +copyright = u'2016, OpenStack contributors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.1' +# The full version, including alpha/beta/rc tags. +release = '0.1' + +# A few variables have to be set for the log-a-bug feature. +# giturl: The location of conf.py on Git. Must be set manually. +# gitsha: The SHA checksum of the bug description. Automatically extracted +# from git log. +# bug_tag: Tag for categorizing the bug. Must be set manually. +# These variables are passed to the logabug code via html_context. +giturl = (u'http://git.openstack.org/cgit/openstack/freezer-api/tree/' + u'install-guide/source') +git_cmd = "/usr/bin/git log | head -n1 | cut -f2 -d' '" +gitsha = os.popen(git_cmd).read().strip('\n') +html_context = {"gitsha": gitsha, "bug_tag": bug_tag, + "giturl": giturl} + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'openstackdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +html_theme_path = [openstackdocstheme.get_html_theme_path()] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = [] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# So that we can enable "log-a-bug" links from each output HTML page, this +# variable must be set to a format that includes year, month, day, hours and +# minutes. +html_last_updated_fmt = '%Y-%m-%d %H:%M' + + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +html_use_index = False + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +html_show_sourcelink = False + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'install-guide' + +# If true, publish source files +html_copy_source = False + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'InstallGuide.tex', u'Install Guide', + u'OpenStack contributors', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'installguide', u'Install Guide', + [u'OpenStack contributors'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'InstallGuide', u'Install Guide', + u'OpenStack contributors', 'InstallGuide', + 'This guide shows OpenStack end users how to install ' + 'an OpenStack cloud.', 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] + +# -- Options for PDF output -------------------------------------------------- + +pdf_documents = [ + ('index', u'InstallGuide', u'Install Guide', + u'OpenStack contributors') +] diff --git a/install-guide/source/devstack_plugin.rst b/install-guide/source/devstack_plugin.rst new file mode 100644 index 00000000..a19755a8 --- /dev/null +++ b/install-guide/source/devstack_plugin.rst @@ -0,0 +1,48 @@ +.. _devstack_plugin: + +Devstack Plugin +=============== + +Edit local.conf +--------------- + +To configure the Freezer API with DevStack, you will need to enable the +freezer-api plugin by adding one line to the [[local|localrc]] section +of your local.conf file: + +.. code-block:: ini + + enable_plugin freezer-api [GITREF] + +where + +.. code-block:: json + + is the URL of a freezer-api repository + [GITREF] is an optional git ref (branch/ref/tag). The default is master. + +For example + +.. code-block:: ini + + enable_plugin freezer-api https://git.openstack.org/openstack/freezer-api.git master + +Plugin Options +-------------- + +The plugin makes use of apache2 by default. +To use the *uwsgi* server set the following environment variable + +.. code-block:: bash + + export FREEZER_API_SERVER_TYPE=uwsgi + +The default port is *9090*. To configure the api to listen on a different port +set the variable `FREEZER_API_PORT`. +For example to make use of port 19090 use + +.. code-block:: bash + + export FREEZER_API_PORT=19090 + +For more information, see `openstack_devstack_plugins_install `_ diff --git a/install-guide/source/get_started.rst b/install-guide/source/get_started.rst new file mode 100644 index 00000000..7d9e6667 --- /dev/null +++ b/install-guide/source/get_started.rst @@ -0,0 +1,27 @@ +============================== +Backup and DR service Overview +============================== + +Freezer is a Backup Restore DR as a Service platform that helps you to automate +the data backup and restore process. + +The service features a RESTful API, which can be used to maintain the status of +your jobs, backups and metadata. + +This chapter assumes a working setup of OpenStack following the base +Installation Guide. + + +Concepts and definitions +======================== + +*hostname* is _probably_ going to be the host fqdn. + +*backup_id* +defined as `container_hostname_backupname_timestamp_level` uniquely +identifies a backup + +*backup_set* +defined as `container_hostname_backupname` identifies a group of related +backups which share the same container,hostname and backupname + diff --git a/install-guide/source/index.rst b/install-guide/source/index.rst new file mode 100644 index 00000000..e4ec3e41 --- /dev/null +++ b/install-guide/source/index.rst @@ -0,0 +1,23 @@ +===================== +Backup and DR service +===================== + +.. toctree:: + + get_started.rst + install.rst + api_routes.rst + metadata_structure.rst + client_structure.rst + api_documents.rst + known_issues.rst + + +Freezer is a Backup Restore DR as a Service platform that helps you to automate +the data backup and restore process. + +The service features a RESTful API, which can be used to maintain the status of +your jobs, backups and metadata. + +This chapter assumes a working setup of OpenStack following the base +Installation Guide. \ No newline at end of file diff --git a/install-guide/source/install.rst b/install-guide/source/install.rst new file mode 100644 index 00000000..6a6b4fba --- /dev/null +++ b/install-guide/source/install.rst @@ -0,0 +1,158 @@ +.. _install: + +Installation +============ + +Install freezer-api +------------------- + +.. code-block:: bash + + # git clone https://git.openstack.org/openstack/freezer-api.git + # cd freezer-api + # pip install ./ + +edit config file +---------------- + +.. code-block:: bash + + # sudo cp etc/freezer/freezer-api.conf /etc/freezer/freezer-api.conf + # sudo cp etc/freezer/freezer-paste.ini /etc/freezer/freezer-paste.ini + # sudo vi /etc/freezer/freezer-api.conf + # sudo vi /etc/freezer/freezer-paste.ini + +setup/configure the db +---------------------- + +The currently supported db is Elasticsearch. In case you are using a dedicated instance +of the server, you'll need to start it. Depending on the OS flavor it might be a: + +.. code-block:: bash + + # service elasticsearch start + +or, on systemd + +.. code-block:: bash + + # systemctl start elasticsearch + +Elasticsearch needs to know what type of data each document's field contains. +This information is contained in the `mapping`, or schema definition. +Elasticsearch will use dynamic mapping to try to guess the field type from +the basic datatypes available in JSON, but some field's properties have to be +explicitly declared to tune the indexing engine. +To do that, use the freezer-manage command: +:: + + # freezer-manage db sync + +You should have updated your configuration files before doing this step. +freezer-manage has the following options: + +* To create the db mappings use the following command:: + + # freezer-manage db sync + +* To update the db mappings using the following command. Update means that you + might have some mappings and you want to update it with a more recent ones + :: + + # freezer-manage db update + +* To remove the db mappings using the following command :: + + # freezer-manage db remove + +* To print the db mappings using the following command :: + + # freezer-manage db show + +* To update your settings (number of replicas) all what you need to do is to + change its value in the configuration file and then run the following command :: + + # freezer-manage db update-settings + +If you provided an invalid number of replicas that will cause problems later on, +so it's highly recommended to make sure that you are using the correct number +of replicas. For more info click here `Elasticsearch_Replicas_instructions `_ + +* To get information about optional additional parameters:: + + # freezer-manage -h + +* If you want to add any additional parameter like --yes or --erase, they should + be before the db option. Check the following examples: + +Wrong Example:: + + # freezer-manage db sync -y -e + +Correct Example:: + + # freezer-manage -y -e db sync + +run simple instance +------------------- + +.. code-block:: bash + + # freezer-api + +examples running using uwsgi +---------------------------- + +.. code-block:: bash + + # uwsgi --http :9090 --need-app --master --module freezer_api.cmd.wsgi:application + + # uwsgi --https :9090,foobar.crt,foobar.key --need-app --master --module freezer_api.cmd.wsgi:application + + +example running freezer-api with apache2 +---------------------------------------- + +.. code-block:: apache + + # sudo vi /etc/apache2/sites-enabled/freezer-api.conf + + + WSGIDaemonProcess freezer-api processes=2 threads=2 user=freezer display-name=%{GROUP} + WSGIProcessGroup freezer-api + WSGIApplicationGroup freezer-api + WSGIScriptAlias / /opt/stack/freezer_api/cmd/wsgi.py + + ErrorLog /var/log/freezer-api/freezer-api.log + CustomLog /var/log/freezer-api/freezer-api_access.log combined + LogLevel info + + + Options Indexes FollowSymLinks MultiViews + Require all granted + AllowOverride None + Order allow,deny + allow from all + LimitRequestBody 102400 + + + + +API registration +================ + +.. code-block:: bash + + # openstack user create --domain default --password-prompt freezer + # openstack role add --project service --user freezer admin + + # openstack service create --name freezer --description "Freezer Backup Service" backup + + # openstack endpoint create --region RegionOne backup public http://freezer_api_publicurl:port + # openstack endpoint create --region RegionOne backup internal http://freezer_api_internalurl:port + # openstack endpoint create --region RegionOne backup admin http://freezer_api_adminurl:port + + +.. toctree:: + + devstack_plugin.rst diff --git a/install-guide/source/jobs.rst b/install-guide/source/jobs.rst new file mode 100644 index 00000000..2d455a95 --- /dev/null +++ b/install-guide/source/jobs.rst @@ -0,0 +1,357 @@ +.. _jobs: + +Jobs +==== + +A job describes a single action to be executed by a freezer client, for example a backup, or a restore. +It contains the necessary information as if they were provided on the command line. + +A job is stored in the api together with some metadata information such as: +job_id, user_id, client_id, status, scheduling information etc + +Scheduling information enables future/recurrent execution of jobs + +.. code-block:: none + + +---------------------+ + | Job | + +---------------------+ job_actions +--------------+ + | +---------------->| job_action | + | +job_id | 0..* +--------------+ freezer_action + | +client_id | | +mandatory |-------------+ + | +user_id | | +retries | | +----------------+ + | +description | job_schedule +--------------+ +->| freezer_action | + | +---------------+ +----------------+ + | | | +-------------------+ + +---------------------+ +-->| job schedule dict | + +-------------------+ + + +job document structure + +.. code-block:: json + + "job": { + "job_action": { parameters for freezer to execute a specific action } + "job_schedule": { scheduling information } + "job_id": string + "client_id": string + "user_id": string + "description": string + } + + "job_actions": + [ + { + "freezer_action" : + { + "action" : string + "mode" : string + "src_file" : string + "backup_name" : string + "container" : string + ... + }, + "mandatory": False, + "max_retries": 3, + "max_retry_interval": 60 + }, + { + "freezer_action" : + { + ... + }, + "mandatory": False, + "max_retries": 3, + "max_retry_interval": 60 + + } + ] + + "job_schedule": { + "time_created": int (timestamp) + "time_started": int (timestamp) + "time_ended": int (timestamp) + "status": string ["stop", "scheduled", "running", "aborting", "removed"] + "event": string ["", "stop", "start", "abort", "remove"] + "result": string ["", "success", "fail", "aborted"] + + SCHEDULING TIME INFORMATION + } + + +Scheduling Time Information +--------------------------- + +Three types of scheduling can be identified + + * date - used for single run jobs + * interval - periodic jobs, providing an interval value + * cron-like jobs + +Each type has specific parameters which can be given. + +date scheduling +--------------- + +.. code-block:: json + + "schedule_date": : datetime isoformat + +interval scheduling +------------------- + +.. code-block:: json + + "schedule_interval" : "continuous", "N weeks" / "N days" / "N hours" / "N minutes" / "N seconds" + + "schedule_start_date" : datetime isoformat + "schedule_end_date" : datetime isoformat + +cron-like scheduling +-------------------- + +.. code-block:: json + + "schedule_year" : 4 digit year + "schedule_month" : 1-12 + "schedule_day" : 1-31 + "schedule_week" : 1-53 + "schedule_day_of_week": 0-6 or string mon,tue,wed,thu,fri,sat,sun + "schedule_hour" : 0-23 + "schedule_minute" : 0-59 + "schedule_second" : 0-59 + + "schedule_start_date" : datetime isoformat + "schedule_end_date" : datetime isoformat + +Job examples +------------ + +example backup freezer_action + +.. code-block:: json + + "freezer_action": { + "action" : "backup" + "mode" : "fs" + "src_file" : "/home/tylerdurden/project_mayhem" + "backup_name" : "project_mayhem_backup" + "container" : "my_backup_container" + "max_backup_level" : int + "always_backup_level": int + "restart_always_backup": int + "no_incremental" : bool + "encrypt_pass_file" : private_key_file + "log_file" : "/var/log/freezer.log" + "hostname" : false + "max_cpu_priority" : false + } + +example restore freezer_action + +.. code-block:: json + + "freezer_action": { + "action": "restore" + "restore-abs-path": "/home/tylerdurden/project_mayhem" + "container" : "my_backup_container" + "backup-name": "project_mayhem_backup" + "restore-from-host": "another_host" + "max_cpu_priority": true + } + + +example scheduled backup job. +job will be executed once at the provided datetime + +.. code-block:: json + + "job": { + "job_actions": + [ + { + "freezer_action": + { + "action" : "backup", + "mode" : "fs", + "src_file" : "/home/tylerdurden/project_mayhem", + "backup_name" : "project_mayhem_backup", + "container" : "my_backup_container", + } + "exit_status": "fail|success" + "max_retries": int, + "max_retries_interval": secs, + "mandatory": bool + }, + { + action + ... + }, + { + action + ... + } + ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + "status": "stop | scheduled | running", + "schedule_date": "2015-06-02T16:20:00", + } + "job_id": "blabla", + "client_id": "blabla", + "user_id": "blabla", + "description": "scheduled one shot", + } + + + "job": { + "job_actions": + [ ... ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + + "status": "stop", + "event": "start" + "schedule_interval" : "1 days" + "schedule_start_date" : "2015-06-02T16:20:00" + }, + "job_id": "4822e482fcbb439189a1ad616ac0a72f", + "client_id": "26b4ea367ac64702868653912e9428cc_freezer.mydomain.myid", + "user_id": "35a322dfb2b14f40bc53a29a14309021", + "description": "daily backup", + } + + +multiple scheduling choices allowed + +.. code-block:: json + + "job": { + "job_actions": + [ ... ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + "status": "scheduled" + "schedule_month" : "1-6, 9-12" + "schedule_day" : "mon, wed, fri" + "schedule_hour": "03" + "schedule_minute": "25" + } + "job_id": "blabla", + "client_id": "blabla", + "user_id": "blabla", + "description": "daily backup", + } + + +Finished job with result + +.. code-block:: json + + "job": { + "job_actions": [ ... ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 4321, + "status": "stop", + "event": "", + "result": "success", + "schedule_time": "2015-06-02T16:20:00" + }, + "job_id": "blabla", + "client_id": "blabla", + "user_id": "blabla", + "description": "one shot job", + } + + +Actions default values +---------------------- + +It is possible to define properties that span across multiple actions +This allow not to rewrite values that might be the same in multiple actions. +If properties are specifically set in one action, then the specified value is the one used. + +Example + +.. code-block:: json + + "job": { + "action_defaults": { + "log_file": "/tmp/freezer_tmp_log", + "container": "my_backup_container" + }, + "job_actions": [{ + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user1/file", + "backup_name": "user1_backup" + } + }, { + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user2/file", + "backup_name": "user2_backup" + } + }, { + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user3/file", + "backup_name": "user2_backup", + "log_file": "/home/user3/specific_log_file" + } + }], + "description": "scheduled one shot" + } + + +Is Equivalent to + +.. code-block:: json + + "job": { + "job_actions": [{ + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user1/file", + "backup_name": "user1_backup", + "log_file": "/tmp/freezer_tmp_log", + "container": "my_backup_container" + } + }, { + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user2/file", + "backup_name": "user2_backup", + "log_file": "/tmp/freezer_tmp_log", + "container": "my_backup_container" + } + }, { + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/user3/file", + "backup_name": "user2_backup", + "log_file": "/home/user3/specific_log_file", + "container": "my_backup_container" + } + }], + "description": "scheduled one shot" + } diff --git a/install-guide/source/known_issues.rst b/install-guide/source/known_issues.rst new file mode 100644 index 00000000..03cee098 --- /dev/null +++ b/install-guide/source/known_issues.rst @@ -0,0 +1,22 @@ +.. _known_issues: + +Known Issues +============ + +Versions of falcon < 0.1.8 +-------------------------- + +Versions of `falcon `_ prior to 0.1.8 (to be precise, +before `this commit `_) +do not have support for error handlers, which are used internally by freezer-api +to specify the outcomes of various actions. + +The absence of this error handling support means that freezer-api **will not start** +on systems running the following, otherwise supported stable versions of +falcon: + +* 0.1.6 +* 0.1.7 + +falcon 0.1.8, which was released on Jan 14, 2014, and all newer versions support +this functionality. \ No newline at end of file diff --git a/install-guide/source/metadata_structure.rst b/install-guide/source/metadata_structure.rst new file mode 100644 index 00000000..867dd9eb --- /dev/null +++ b/install-guide/source/metadata_structure.rst @@ -0,0 +1,55 @@ +.. _metadata_structure: + +Backup metadata structure +========================= + +.. note:: + sizes are in MB + +.. code-block:: json + + backup_metadata:= + { + "container": string, + "host_name": string, # fqdn, client has to provide consistent information here ! + "backup_name": string, + "time_stamp": int, + "level": int, + "max_level": int, + "mode" : string, (fs mongo mysql) + "fs_real_path": string, + "vol_snap_path": string, + "total_broken_links" : int, + "total_fs_files" : int, + "total_directories" : int, + "backup_size_uncompressed" : int, + "backup_size_compressed" : int, + "compression_alg": string, (gzip bzip xz) + "encrypted": bool, + "client_os": string + "broken_links" : [string, string, string], + "excluded_files" : [string, string, string] + "cli": string, equivalent cli used when executing the backup ? + "version": string + } + + +The api wraps backup_metadata dictionary with some additional information. +It stores and returns the information provided in this form + +.. code-block:: json + + { + "backup_id": string # container_hostname_backupname_timestamp_level + "user_id": string, # owner of the backup metadata (OS X-User-Id, keystone provided) + "user_name": string # owner of the backup metadata (OS X-User-Name, keystone provided) + + "backup_metadata": { #--- actual backup_metadata provided + "container": string, + "host_name": string, + "backup_name": string, + "timestamp": int, + ... + } + } + diff --git a/install-guide/source/sessions.rst b/install-guide/source/sessions.rst new file mode 100644 index 00000000..54e99819 --- /dev/null +++ b/install-guide/source/sessions.rst @@ -0,0 +1,104 @@ +.. _sessions: + +Sessions +======== + +A session is a group of jobs which share the same scheduling time. A session is identified +by its **session_id** and has a numeric tag (**session_tag**) which is incremented each time that a new session +is started. +The purpose of the *session_tag* is that of identifying a group of jobs which have been executed +together and which therefore represent a snapshot of a distributed system. + +When a job is added to a session, the scheduling time of the session is copied into the +job data structure, so that any job belonging to the same session will start at the same time. + + +Session Data Structure +---------------------- + +.. code-block:: json + + session = + { + "session_id": string, + "session_tag": int, + "description": string, + "hold_off": int (seconds), + "schedule": { scheduling information, same as jobs }, + "jobs": { 'job_id_1': { + "client_id": string, + "status": string, + "result": string + "time_started": int (timestamp), + "time_ended": int (timestamp), + }, + 'job_id_2': { + "client_id": string, + "status": string, + "result": string + "time_started": int (timestamp), + "time_ended": int (timestamp), + } + } + "time_start": int timestamp, + "time_end": int timestamp, + "time_started": int (timestamp), + "time_ended": int (timestamp), + "status": string "completed" "running", + "result": string "success" "fail", + "user_id": string + } + +Session actions +--------------- + +When the freezer scheduler running on a node wants to start a session, +it sends a POST request to the following endpoint: + +.. code-block:: json + + POST /v1/sessions/{sessions_id}/action + +The body of the request bears the action and parameters + +Session START action +-------------------- + +.. code-block:: json + + { + "start": { + "job_id": "JOB_ID_HERE", + "current_tag": 22 + } + } + +Example of a successful response + +.. code-block:: json + + { + 'result': 'success', + 'session_tag': 23 + } + +Session STOP action +------------------- + +.. code-block:: json + + { + "end": { + "job_id": "JOB_ID_HERE", + "current_tag": 23, + "result": "success|fail" + } + } + +Session-Job association +----------------------- + +.. code-block:: rest + + PUT /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session + DELETE /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session diff --git a/test-requirements.txt b/test-requirements.txt index 23d4d1b9..1b7c6fd5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,6 +12,7 @@ python-subunit>=0.0.18 # Apache-2.0/BSD sphinx!=1.3b1,<1.3,>=1.2.1 # BSD testrepository>=0.0.18 # Apache-2.0/BSD testtools>=1.4.0 # MIT +os-api-ref>=1.0.0 # Apache-2.0 # Tempest Plugin tempest-lib>=0.14.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 9326859d..15017f89 100644 --- a/tox.ini +++ b/tox.ini @@ -88,4 +88,12 @@ exclude = .venv,.tox,dist,doc,test,*egg,tests,specs,build [testenv:docs] commands = python setup.py build_sphinx + sphinx-build -b html api-ref/source api-ref/build/html +[testenv:api-ref] +whitelist_externals = bash + rm +install_command = pip install -U --force-reinstall {opts} {packages} +commands = + rm -rf api-ref/build + sphinx-build -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html