From 56238e890c67ce9d020254ee22aed0efb8b2cf6a Mon Sep 17 00:00:00 2001 From: Phil Bridges Date: Thu, 11 Feb 2016 18:38:46 -0600 Subject: [PATCH] Add more documentation --- bin/swiftonhpss-nstool | 2 +- doc/asciidoc/swiftonhpss_deploy.txt | 522 ++++++++++++++++++++++++++++ doc/troff/swiftonhpss-nstool.1 | 35 ++ setup.py | 13 +- 4 files changed, 570 insertions(+), 2 deletions(-) create mode 100644 doc/asciidoc/swiftonhpss_deploy.txt create mode 100644 doc/troff/swiftonhpss-nstool.1 diff --git a/bin/swiftonhpss-nstool b/bin/swiftonhpss-nstool index a9666ea..c6525a5 100755 --- a/bin/swiftonhpss-nstool +++ b/bin/swiftonhpss-nstool @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # Copyright (c) 2015-2016 IBM Corporation # diff --git a/doc/asciidoc/swiftonhpss_deploy.txt b/doc/asciidoc/swiftonhpss_deploy.txt new file mode 100644 index 0000000..dc0bafe --- /dev/null +++ b/doc/asciidoc/swiftonhpss_deploy.txt @@ -0,0 +1,522 @@ +SwiftOnHPSS Preliminary Deployment Guide +======================================== +Phil Bridges +:Author Initials: PGB +:toc: +:icons: +:numbered: +:website: http://www.hpss-collaboration.org + +[red]*Disclaimer: This interface is not certified for use in an HPSS production +environment to store/retrieve production (valued) data! + +It has not been extensively tested, and is still early in development. + +Only for use on a dedicated SwiftOnHPSS development system!* + +This document is meant to provide general instruction on the installation and +configuration of a stand-alone Swift node running SwiftOnHPSS. + +[[X1]] +Working Assumptions +------------------- +This guide assumes a good working knowledge of: + +* OpenStack Swift +* OpenStack Keystone +* HPSS administration +* Basic Linux system administration + +This guide also makes assumptions on your: + +* Client machine: running RHEL 7.1 on a reasonably modern x86-64 processor, + with Internet access for the duration of the install and LAN connectivity + to the HPSS core and movers at all times. +* HPSS installation: HPSS 7.4.3p2 or newer, using UNIX authentication with + local or system passwd files +* OpenStack infrastructure: that there exists a Keystone server already, + configured with a service tenant and 'swift' service account. + + +[[X2]] +Initial Considerations +---------------------- +Contact your HPSS support representative for copies of the FUSE driver and +client RPMs for the specific version of HPSS that is running on your site. We +have only tested with HPSS FUSE 2.01 and newer. + +Install the HPSS client and HPSS FUSE. This machine will run a stand-alone +Swift node with SwiftOnHPSS. + +Add a 'swift' service user in HPSS and on the client machine: + +---- + # adduser swift + # /opt/hpss/bin/hpss_unix_user add swift +---- + +Next, create a few FUSE mount points. For the account and container servers, +create a local directory (we have it at /srv/swift/hpss_meta) and use FUSE to +mount it to a directory in HPSS (we use /swift/meta). For the object +server, we're using /srv/swift/hpss to mount /swift/objects. Specific +commands are as shown below. + +[WARNING] +.Choose the classes of service carefully! +=============================================================================== +For the Swift metadata mount point, be sure to choose a COS that: + +* Is geared towards 'small' files +* Has a migration policy to tape, AND +* [red]*Does not purge anything from disk!* + +Otherwise, Swift will not perform very well at all if not accessed for some +time! + +For the Swift object mount point, choose a COS that can accept a wide range +of file sizes. Migration and purge policies and storage hierarchy can be +as desired. +=============================================================================== + +.Creating the mountpoints +----- +# export SWIFT_COS=1 # Replace this with metadata COS +# export SWIFT_COS=2 # Replace this with object COS +# /opt/hpss/bin/scrub +scrub> mkdir /swift +scrub> mkdir /swift/meta +scrub> mkdir /swift/objects +scrub> quit +# mkdir -p /srv/swift/hpss +# vim /etc/fstab +----- + +.Example /etc/fstab +[source,conf] +---- +# +# /etc/fstab +# Created by anaconda on Tue Sep 22 18:37:31 2015 +# +# Accessible filesystems, by reference, are maintained under '/dev/disk' +# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info +# +UUID=34658c22-0e55-4e01-b225-742afd5da2fe / ext4 defaults 1 1 +UUID=5b1f787b-0e8d-49b4-aa6f-eb5600a2516d /boot ext4 defaults 1 2 +UUID=c3e6347e-a8bc-4b5e-9477-1bf98c31a032 /localhome ext4 defaults 1 2 +UUID=ed54b2eb-e96d-4de9-8722-44f51c31140b swap swap defaults 0 0 + +/dev/mapper/mpathd /db2_backup ext4 defaults 0 0 +/dev/mapper/mpathf /db2_log ext4 defaults 0 0 +/dev/mapper/mpatha /fake_resources ext4 defaults 0 0 +/dev/mapper/mpathc /hpss_tbspc ext4 defaults 0 0 +/dev/mapper/mpathe /var/hpss ext4 defaults 0 0 + +# Replace the COS number with the COS number chosen above. +/swift/objects /srv/swift/hpss hpssfs rw,noauto,allow_other,cksum=md5,nch=g,cos=1 0 0 +/swift/meta /srv/swift/hpss_meta hpssfs rw,noauto,allow_other,cos=2 0 0 +---- + +Also, make sure that the /swift folder in HPSS is owned by the 'swift' service +user. + +---- +# /opt/hpss/bin/scrub +scrub> chown /swift swift recurse top +scrub> quit +---- + +[[X3]] +Install Prerequisites for SwiftOnHPSS +------------------------------------- +It's recommended to install 'pip', which is a Python package management tool. +That can be found at https://pip.pypa.io: + +---- +# wget https://pip.pypa.io/get-pip.py +# python get-pip.py +---- + +Install memcached, start the service and make it run on startup. + +---- +# yum install memcached python-memcached +# service memcached start +# systemctl enable memcached +---- + + +[[X4]] +Install Keystone (if not already existing) +------------------------------------------ +IF YOU DO NOT HAVE KEYSTONE YET: + +Install OpenStack Keystone, and deploy it following +this guide: https://docs.openstack.org/kilo/install-guide/yum/content/keystone-install.html + +After installing Keystone and verifying that it works using that guide, +create the service entity and API endpoint, but don't create any users or +projects yet if the HPSS-specific Keystone backend is desired. + +If it is desired, install the HPSS Keystone backend, and configure +/etc/keystone/keystone.conf to use it as the sole identity provider. + +---- +# cd /opt/openstack +# git clone https://github.com/hpss-collaboration/hpss-keystone-driver +# cd hpss-keystone-driver +# pip install -r requirements.txt +# python setup.py develop +---- + +./etc/keystone/keystone.conf +---- + +[identity] +driver = hpss_keystone.identity.Identity + +---- + + +[[X5]] +Install and Configure SwiftOnHPSS +--------------------------------- +We have tested SwiftOnHPSS against the Kilo stable release of Swift. +Download the Kilo release of Swift (https://launchpad.net/swift/juno/2.2.0, or +more optimally from https://github.com/openstack/swift) and install it. +Additionally, install the 'keystonemiddleware' library using Pip if you are +using Keystone. + +---- +# mkdir /opt/openstack +# cd /opt/openstack +# git clone https://github.com/openstack/swift stable/kilo +# git clone https://github.com/openstack/keystone stable/kilo +# git clone https://github.com/hpss-collaboration/swiftonhpss +# pip install keystonemiddleware +# pip install -r swift/requirements.txt +# pip install -r swift/setup_requirements.txt +# pip install -r keystone/requirements.txt +# cd swiftonhpss +# python setup.py develop +---- + +Contact your HPSS support representative for the 'hpss', 'hpssfs' and +'swiftonhpss' Python libraries appropriate for your site's version of HPSS. +Upon receipt of those libraries, install them. Assuming the libraries are in +/opt/hpss_lib: + +---- +# cd /opt/hpss_lib +# python hpssfs/setup.py install +# python hpsspy/setup.py install +# pip install -r swiftonhpss/prereqs.txt +# python swiftonhpss/setup.py develop +---- + +Copy the sample configuration files over from the 'etc' folder of the Swift +installation over to /etc/swift, and remove the extra account and object +server configurations. + +---- +# cp -r doc/saio/etc/ /etc/swift + +# rm /etc/swift/account-server/2.conf +# rm /etc/swift/account-server/3.conf +# rm /etc/swift/account-server/4.conf + +# rm /etc/swift/container-server/2.conf +# rm /etc/swift/container-server/3.conf +# rm /etc/swift/container-server/4.conf + +# rm /etc/swift/object-server/2.conf +# rm /etc/swift/object-server/3.conf +# rm /etc/swift/object-server/4.conf +---- + +Then go through each one of the remaining account, container, and object +configuration files and change them as so: + +./etc/swift/account-server/1.conf: +[source,conf] +---- +[DEFAULT] +devices = /srv/swift/hpss_meta +mount_check = false +disable_fallocate = true +bind_ip = 127.0.0.1 +bind_port = 6012 +workers = 1 +user = swift +log_facility = LOG_LOCAL2 +recon_cache_path = /var/cache/swift +eventlet_debug = true + +[pipeline:main] +pipeline = recon account-server + +[app:account-server] +use = egg:swift#account + +[filter:recon] +use = egg:swift#recon + +#[account-replicator] +#vm_test_mode = yes + +#[account-auditor] + +#[account-reaper] +---- + +./etc/swift/container-server/1.conf +[source,conf] +---- +[DEFAULT] +devices = /srv/swift/hpss_meta +mount_check = false +disable_fallocate = true +bind_ip = 127.0.0.1 +bind_port = 6011 +workers = 1 +user = swift +log_facility = LOG_LOCAL2 +recon_cache_path = /var/cache/swift +eventlet_debug = true +allow_versions = true + +[pipeline:main] +pipeline = recon container-server + +[app:container-server] +use = egg:swift#container + +[filter:recon] +use = egg:swift#recon + +#[container-replicator] +#vm_test_mode = yes + +#[container-updater] + +#[container-auditor] + +#[container-sync] +---- + +./etc/swift/object-server/1.conf: +[source,conf] +---- +[DEFAULT] +devices = /srv/swift/hpss/.obfuscated +mount_check = false +disable_fallocate = true +bind_ip = 127.0.0.1 +bind_port = 6010 +workers = 1 +user = swift +log_facility = LOG_LOCAL2 +recon_cache_path = /var/cache/swift +eventlet_debug = true + +[pipeline:main] +pipeline = recon object-server + +[app:object-server] +use = egg:swiftonhpss#object + +[filter:recon] +use = egg:swift#recon + +### Specifically comment these out, because we don't want to use any of these +### extra servers. + +#[object-replicator] +#vm_test_mode = yes + +#[object-reconstructor] + +#[object-updater] + +#[object-auditor] +---- + +./etc/swift/proxy-server.conf: +[source,conf] +---- +[DEFAULT] +bind_ip = 127.0.0.1 # Replace this with your node's IP address! +bind_port = 8080 +workers = 1 +user = swift +log_facility = LOG_LOCAL1 +eventlet_debug = true + +[pipeline:main] +# Yes, proxy-logging appears twice. This is so that +# middleware-originated requests get logged too. +pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain authtoken keystoneauth staticweb container-quotas account-quotas slo dlo proxy-logging proxy-server + +[filter:catch_errors] +use = egg:swift#catch_errors + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:proxy-logging] +use = egg:swift#proxy_logging + +[filter:bulk] +use = egg:swift#bulk + +[filter:ratelimit] +use = egg:swift#ratelimit + +[filter:crossdomain] +use = egg:swift#crossdomain + +[filter:dlo] +use = egg:swift#dlo + +[filter:slo] +use = egg:swift#slo + +[filter:tempurl] +use = egg:swift#tempurl + +#[filter:tempauth] +#use = egg:swift#tempauth +#user_admin_admin = admin .admin .reseller_admin +#user_test_tester = testing .admin +#user_test2_tester2 = testing2 .admin +#user_test_tester3 = testing3 + + +### Replace all of this with your Keystone server's configuration! +[filter:authtoken] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory +identity_uri = http://127.0.0.1:35357/ # Replace with Keystone server IP +admin_tenant_name = service +admin_user = swift +admin_password = password +auth_uri = http://127.0.0.1:5000/ # Replace with Keystone server IP +#cache = swift.cache +include_service_catalog = False +delay_auth_decision = True + +[filter:keystoneauth] +use = egg:swift#keystoneauth +operator_roles = admin, swiftoperator +reseller_prefix = KEY + +[filter:staticweb] +use = egg:swift#staticweb + +[filter:account-quotas] +use = egg:swift#account_quotas + +[filter:container-quotas] +use = egg:swift#container_quotas + +[filter:cache] +use = egg:swift#memcache + +[filter:gatekeeper] +use = egg:swift#gatekeeper + +[app:proxy-server] +use = egg:swift#proxy +allow_account_management = true +account_autocreate = true +---- + +After editing the configuration scripts, it is time to generate the Swift +server rings. There is a script to automate doing this in +doc/saio/bin/remakerings that could be used, but it needs some modification +to fit the recommended configuration. + +---- +# cp doc/saio/bin/remakerings ./remakerings-for-HPSS +# vim remakerings-for-HPSS +---- + +../remakerings-for-HPSS: +[source,shell] +---- +#!/bin/bash +cd /etc/swift +rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz +swift-ring-builder object.builder create 0 1 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6010/.obfuscated 1 +#swift-ring-builder object.builder add r1z2-127.0.0.1:6020/sdb2 1 +#swift-ring-builder object.builder add r1z3-127.0.0.1:6030/sdb3 1 +#swift-ring-builder object.builder add r1z4-127.0.0.1:6040/sdb4 1 +swift-ring-builder object.builder rebalance +#swift-ring-builder object-1.builder create 10 2 1 +#swift-ring-builder object-1.builder add r1z1-127.0.0.1:6010/sdb1 1 +#swift-ring-builder object-1.builder add r1z2-127.0.0.1:6020/sdb2 1 +#swift-ring-builder object-1.builder add r1z3-127.0.0.1:6030/sdb3 1 +#swift-ring-builder object-1.builder add r1z4-127.0.0.1:6040/sdb4 1 +#swift-ring-builder object-1.builder rebalance +#swift-ring-builder object-2.builder create 10 6 1 +#swift-ring-builder object-2.builder add r1z1-127.0.0.1:6010/sdb1 1 +#swift-ring-builder object-2.builder add r1z1-127.0.0.1:6010/sdb5 1 +#swift-ring-builder object-2.builder add r1z2-127.0.0.1:6020/sdb2 1 +#swift-ring-builder object-2.builder add r1z2-127.0.0.1:6020/sdb6 1 +#swift-ring-builder object-2.builder add r1z3-127.0.0.1:6030/sdb3 1 +#swift-ring-builder object-2.builder add r1z3-127.0.0.1:6030/sdb7 1 +#swift-ring-builder object-2.builder add r1z4-127.0.0.1:6040/sdb4 1 +#swift-ring-builder object-2.builder add r1z4-127.0.0.1:6040/sdb8 1 +#swift-ring-builder object-2.builder rebalance +swift-ring-builder container.builder create 0 1 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6011/metadata 1 +#swift-ring-builder container.builder add r1z2-127.0.0.1:6021/sdb2 1 +#swift-ring-builder container.builder add r1z3-127.0.0.1:6031/sdb3 1 +#swift-ring-builder container.builder add r1z4-127.0.0.1:6041/sdb4 1 +swift-ring-builder container.builder rebalance +swift-ring-builder account.builder create 0 1 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6012/metadata 1 +#swift-ring-builder account.builder add r1z2-127.0.0.1:6022/sdb2 1 +#swift-ring-builder account.builder add r1z3-127.0.0.1:6032/sdb3 1 +#swift-ring-builder account.builder add r1z4-127.0.0.1:6042/sdb4 1 +swift-ring-builder account.builder rebalance +---- + +---- +# ./remakerings-for-HPSS +---- + +Next, configure rsyslog for Swift. Swift comes with a perfectly usable +example configuration, so just copy it over. + +---- +# cp doc/saio/rsyslog.d/10-swift.conf /etc/rsyslog.d/10-swift.conf +# service syslog restart +---- + +Start up the Swift server, and make sure that it is accessible and works +properly. + +---- +# swift-init main start +---- + +[[X6]] +Where To Go From Here +--------------------- + +At this point, the SwiftOnHPSS archive machine is fully configured and ready +to develop on and use for experimental purposes. + +A command-line client can be had by installing the "python-openstackclient" +package with 'pip'. If Keystone is in use, then in order to log into Swift it +is necessary to create a Swift service user in Keystone and grant it the +'admin' role in the 'service' project in the default domain, and also to add +the Swift URL to the service endpoint catalog. + +SwiftOnHPSS now has a tool to synchronize changes made through other HPSS +interfaces with the Swift metadata databases, called 'swiftonhpss-nstool'. +It is included in the SwiftOnHPSS Git repository, and should have been +automatically installed along with the rest of the system. + +For further reading on how the Swift object storage system works and how to do +further customization, see: +https://docs.openstack.org/developer/swift/index.html \ No newline at end of file diff --git a/doc/troff/swiftonhpss-nstool.1 b/doc/troff/swiftonhpss-nstool.1 new file mode 100644 index 0000000..df4dd7e --- /dev/null +++ b/doc/troff/swiftonhpss-nstool.1 @@ -0,0 +1,35 @@ +.TH swiftonhpss-nstool 1 +.SH NAME +swiftonhpss-nstool \- Synchronizes Swift and HPSS namespaces +.SH SYNOPSIS +.B swiftonhpss-nstool +\fB-d\fR \fIDEVICE_PATH\fR +\fB-k\fR \fIKEYSTONE_URL\fR +\fB-s\fR \fISTORAGE_URL\fR +[\fB-u\fR \fIUSERNAME\fR] +.IR [account] +.IR [container] +.SH DESCRIPTION +.B swiftonhpss-nstool +synchronizes the Swift container and object metadata with the files that +actually exist in HPSS. The \fIaccount\fR and \fIcontainer\fR optional arguments +limit this synchronization to a specified account and container. +.SH OPTIONS +.TP +\fB\-d \fIdevice\fR +The location of the HPSS FUSE mountpoint Swift uses +.TP +\fB\-k \fIkeystone_url\fR +The Keystone identity service URL to auth into +.TP +\fB\-s \fIstorage_url\fR +The HTTP endpoint URL for SwiftOnHPSS +.TP +\fB\-u \fIusername\fR +The HPSS Keystone admin username +.SH LIMITATIONS +.IP \(bu +The tool only supports the Keystone v2 API for now. +.IP \(bu +The tool does not yet do any multiprocessing to speed up processing of the +file tree. \ No newline at end of file diff --git a/setup.py b/setup.py index 71886d5..b6ed314 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,9 @@ from setuptools import setup, find_packages from swiftonhpss.swift import _pkginfo - +import os +import sys +import shutil setup( name=_pkginfo.name, @@ -55,3 +57,12 @@ setup( ], }, ) + +if 'install' in sys.argv: + # Install man pages the crappy hacky way, because setuptools doesn't + # have any facility to do it. + man_path = '/usr/local/share/man/1' + man_pages = filter(lambda x: os.path.isfile('./doc/troff/'+x, + os.listdir('./doc/troff') + for page in man_pages: + shutil.copyfile('./doc/troff/'+page, man_path)