Migration CLI framework

Refactored migration import script to support the following:

- Multiple (potential) import adapters.
- Source project name vs. Target project name.
- Cache of imported resources/stories to avoid duplicate imports.
- Pre-warming of tags and users to reduce points of failure during
import.

CLI Syntax:
storyboard-migrate --from-project zuul --to-project openstack-infra/zuul

The import script will complain if the project does not yet exist on
the StoryBoard side.

Change-Id: I3ae381af0323de57d46b55501448d1cd41689a54
This commit is contained in:
Michael Krotscheck 2014-09-15 14:54:43 -07:00
parent 6df6a6037f
commit b897e2e24f
8 changed files with 399 additions and 210 deletions

View File

@ -34,6 +34,7 @@ console_scripts =
storyboard-api = storyboard.api.app:start
storyboard-subscriber = storyboard.notifications.subscriber:subscribe
storyboard-db-manage = storyboard.db.migration.cli:main
storyboard-migrate = storyboard.migrate.cli:main
[build_sphinx]
source-dir = doc/source

View File

@ -1,210 +0,0 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import json
import sys
import warnings
from launchpadlib import launchpad
from openid.consumer import consumer
from openid import cryptutil
from sqlalchemy.exc import SADeprecationWarning
from storyboard.common import event_types
from storyboard.db.api import base as db_api
from storyboard.db.models import Comment
from storyboard.db.models import Story
from storyboard.db.models import StoryTag
from storyboard.db.models import Task
from storyboard.db.models import TimeLineEvent
from storyboard.db.models import User
warnings.simplefilter("ignore", SADeprecationWarning)
user_openid_map = dict()
user_obj_map = dict()
tag_obj_map = dict()
def map_openid(username):
if username is None:
return username
if username not in user_openid_map:
openid_consumer = consumer.Consumer(
dict(id=cryptutil.randomString(16, '0123456789abcdef')), None)
openid_request = openid_consumer.begin(
"https://launchpad.net/~%s" % username)
user_openid_map[username] = openid_request.endpoint.getLocalID()
return dict(name=username, openid=user_openid_map[username])
def map_priority(importance):
if importance in ('Unknown', 'Undecided', 'Medium'):
return 'medium'
elif importance in ('Critical', 'High'):
return 'high'
return 'low'
def map_status(status):
('todo', 'inprogress', 'invalid', 'review', 'merged')
if status in ('Unknown', 'New', 'Confirmed', 'Triaged'):
return 'todo'
elif status in (
'Incomplete', 'Opinion', 'Invalid', "Won't Fix", 'Expired'):
return 'invalid'
elif status == 'In Progress':
return 'inprogress'
elif status in ('Fix Committed', 'Fix Released'):
return 'merged'
def fetch_bugs(project_name='openstack-ci'):
lp = launchpad.Launchpad.login_anonymously('storyboard', 'production')
project_name = project_name
project = lp.projects[project_name]
tasks = []
for task in project.searchTasks():
messages = []
bug = task.bug
for message in bug.messages:
messages.append(dict(
author=map_openid(message.owner.name),
content=message.content,
created_at=message.date_created.strftime(
'%Y-%m-%d %H:%M:%S %z'),
))
tasks.append(dict(
bug=dict(
creator=map_openid(bug.owner.name),
title=bug.title,
description=bug.description,
created_at=bug.date_created.strftime('%Y-%m-%d %H:%M:%S %z'),
updated_at=bug.date_last_updated.strftime(
'%Y-%m-%d %H:%M:%S %z'),
is_bug=True,
tags=bug.tags,
),
task=dict(
creator=map_openid(task.owner.name),
status=map_status(task.status),
assignee=map_openid((task.assignee and task.assignee.name)),
priority=map_priority(task.importance),
created_at=task.date_created.strftime('%Y-%m-%d %H:%M:%S %z'),
),
messages=messages,
))
return tasks
def get_user(user, session):
if user is None:
return user
if user['name'] not in user_obj_map:
db_user = session.query(User).filter_by(username=user["name"]).first()
if not db_user:
db_user = User()
user.username = user['name']
user.openid = user['openid']
user.email = "%s@example.com" % user['name']
user.last_login = datetime.datetime.now()
session.add(db_user)
user_obj_map[user['name']] = db_user
return user_obj_map[user['name']]
def get_tag(tag, session):
if tag not in tag_obj_map:
db_tag = session.query(StoryTag).filter_by(name=tag).first()
if not db_tag:
db_tag = StoryTag()
db_tag.name = tag
session.add(db_tag)
tag_obj_map[tag] = db_tag
return tag_obj_map[tag]
def write_tasks(tasks):
session = db_api.get_session()
with session.begin():
for collection in tasks:
bug = collection['bug']
task = collection['task']
messages = collection['messages']
# First create the bug, then tags, then task, then comments
story_obj = Story()
story_obj.description = bug['description']
story_obj.created_at = bug['created_at']
story_obj.creator = get_user(bug['creator'], session)
story_obj.is_bug = True
story_obj.title = bug['title']
story_obj.updated_at = bug['updated_at']
session.add(story_obj)
for tag in bug['tags']:
story_obj.tags.append(get_tag(tag, session))
task_obj = Task()
task_obj.assignee = get_user(task['assignee'], session)
task_obj.created_at = bug['created_at']
task_obj.creator = get_user(bug['creator'], session)
task_obj.priority = bug['priority']
task_obj.status = bug['status']
task_obj.story = story_obj
session.add(task_obj)
for message in messages:
comment_obj = Comment()
comment_obj.content = message['content']
comment_obj.created_at = message['created_at']
session.add(comment_obj)
timeline_obj = TimeLineEvent()
timeline_obj.story = story_obj
timeline_obj.comment = comment_obj
timeline_obj.author = get_user(message['author'], session)
timeline_obj.event_type = event_types.USER_COMMENT
timeline_obj.created_at = message['created_at']
session.add(timeline_obj)
def do_load_models(project):
tasks = fetch_bugs(project)
write_tasks(tasks)
def dump_tasks(tasks, outfile):
json.dump(tasks, open(outfile, 'w'), sort_keys=True, indent=2)
def main():
dump_tasks(fetch_bugs(sys.argv[1]), sys.argv[2])
if __name__ == '__main__':
main()

View File

46
storyboard/migrate/cli.py Normal file
View File

@ -0,0 +1,46 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo.config import cfg
from storyboard.migrate.launchpad.loader import LaunchpadLoader
from storyboard.openstack.common import log
IMPORT_OPTS = [
cfg.StrOpt("from-project",
default="storyboard",
help="The name of the remote project to import."),
cfg.StrOpt("to-project",
default="openstack-infra/storyboard",
help="The local destination project for the remote stories."),
cfg.StrOpt("origin",
default="launchpad",
help="The origin system from which to import.")
]
CONF = cfg.CONF
LOG = log.getLogger(__name__)
def main():
log.setup('storyboard')
CONF.register_cli_opts(IMPORT_OPTS)
CONF(project='storyboard')
if CONF.origin is 'launchpad':
loader = LaunchpadLoader(CONF.from_project, CONF.to_project)
loader.run()
else:
print 'Unsupported import origin: %s' % CONF.origin
return

View File

View File

@ -0,0 +1,94 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import shelve
import tempfile
from storyboard.migrate.launchpad.reader import LaunchpadReader
from storyboard.migrate.launchpad.writer import LaunchpadWriter
class LaunchpadLoader(object):
def __init__(self, from_project, to_project):
"""Create a new loader instance from launchpad.org
"""
tmp_dir = tempfile.gettempdir()
self.cache = shelve.open("%s/launchpad_migrate.db" % (tmp_dir))
self.writer = LaunchpadWriter(to_project)
self.reader = LaunchpadReader(from_project)
def run(self):
for lp_bug in self.reader:
bug = lp_bug.bug
cache_key = str(unicode(bug.self_link))
if cache_key not in self.cache:
# Preload the tags.
tags = self.writer.write_tags(bug)
# Preload the story owner.
owner = self.writer.write_user(bug.owner)
# Preload the story's assignee (stored on lp_bug, not bug).
if hasattr(lp_bug, 'assignee') and lp_bug.assignee:
assignee = self.writer.write_user(lp_bug.assignee)
else:
assignee = None
# Preload the story discussion participants.
for message in bug.messages:
self.writer.write_user(message.owner)
# Write the bug.
priority = map_lp_priority(lp_bug.importance)
status = map_lp_status(lp_bug.status)
story = self.writer.write_bug(bug=bug,
owner=owner,
assignee=assignee,
priority=priority,
status=status,
tags=tags)
# Cache things.
self.cache[cache_key] = story.id
def map_lp_priority(lp_priority):
"""Map a launchpad priority to a storyboard priority.
"""
if lp_priority in ('Unknown', 'Undecided', 'Medium'):
return 'medium'
elif lp_priority in ('Critical', 'High'):
return 'high'
return 'low'
def map_lp_status(lp_status):
"""Map a launchpad status to a storyboard priority.
"""
# ('todo', 'inprogress', 'invalid', 'review', 'merged')
if lp_status in ('Unknown', 'New', 'Confirmed', 'Triaged'):
return 'todo'
elif lp_status in (
'Incomplete', 'Opinion', 'Invalid', "Won't Fix",
'Expired'):
return 'invalid'
elif lp_status == 'In Progress':
return 'inprogress'
elif lp_status in ('Fix Committed', 'Fix Released'):
return 'merged'
return 'invalid'

View File

@ -0,0 +1,38 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from launchpadlib import launchpad
class LaunchpadReader(object):
"""A generator that allows us to easily loop over launchpad bugs in any
given project.
"""
def __init__(self, project_name):
self.lp = launchpad.Launchpad.login_anonymously('storyboard',
'production')
self.project_name = project_name
self.project = self.lp.projects[project_name]
self.tasks = self.project.searchTasks()
self.task_iterator = self.tasks.__iter__()
def __iter__(self):
return self
def __next__(self):
return self.next()
def next(self):
return self.task_iterator.next()

View File

@ -0,0 +1,220 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import sys
from openid.consumer import consumer
from openid import cryptutil
import storyboard.common.event_types as event_types
from storyboard.db.api import base as db_api
from storyboard.db.api import comments as comments_api
from storyboard.db.api import projects as projects_api
from storyboard.db.api import tags as tags_api
from storyboard.db.api import users as users_api
from storyboard.db.models import Story
from storyboard.db.models import Task
from storyboard.db.models import TimeLineEvent
class LaunchpadWriter(object):
def __init__(self, project_name):
"""Create a new instance of the launchpad-to-storyboard data writer.
"""
# username -> openid
self._openid_map = dict()
# openid -> SB User Entity
self._user_map = dict()
# tag_name -> SB StoryTag Entity
self._tag_map = dict()
# SB Project Entity + Sanity check.
self.project = projects_api.project_get_by_name(project_name)
if not self.project:
print "Local project %s not found in storyboard, please create " \
"it first." % (project_name)
sys.exit(1)
def write_tags(self, bug):
"""Extracts the tags from a launchpad bug, seeds/loads them in the
StoryBoard database, and returns a list of the corresponding entities.
"""
tags = list()
# Make sure the tags field exists and has some content.
if hasattr(bug, 'tags') and bug.tags:
for tag_name in bug.tags:
tags.append(self.build_tag(tag_name))
return tags
def build_tag(self, tag_name):
"""Retrieve the SQLAlchemy record for the given tag name, creating it
if necessary.
:param tag_name: Name of the tag to retrieve and/or create.
:return: The SQLAlchemy entity corresponding to the tag name.
"""
if tag_name not in self._tag_map:
# Does it exist in the database?
tag = tags_api.tag_get_by_name(tag_name)
if not tag:
# Go ahead and create it.
print "Importing tag '%s'" % (tag_name)
tag = tags_api.tag_create(dict(
name=tag_name
))
# Add it to our memory cache
self._tag_map[tag_name] = tag
return self._tag_map[tag_name]
def write_user(self, lp_user):
"""Writes a launchpad user record into our user cache, resolving the
openid if necessary.
:param lp_user: The launchpad user record.
:return: The SQLAlchemy entity for the user record.
"""
if lp_user is None:
return lp_user
username = lp_user.name
display_name = lp_user.display_name
# Resolve the openid.
if username not in self._openid_map:
openid_consumer = consumer.Consumer(
dict(id=cryptutil.randomString(16, '0123456789abcdef')), None)
openid_request = openid_consumer.begin(lp_user.web_link)
openid = openid_request.endpoint.getLocalID()
self._openid_map[username] = openid
openid = self._openid_map[username]
# Resolve the user record from the openid.
if openid not in self._user_map:
# Check for the user, create if new.
user = users_api.user_get_by_openid(openid)
if not user:
print "Importing user '%s'" % (username)
# Use a temporary email address, since LP won't give this to
# us and it'll be updated on first login anyway.
user = users_api.user_create({
'username': username,
'openid': openid,
'full_name': display_name,
'email': "%s@example.com" % (username)
})
self._user_map[openid] = user
return self._user_map[openid]
def write_bug(self, owner, assignee, priority, status, tags, bug):
"""Writes the story, task, task history, and conversation.
:param owner: The bug owner SQLAlchemy entity.
:param tags: The tag SQLAlchemy entities.
:param bug: The Launchpad Bug record.
"""
if hasattr(bug, 'date_created'):
created_at = bug.date_created.strftime('%Y-%m-%d %H:%M:%S')
else:
created_at = None
if hasattr(bug, 'date_last_updated'):
updated_at = bug.date_last_updated.strftime('%Y-%m-%d %H:%M:%S')
else:
updated_at = None
print "Importing %s" % (bug.self_link)
story = db_api.entity_create(Story, {
'description': bug.description,
'created_at': created_at,
'creator': owner,
'is_bug': True,
'title': bug.title,
'updated_at': updated_at,
'tags': tags
})
task = db_api.entity_create(Task, {
'title': bug.title,
'assignee_id': assignee.id if assignee else None,
'project_id': self.project.id,
'story_id': story.id,
'created_at': created_at,
'updated_at': updated_at,
'priority': priority,
'status': status
})
# Create the creation event for the story manually, so we don't trigger
# event notifications.
db_api.entity_create(TimeLineEvent, {
'story_id': story.id,
'author_id': owner.id,
'event_type': event_types.STORY_CREATED,
'created_at': created_at
})
# Create the creation event for the task.
db_api.entity_create(TimeLineEvent, {
'story_id': story.id,
'author_id': owner.id,
'event_type': event_types.TASK_CREATED,
'created_at': created_at,
'event_info': json.dumps({
'task_id': task.id,
'task_title': task.title
})
})
# Create the discussion.
comment_count = 0
for message in bug.messages:
message_created_at = message.date_created \
.strftime('%Y-%m-%d %H:%M:%S')
message_owner = self.write_user(message.owner)
comment = comments_api.comment_create({
'content': message.content,
'created_at': message_created_at
})
db_api.entity_create(TimeLineEvent, {
'story_id': story.id,
'author_id': message_owner.id,
'event_type': event_types.USER_COMMENT,
'comment_id': comment.id,
'created_at': message_created_at
})
comment_count += 1
print '- Imported %d comments\r' % (comment_count),
# Advance the stdout line
print ''
return story