330 lines
12 KiB
Python
330 lines
12 KiB
Python
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
|
# use this file except in compliance with the License. You may obtain a copy
|
|
# of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
import uuid
|
|
|
|
from openid.consumer import consumer
|
|
from openid.consumer.discover import DiscoveryFailure
|
|
from openid import cryptutil
|
|
|
|
import storyboard.common.event_types as event_types
|
|
from storyboard.db.api import base as db_api
|
|
from storyboard.db.models import Branch
|
|
from storyboard.db.models import Comment
|
|
from storyboard.db.models import Project
|
|
from storyboard.db.models import Story
|
|
from storyboard.db.models import StoryTag
|
|
from storyboard.db.models import Task
|
|
from storyboard.db.models import TimeLineEvent
|
|
from storyboard.db.models import User
|
|
|
|
|
|
class LaunchpadWriter(object):
|
|
def __init__(self, project_name):
|
|
"""Create a new instance of the launchpad-to-storyboard data writer.
|
|
"""
|
|
|
|
# username -> openid
|
|
self._openid_map = dict()
|
|
# openid -> SB User Entity
|
|
self._user_map = dict()
|
|
# tag_name -> SB StoryTag Entity
|
|
self._tag_map = dict()
|
|
# Build a session for the writer
|
|
self.session = db_api.get_session(in_request=False)
|
|
|
|
# SB Project Entity + Sanity check.
|
|
self.project = db_api.model_query(Project, self.session) \
|
|
.filter_by(name=project_name) \
|
|
.first()
|
|
if not self.project:
|
|
print("Local project %s not found in storyboard, please create \
|
|
it first." % (project_name))
|
|
sys.exit(1)
|
|
|
|
self.branch = db_api.model_query(Branch, self.session) \
|
|
.filter_by(project_id=self.project.id, name='master') \
|
|
.first()
|
|
if not self.branch:
|
|
print("No master branch found for project %s, please create \
|
|
one first." % (project_name))
|
|
sys.exit(1)
|
|
|
|
def write_tags(self, bug):
|
|
"""Extracts the tags from a launchpad bug, seeds/loads them in the
|
|
StoryBoard database, and returns a list of the corresponding entities.
|
|
"""
|
|
tags = list()
|
|
|
|
# Make sure the tags field exists and has some content.
|
|
if hasattr(bug, 'tags') and bug.tags:
|
|
for tag_name in bug.tags:
|
|
tags.append(self.build_tag(tag_name))
|
|
|
|
return tags
|
|
|
|
def build_tag(self, tag_name):
|
|
"""Retrieve the SQLAlchemy record for the given tag name, creating it
|
|
if necessary.
|
|
|
|
:param tag_name: Name of the tag to retrieve and/or create.
|
|
:return: The SQLAlchemy entity corresponding to the tag name.
|
|
"""
|
|
if tag_name not in self._tag_map:
|
|
|
|
# Does it exist in the database?
|
|
tag = db_api.model_query(StoryTag, self.session) \
|
|
.filter_by(name=tag_name) \
|
|
.first()
|
|
|
|
if not tag:
|
|
# Go ahead and create it.
|
|
print("Importing tag '%s'" % tag_name)
|
|
tag = db_api.entity_create(StoryTag, {
|
|
'name': tag_name
|
|
}, session=self.session)
|
|
|
|
# Add it to our memory cache
|
|
self._tag_map[tag_name] = tag
|
|
|
|
return self._tag_map[tag_name]
|
|
|
|
def write_user(self, lp_user):
|
|
"""Writes a launchpad user record into our user cache, resolving the
|
|
openid if necessary.
|
|
|
|
:param lp_user: The launchpad user record.
|
|
:return: The SQLAlchemy entity for the user record.
|
|
"""
|
|
if lp_user is None:
|
|
return lp_user
|
|
|
|
display_name = lp_user.display_name
|
|
user_link = lp_user.web_link
|
|
|
|
# Resolve the openid.
|
|
if user_link not in self._openid_map:
|
|
try:
|
|
openid_consumer = consumer.Consumer(
|
|
dict(id=cryptutil.randomString(16, '0123456789abcdef')),
|
|
None)
|
|
openid_request = openid_consumer.begin(user_link)
|
|
openid = openid_request.endpoint.getLocalID()
|
|
|
|
self._openid_map[user_link] = openid
|
|
except DiscoveryFailure:
|
|
# If we encounter a launchpad maintenance user,
|
|
# give it an invalid openid.
|
|
print("WARNING: Invalid OpenID for user \'%s\'"
|
|
% (display_name,))
|
|
self._openid_map[user_link] = \
|
|
'http://example.com/invalid/~%s' % (display_name,)
|
|
|
|
openid = self._openid_map[user_link]
|
|
|
|
# Resolve the user record from the openid.
|
|
if openid not in self._user_map:
|
|
|
|
# Check for the user, create if new.
|
|
user = db_api.model_query(User, self.session) \
|
|
.filter_by(openid=openid) \
|
|
.first()
|
|
if not user:
|
|
print("Importing user '%s'" % (user_link))
|
|
|
|
# Use a temporary email address, since LP won't give this to
|
|
# us and it'll be updated on first login anyway.
|
|
user = db_api.entity_create(User, {
|
|
'openid': openid,
|
|
'full_name': display_name,
|
|
'email': "%s-%s@example.com" % (display_name, uuid.uuid4())
|
|
}, session=self.session)
|
|
|
|
self._user_map[openid] = user
|
|
|
|
return self._user_map[openid]
|
|
|
|
def check_branch(self, branch):
|
|
|
|
#Look in db for branches that are in project
|
|
#if branch is in project return True
|
|
exists = (db_api.model_query(Branch, self.session)
|
|
.filter_by(project_id=self.project.id)
|
|
.filter_by(name=branch).all())
|
|
return exists
|
|
|
|
def get_branch(self, branch):
|
|
result = (db_api.model_query(Branch, self.session)
|
|
.filter_by(project_id=self.project.id)
|
|
.filter_by(name=branch)
|
|
.first())
|
|
return result
|
|
|
|
def write_bug(self, owner, assignee, priority, status, tags, bug,
|
|
branches):
|
|
"""Writes the story, task, task history, and conversation.
|
|
|
|
:param owner: The bug owner SQLAlchemy entity.
|
|
:param tags: The tag SQLAlchemy entities.
|
|
:param bug: The Launchpad Bug record.
|
|
"""
|
|
#Checks to make sure that the branch for the bug exists
|
|
for branch in branches:
|
|
if not self.check_branch(branch):
|
|
print('No %s branch found for %s project. Creating one now.' %
|
|
(branch, self.project.name))
|
|
db_api.entity_create(Branch, {
|
|
'name': branch,
|
|
'project_id': self.project.id
|
|
}, session=self.session)
|
|
|
|
if hasattr(bug, 'date_created'):
|
|
created_at = bug.date_created
|
|
else:
|
|
created_at = None
|
|
|
|
if hasattr(bug, 'date_last_updated'):
|
|
updated_at = bug.date_last_updated
|
|
else:
|
|
updated_at = None
|
|
|
|
# Extract the launchpad ID from the self link.
|
|
# example url: https://api.launchpad.net/1.0/bugs/1057477
|
|
url_match = re.search("([0-9]+)$", str(bug.self_link))
|
|
if not url_match:
|
|
print('ERROR: Unable to extract launchpad ID from %s.'
|
|
% (bug.self_link,))
|
|
print('ERROR: Please file a ticket.')
|
|
return
|
|
launchpad_id = int(url_match.groups()[0])
|
|
|
|
# If the title is too long, prepend it to the description and
|
|
# truncate it.
|
|
title = bug.title
|
|
description = bug.description
|
|
|
|
if len(title) > 100:
|
|
title = title[:97] + '...'
|
|
description = bug.title + '\n\n' + description
|
|
|
|
# Sanity check.
|
|
story = {
|
|
'id': launchpad_id,
|
|
'description': description,
|
|
'created_at': created_at,
|
|
'creator': owner,
|
|
'is_bug': True,
|
|
'title': title,
|
|
'updated_at': updated_at,
|
|
'tags': tags
|
|
}
|
|
duplicate = db_api.entity_get(Story, launchpad_id,
|
|
session=self.session)
|
|
if not duplicate:
|
|
print("Importing Story: %s" % (bug.self_link,))
|
|
story = db_api.entity_create(Story, story, session=self.session)
|
|
else:
|
|
print("Existing Story: %s" % (bug.self_link,))
|
|
story = duplicate
|
|
|
|
# Duplicate check- launchpad import creates one task per story,
|
|
# so if we already have a project task on this story, skip it. This
|
|
# is to properly replay imports in the case where errors occurred
|
|
# during import.
|
|
existing_task = db_api.model_query(Task, session=self.session) \
|
|
.filter(Task.story_id == launchpad_id) \
|
|
.filter(Task.project_id == self.project.id) \
|
|
.first()
|
|
if not existing_task:
|
|
print("- Adding task in project %s" % (self.project.name,))
|
|
|
|
for branch in branches:
|
|
task = db_api.entity_create(Task, {
|
|
'title': title,
|
|
'assignee_id': assignee.id if assignee else None,
|
|
'project_id': self.project.id,
|
|
'branch_id': self.get_branch(branch).id,
|
|
'story_id': launchpad_id,
|
|
'created_at': created_at,
|
|
'updated_at': updated_at,
|
|
'priority': priority,
|
|
'status': status
|
|
}, session=self.session)
|
|
else:
|
|
print("- Existing task in %s" % (self.project.name,))
|
|
task = existing_task
|
|
|
|
# Duplication Check - If this story already has a creation event,
|
|
# we don't need to create a new one. Otherwise, create it manually so
|
|
# we don't trigger event notifications.
|
|
story_created_event = db_api \
|
|
.model_query(TimeLineEvent, session=self.session) \
|
|
.filter(TimeLineEvent.story_id == launchpad_id) \
|
|
.filter(TimeLineEvent.event_type == event_types.STORY_CREATED) \
|
|
.first()
|
|
if not story_created_event:
|
|
print("- Generating story creation event")
|
|
db_api.entity_create(TimeLineEvent, {
|
|
'story_id': launchpad_id,
|
|
'author_id': owner.id,
|
|
'event_type': event_types.STORY_CREATED,
|
|
'created_at': created_at
|
|
}, session=self.session)
|
|
|
|
# Create the creation event for the task, but only if we just created
|
|
# a new task.
|
|
if not existing_task:
|
|
print("- Generating task creation event")
|
|
db_api.entity_create(TimeLineEvent, {
|
|
'story_id': launchpad_id,
|
|
'author_id': owner.id,
|
|
'event_type': event_types.TASK_CREATED,
|
|
'created_at': created_at,
|
|
'event_info': json.dumps({
|
|
'task_id': task.id,
|
|
'task_title': title
|
|
})
|
|
}, session=self.session)
|
|
|
|
# Create the discussion, loading any existing comments first.
|
|
current_count = db_api \
|
|
.model_query(TimeLineEvent, session=self.session) \
|
|
.filter(TimeLineEvent.story_id == launchpad_id) \
|
|
.filter(TimeLineEvent.event_type == event_types.USER_COMMENT) \
|
|
.count()
|
|
desired_count = len(bug.messages)
|
|
print("- %s of %s comments already imported." % (current_count,
|
|
desired_count))
|
|
for i in range(current_count, desired_count):
|
|
print('- Importing comment %s of %s' % (i + 1, desired_count))
|
|
message = bug.messages[i]
|
|
message_created_at = message.date_created
|
|
message_owner = self.write_user(message.owner)
|
|
|
|
comment = db_api.entity_create(Comment, {
|
|
'content': message.content,
|
|
'created_at': message_created_at
|
|
}, session=self.session)
|
|
|
|
db_api.entity_create(TimeLineEvent, {
|
|
'story_id': launchpad_id,
|
|
'author_id': message_owner.id,
|
|
'event_type': event_types.USER_COMMENT,
|
|
'comment_id': comment.id,
|
|
'created_at': message_created_at
|
|
}, session=self.session)
|