initial version of import-goal that creates the story and tasks

Change-Id: I790541f7e9badde372ac746dda295911861d25b2
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

goal_tools/import_goal.py

@@ -14,18 +14,27 @@
 
 import argparse
 import configparser
+import logging
 import os
 import os.path
+import re
 import textwrap
 
 import appdirs
+import bs4 as beautifulsoup
+import requests
 from storyboardclient.v1 import client
+import yaml
 
-_DEFAULT_URL = 'https://storyboard.openstack.org'
+_DEFAULT_URL = 'https://storyboard.openstack.org/api/v1'
+_GOVERNANCE_PROJECT_ID = 923
+_STORY_URL_TEMPLATE = 'https://storyboard.openstack.org/#!/story/{}'
+
+LOG = logging.getLogger()
 
 
 def _write_empty_config_file(filename):
-    print('Creating {}'.format(filename))
+    log.info('creating {}'.format(filename))
     cfg_dir = os.path.dirname(filename)
     if cfg_dir and not os.path.exists(cfg_dir):
         os.makedirs(cfg_dir)
@@ -36,6 +45,34 @@ def _write_empty_config_file(filename):
         access_token =
         '''.format(_DEFAULT_URL)))
 
+
+_SITE_TITLE = '— OpenStack Technical Committee Governance Documents'
+def _parse_goal_page(html):
+    data = {
+        'title': '',
+        'description': '',
+    }
+    bs = beautifulsoup.BeautifulSoup(html, 'html.parser')
+    data['title'] = bs.title.string
+    if data['title'].endswith(_SITE_TITLE):
+        data['title'] = data['title'][:-len(_SITE_TITLE)].strip()
+    data['description'] = bs.p.string
+    return data
+
+
+def _get_goal_info(url):
+    html = requests.get(url)
+    data = _parse_goal_page(html.text)
+    data['url'] = url
+    return data
+
+
+def _get_project_info(url):
+    response = requests.get(url)
+    data = yaml.safe_load(response.text)
+    return data
+
+
 def main():
     parser = argparse.ArgumentParser()
     config_dir = appdirs.user_config_dir('OSGoalTools', 'OpenStack')
@@ -45,16 +82,43 @@ def main():
         default=config_file,
         help='configuration file (%(default)s)',
     )
+    parser.add_argument(
+        '--project-list',
+        default='http://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml',
+        help='URL for projects.yaml',
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '-v',
+        help='verbose mode',
+        dest='log_level',
+        default=logging.INFO,
+        action='store_const',
+        const=logging.DEBUG,
+    )
+    group.add_argument(
+        '-q',
+        help='quiet mode',
+        dest='log_level',
+        action='store_const',
+        const=logging.WARNING,
+    )
+    parser.add_argument(
+        'goal_url',
+        help='published HTML page describing the goal',
+    )
     args = parser.parse_args()
 
-    print('Loading config from {}'.format(args.config_file))
+    logging.basicConfig(level=args.log_level, format='%(message)s')
+
+    LOG.debug('loading config from {}'.format(args.config_file))
     config = configparser.ConfigParser()
     found_config = config.read(args.config_file)
 
     if not found_config:
-        print('Could not load configuration!')
+        LOG.error('could not load configuration')
         _write_empty_config_file(args.config_file)
-        print('Please update {} and try again.'.format(args.config_file))
+        LOG.error('lease update {} and try again'.format(args.config_file))
         return 1
 
     try:
@@ -63,16 +127,60 @@ def main():
         access_token = ''
 
     if not access_token:
-        print('Could not find access_token in {}'.format(args.config_file))
-        return 1
+        parser.error('Could not find access_token in {}'.format(args.config_file))
 
     try:
-        url = config.get('DEFAULT', 'url')
+        LOG.debug('reading goal info from {}'.format(args.goal_url))
+        goal_info = _get_goal_info(args.goal_url)
+    except Exception as err:
+        parser.error(err)
+
+    try:
+        LOG.debug('reading project list from {}'.format(args.project_list))
+        project_info = _get_project_info(args.project_list)
+    except Exception as err:
+        parser.error(err)
+
+    project_names = sorted(project_info.keys(), key=lambda x: x.lower())
+
+    try:
+        storyboard_url = config.get('DEFAULT', 'url')
     except configparser.NoOptionError:
-        url = _DEFAULT_URL
-    print('Connecting to {}'.format(url))
+        storyboard_url = _DEFAULT_URL
 
-    storyboard = client.Client(url, access_token)
+    print('Connecting to {}'.format(storyboard_url))
+    storyboard = client.Client(storyboard_url, access_token)
 
-    
+    existing = storyboard.stories.get_all(title=goal_info['title'])
+    if not existing:
+        LOG.info('creating new story')
+        story = storyboard.stories.create(
+            title=goal_info['title'],
+            description=goal_info['description'] + '\n\n' + goal_info['url'],
+        )
+        LOG.info('created story {}'.format(story.id))
+    else:
+        story = existing[0]
+        LOG.info('found existing story {}'.format(story.id))
+        print(story)
+
+    # NOTE(dhellmann): After we migrate all projects to storyboard we
+    # can change this to look for tasks using the project id. Until
+    # then, all tasks are assocated with the openstack/governance
+    # project.
+    project_names_to_task = {
+        task.title: task
+        for task in story.tasks.get_all()
+    }
+
+    for project_name in project_names:
+        if project_name not in project_names_to_task:
+            LOG.info('adding task for %s', project_name)
+            storyboard.tasks.create(
+                title=project_name,
+                project_id=_GOVERNANCE_PROJECT_ID,
+                story_id=story.id,
+            )
+
+    print(_STORY_URL_TEMPLATE.format(story.id))
     return 0

requirements.txt

@@ -1,3 +1,6 @@
 # the storyboardclient lib is not on PyPI yet
 # python-storyboardclient
 appdirs>=1.4.3
+beautifulsoup4>=4.6.0
+requests
+pyyaml