add lunr.py to parse HTML to create JSON for Lunr search

Change-Id: I44aff67391fa6f925b8cf09f787d7ec251faef0c
This commit is contained in:
Jim Phillips 2015-09-14 11:42:38 -04:00
parent a1bb884922
commit d49fa250e3
5 changed files with 479 additions and 2 deletions

View File

@ -61,7 +61,7 @@ images: $(PDFs)
all: clean html dirhtml singlehtml latexpdf pdf
html: images
rawhtml: images
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
@ -177,6 +177,11 @@ doctest:
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
html: rawhtml
./lunr.py
@echo
@echo "Lunr search index complete"
SPELL = aspell
ASPELLOPTS = --dont-backup -d en --personal=.aspell_en.wordlist
RSTS := $(shell find pages/ -type f -name '*.rst')

View File

@ -0,0 +1,413 @@
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>Search &mdash; Mirantis OpenStack v6.1 | Documentation</title>
<link rel="stylesheet" href="_static/basic.css" type="text/css" />
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css" type="text/css" />
<link rel="stylesheet" href="_static/bootstrap-sphinx.css" type="text/css" />
<link rel="stylesheet" href="_static/styles.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: './',
VERSION: '6.1',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
<script type="text/javascript" src="_static/underscore.js"></script>
<script type="text/javascript" src="_static/doctools.js"></script>
<script type="text/javascript" src="_static/searchtools.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/modernizr/2.8.3/modernizr.js"></script>
<script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/lunr.js/0.5.11/lunr.min.js"></script>
<link rel="shortcut icon" href="_static/mirantis_icon.ico"/>
<link rel="top" title="Mirantis OpenStack v6.1 | Documentation" href="contents.html" />
<script type="text/javascript">
jQuery(function() { Search.loadIndex("searchindex.js"); });
</script>
<script type="text/javascript" id="searchindexloader"></script>
<style>
.loader {
margin: 100px auto 0;
width: 70px;
text-align: center;
}
.loader > div {
width: 18px;
height: 18px;
background-color: #333;
border-radius: 100%;
display: inline-block;
-webkit-animation: search-bouncedelay 1.4s infinite ease-in-out both;
animation: search-bouncedelay 1.4s infinite ease-in-out both;
}
.loader .bounce1 {
-webkit-animation-delay: -0.32s;
animation-delay: -0.32s;
}
.loader .bounce2 {
-webkit-animation-delay: -0.16s;
animation-delay: -0.16s;
}
@-webkit-keyframes search-bouncedelay {
0%, 80%, 100% { -webkit-transform: scale(0) }
40% { -webkit-transform: scale(1.0) }
}
@keyframes search-bouncedelay {
0%, 80%, 100% {
-webkit-transform: scale(0);
transform: scale(0);
} 40% {
-webkit-transform: scale(1.0);
transform: scale(1.0);
}
}
#search-results-lunr {
margin-top: 35px;
margin-bottom: 50px;
}
</style>
<meta charset='utf-8'>
<meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
<meta name="apple-mobile-web-app-capable" content="yes">
</head>
<body role="document">
<div id="navbar" class="navbar navbar-default">
<div class="container">
<div class="navbar-header">
<!-- .btn-navbar is used as the toggle for collapsed navbar content -->
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="index.html" title="Mirantis OpenStack"><img src="_static/mirantis_logo.png">
</a>
<span id="site-title">Documentation</a>
</div>a
</div>
</div>
<div class="container">
<div class="row">
<div class="col-md-12">
<h1 id="search-documentation">Search</h1>
<div id="fallback" class="admonition warning">
<script type="text/javascript">$('#fallback').hide();</script>
<p>
Please activate JavaScript to enable the search
functionality.
</p>
</div>
<p>
From here you can search these documents. Enter your search
words into the box below and click "search". Note that the search
function will automatically search for all of the words. Pages
containing fewer words won't appear in the result list.
</p>
<form class="form-inline" action="" method="get">
<div class="form-group">
<input type="text" class="form-control search-field" name="q" value="" />
</div>
<input type="submit" class="btn btn-default" value="search" />
<span id="search-progress" style="padding-left: 10px"></span>
</form>
<div class="row">
<div class="col-md-8">
<div id="search-results-lunr">
</div>
</div>
</div>
</div>
</div>
</div>
<script>
function getParameterByName(name) {
name = name.replace(/[\[]/, "\\[").replace(/[\]]/, "\\]");
var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"),
results = regex.exec(location.search);
return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " "));
}
$(document).ready(function () {
var index = lunr(function () {
this.field('title', {boost: 10});
this.field('body');
this.field('href');
this.ref('id');
});
var store = {};
if (getParameterByName('q')) {
$('#search-results-lunr').append('<div class="loader"><div class="bounce1"></div><div class="bounce2"></div><div class="bounce3"></div></div>');
$.getJSON("_static/data.json", function (data) {
$(data).each(function (i, item) {
index.add({
title: item.title,
body: item.body,
href: item.url,
guide: item.guide,
id: i
});
store[i] = {title: item.title, body: item.body, guide: item.guide, href: item.url};
});
var query = getParameterByName('q');
var results = index.search('fuel');
var length = 210;
var bodyText = '';
var results = index.search(query);
$('#search-progress').hide();
$('#search-results-lunr').empty().append(
results.length ?
results.map(function (result) {
var el = $('<p>')
.append($('<a>')
.attr('href', store[result.ref].href)
.text(store[result.ref].title)
);
var body = store[result.ref].body.toLowerCase();
var bodySearch = body.search(query.toLowerCase());
if (bodySearch > 100) {
bodyText = jQuery.trim(store[result.ref].body).substring(bodySearch, bodySearch + length).split(" ").slice(0, -1).join(" ") + "...";
} else {
bodyText = jQuery.trim(store[result.ref].body).substring(0, length).split(" ").slice(0, -1).join(" ") + "...";
}
el.append($('<p>').text());
el.append($('<p>').html('<span class="text-muted">Guide: <em>' + store[result.ref].guide + '</em></span><br>' + bodyText));
return el;
}) : $('<p><strong>No results found</strong></p>')
);
});
}
});
</script>
<footer>
<section class="fsprod">
<div class="container">
<div class="row">
<div class="col-md-6">
<ul style="margin-bottom:0;">
<li><a class="fmenu_section" href="https://www.mirantis.com/products/mirantis-openstack-software/"
style="margin-top: 15px;">Products</a></li>
</ul>
<div class="row">
<div class="col-md-4 col-sm-4 col-xs-4"><a href="https://www.mirantis.com/products/mirantis-openstack-software/"
class="fproduct">
<div class="footerglyph1"></div>
<span>Mirantis <br>OpenStack Software</span></a></div>
<div class="col-md-4 col-sm-4 col-xs-4"><a href="https://www.mirantis.com/products/application-catalog-murano/"
class="fproduct">
<div class="footerglyph2"></div>
<span>Application <br>Catalog Murano</span></a></div>
<div class="col-md-4 col-sm-4 col-xs-4"><a href="https://www.mirantis.com/products/openstack-drivers-and-plugins/"
class="fproduct">
<div class="footerglyph3"></div>
<span>OpenStack Drivers <br>and Plugins</span></a></div>
</div>
</div>
<hr class="hidden-lg hidden-md">
<div class="col-md-3 col-sm-6 col-xs-6 hidden-sm hidden-xs ">
<ul>
<li><a class="fmenu_section" style="margin-top: 15px;">Solutions</a></li>
<li><a href="https://www.mirantis.com/solutions/dev-test/">Dev/Test</a></li>
<li><a href="https://www.mirantis.com/solutions/platform-as_a_service_paas/">PaaS</a></li>
<li><a href="https://www.mirantis.com/solutions/container-technologies/">Containers</a></li>
<li><a href="https://www.mirantis.com/solutions/hybrid-cloud/">Hybrid Cloud</a></li>
<li><a href="https://www.mirantis.com/solutions/industry-solutions/">Industry Solutions</a></li>
</ul>
</div>
<div class="col-md-3 col-sm-6 col-xs-6">
<ul>
<li><a class="fmenu_section" style="margin-top: 15px;">Get Started</a></li>
<li><a href="https://www.mirantis.com/get-started">Quick Start Guide</a></li>
<li><a href="https://www.mirantis.com/products/mirantis-openstack-software/">Download and
Install</a></li>
<li><a href="https://www.mirantis.com/?s=mox+tutorial&amp;order=asc">Developer Tutorials</a>
</li>
<li><a href="https://www.mirantis.com/products/mirantis-openstack-software/documentation/">Documentation</a>
</li>
</ul>
</div>
<div class="hidden-lg hidden-md col-sm-6 col-xs-6">
<ul>
<li><a class="fmenu_section" href="https://www.mirantis.com/company/about-pure-play-openstack/"
style="margin-top: 15px;">Company</a></li>
<li><a href="https://online.mirantis.com/contact-us">Get In Touch</a></li>
<li><a href="https://www.mirantis.com/contact/">Locations</a></li>
<li><a href="https://www.mirantis.com/careers/">Careers</a></li>
<li><a href="https://www.mirantis.com/company/leadership/">Management</a></li>
<li><a href="https://www.mirantis.com/company/investors/">Investors</a></li>
<li><a href="https://www.mirantis.com/company/leadership/board-of-directors/">Board of Directors</a></li>
<li><a href="https://www.mirantis.com/company/press-center/info/">Press Center</a></li>
</ul>
</div>
</div>
<hr class="hidden-lg hidden-md hidden-sm hidden-xs">
</div>
</section>
<section class="fscomp">
<div class="container footsm hidden-sm hidden-xs">
<div class="row">
<div class="col-md-2">
<ul>
<li><a class="fmenu_section" style="font-weight:normal;font-size: 1.1em;">Services</a></li>
<li><a href="https://www.mirantis.com/services/managed-services/">Managed Services</a></li>
<li><a href="https://www.mirantis.com/services/assessment/">Assessment</a></li>
<li><a href="https://www.mirantis.com/services/implementation/">Implementation</a></li>
<li><a href="https://www.mirantis.com/services/enterprise-support-services/">Enterprise Support</a></li>
<li><a href="https://www.mirantis.com/services/migration-services/">Migration Services</a></li>
<li><a href="https://www.mirantis.com/services/drivers-upstreaming-maintenance/">Driver Development</a></li>
</ul>
</div>
<div class="col-md-2 hidden-sm">
<ul>
<li><a class="fmenu_section" style="font-weight:normal;font-size: 1.1em;">Training</a></li>
<li><a href="https://training.mirantis.com/openstack-courses">OpenStack Courses</a></li>
<li><a href="https://training.mirantis.com/certification">Certification</a></li>
<li><a href="https://training.mirantis.com/online-virtual-training">Virtual Training</a></li>
<li><a href="https://training.mirantis.com/training/onsite.html">OnSite Training</a></li>
<li><a href="https://training.mirantis.com/online-resources">Training Resources</a></li>
</ul>
</div>
<div class="col-md-2 hidden-sm">
<ul>
<li><a class="fmenu_section" style="font-weight:normal;font-size: 1.1em;">Partners</a></li>
<li><a href="https://www.mirantis.com/partners/">Overview</a></li>
<li><a href="https://www.mirantis.com/partners/mirantis-unlocked-partner-solutions/">Partner list</a></li>
<li><a href="https://www.mirantis.com/partners/mirantis-channel-partners/">Channel Partners</a></li>
<li><a href="https://www.mirantis.com/partners/service-providers/">Service Providers</a></li>
<li><a href="https://training.mirantis.com/mirantis-training-partner-program-intro/">Training
Partners</a></li>
<li><a href="https://www.mirantis.com/partners/become-mirantis-technology-partner/">Become a Partner</a></li>
</ul>
</div>
<div class="col-md-3 hidden-sm">
<ul>
<li><a class="fmenu_section" style="font-weight:normal;font-size: 1.1em;">OpenStack Now</a></li>
<li><a href="https://www.mirantis.com/blog/">Blog</a></li>
<li><a href="https://www.mirantis.com/openstack/">News</a></li>
<li><a href="https://www.mirantis.com/openstack/tutorials/">Tutorials</a></li>
<li><a href="https://www.mirantis.com/openstack/news/community-news/">Community/Events</a></li>
<li><a href="http://online.mirantis.com/openstack-now-newsletter/">Subscribe</a></li>
</ul>
</div>
<div class="col-md-3 hidden-sm">
<ul>
<li><a class="fmenu_section" href="https://www.mirantis.com/company/about-pure-play-openstack/"
style="font-weight:normal;font-size: 1.1em;">Company</a></li>
<li><a href="https://online.mirantis.com/contact-us">Get In Touch</a></li>
<li><a href="https://www.mirantis.com/contact/">Locations</a></li>
<li><a href="https://www.mirantis.com/careers/">Careers</a></li>
<li><a href="https://www.mirantis.com/company/leadership/">Management</a></li>
<li><a href="https://www.mirantis.com/company/investors/">Investors</a></li>
<li><a href="https://www.mirantis.com/company/leadership/board-of-directors/">Board of Directors</a></li>
<li><a href="https://www.mirantis.com/company/press-center/info/">Press Center</a></li>
</ul>
</div>
</div>
</div>
</section>
<div class="clear"></div>
<div class="footer_copy">
<div class="container">
<div class="row">
<div class="col-md-3 col-sm-3 col-logo text-left-lg text-left-md text-left-sm text-center-xs">
<span class="mira_btm">Mirantis Inc.</span> <br>
<span class="pureplay_btm">Pure Play OpenStack.</span>
</div>
<div class="col-md-4 col-sm-5 col-copy text-center">
<span class="copy_btm">&copy; 2005 &mdash; 2015 Mirantis, Inc.<br><span
style="font-size: 10px; opacity:0.6; line-height: 0.9em; display:inline-block;margin-top:5px; ">All rights reserved. "Mirantis" and "FUEL" are are registered trademarks of Mirantis, Inc. All other trademarks are the property of their respective owners.</span></span>
</div>
<div class="col-md-5 col-sm-4 col-address text-right-lg text-right-md text-right-sm text-center-xs">
<span class="mira_btm">615 National Avenue, Suite 100 Mountain View, CA 94043<br>
+1-650-963-9828</span>
</div>
</div>
<div class="row">
<div class="col-md-3 col-sm-3 col-logo">
</div>
<div class="col-md-4 col-sm-4 cc">
<a rel="license" href="http://creativecommons.org/licenses/by-sa/3.0/"><img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by-sa/3.0/88x31.png"></a><br>This work is
licensed under a <a rel="license" href="http://creativecommons.org/licenses/by-sa/3.0/">Creative
Commons Attribution-ShareAlike 3.0 Unported License</a>.
</div>
</div>
</div>
</div>
</footer>
<a href="#0" class="cd-top">Top</a>
<!-- Google Tag Manager -->
<noscript>
<iframe src="//www.googletagmanager.com/ns.html?id=GTM-K2J65K"
height="0" width="0" style="display:none;visibility:hidden"></iframe>
</noscript>
<script>(function (w, d, s, l, i) {
w[l] = w[l] || [];
w[l].push({
'gtm.start': new Date().getTime(), event: 'gtm.js'
});
var f = d.getElementsByTagName(s)[0],
j = d.createElement(s), dl = l != 'dataLayer' ? '&l=' + l : '';
j.async = true;
j.src =
'//www.googletagmanager.com/gtm.js?id=' + i + dl;
f.parentNode.insertBefore(j, f);
})(window, document, 'script', 'dataLayer', 'GTM-K2J65K');</script>
<!-- End Google Tag Manager -->
<!-- RTP tag -->
<script type='text/javascript'>
(function(c,h,a,f,i){c[a]=c[a]||function(){(c[a].q=c[a].q||[]).push(arguments)};
c[a].a=i;var g=h.createElement("script");g.async=true;g.type="text/javascript";
g.src=f+'?aid='+i;var b=h.getElementsByTagName("script")[0];b.parentNode.insertBefore(g,b);
})(window,document,"rtp","//sjrtp7-cdn.marketo.com/rtp-api/v1/rtp.js","mirantis");
rtp('send','view');
rtp('get', 'campaign',true);
</script>
<!-- End of RTP tag -->
</body>
</html>

View File

@ -197,7 +197,7 @@ html_favicon = '_static/mirantis_icon.ico'
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_extra_path = ['_templates/mirantis/static/index.html']
html_extra_path = ['_templates/mirantis/static/index.html', '_templates/mirantis/static/search.html']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.

58
lunr.py Executable file
View File

@ -0,0 +1,58 @@
#!/usr/bin/env python
import glob, ntpath, random, json
from bs4 import BeautifulSoup
exclude = ['_build/html/search.html', '_build/html/index.html', '_build/html/index_content.html', '_build/html/contents.html', '_build/html/genindex.html', '_build/html/terminology.html']
files = glob.glob('_build/html/*.html')
for remove in exclude:
if remove in files: files.remove(remove)
doc = []
def formatheading(filename, headings, guide, type):
title = headings.text[:-1]
parent = headings.parent
url = filename + '#' + headings.parent.attrs['id']
if type == 'h2':
for tag in parent.find_all('h2'):
tag.replaceWith('')
for tag in parent.find_all('h3'):
tag.parent.replaceWith('')
else:
for tag in parent.find_all('h3'):
tag.replaceWith('')
for tag in parent.find_all('h4'):
tag.replaceWith('')
body = parent.get_text(" ", strip=True)
return {
"title": title,
"guide": guide,
"url": url,
"body": body.replace('\n', ' ')
}
for file in files:
filename = ntpath.basename(file)
h2 = BeautifulSoup(open(file), 'html.parser')
h3 = BeautifulSoup(open(file), 'html.parser')
for title in h2.findAll('h1'):
guide = title.text[:-1]
for headings in h3.findAll('h3'):
result = formatheading(filename, headings, guide, 'h3')
if result['body']:
doc.append(result)
for headings in h2.findAll('h2'):
result = formatheading(filename, headings, guide, 'h2')
if result['body']:
doc.append(result)
with open('_build/html/_static/data.json', 'w') as outfile:
json.dump(doc, outfile)

View File

@ -6,3 +6,4 @@ sphinxcontrib-actdiag
sphinxcontrib-seqdiag
sphinxcontrib-nwdiag
sphinxcontrib-plantuml
beautifulsoup4==4.4.0