Skip to content

Commit 13df666

Browse files
lasotelasote
authored
Add elasticsearch (conan-io#1329)
* conan new template (conan-io#1286) * conan new template * added template description * fix code-block * Add detected_os description (conan-io#1276) * Add detected_os description Signed-off-by: Uilian Ries <uilianries@gmail.com> * Update reference/tools.rst Co-Authored-By: Javier G. Sogo <jgsogo@gmail.com> * workpsace yaml list (conan-io#1288) * Adding docs from host-specific proxies (conan-io#1241) * document QNX Neutrino support (conan-io#1290) * - document QNX Neutrino support Signed-off-by: SSE4 <tomskside@gmail.com> * Update integrations/qnx_neutrino.rst Co-Authored-By: Daniel <danimanzaneque@gmail.com> * - update settings.yml Signed-off-by: SSE4 <tomskside@gmail.com> * change default for CMake build_helper (conan-io#1292) * add docs about pre/post_package_info (conan-io#1293) * Add an example of removing system requirements via a wildcard (conan-io#1294) * Add notes about new config global variables CONAN_RETRY and CONAN_RETRY_WAIT (conan-io#1295) * fixed commands help update (Tried linux) * updated conan new --file docs * Deprecated -p in conan upload (conan-io#1300) * Add documentation for tools.to_android_abi (conan-io#1102) * Added missing bits of docs for Android flags (conan-io#1301) * update help messages according to conan/#4896 (conan-io#1285) * cascade policy (conan-io#1296) * Purge 'export-pkg' reference docs (conan-io#1232) * purge 'export-pkg' reference docs * package folder * write a couple of examples! * change workspace by working folder * typos related to formatting * minor fix * Feature/commands help update (conan-io#1299) * Commands help update * Added editable and workspace * Update commands * clarify behavior of package() (conan-io#1304) * conan new jinja template (conan-io#1306) * Add know pip installation issue (conan-io#1311) * Add know pip installation issue * Rephrase * Update installation.rst Co-Authored-By: Javier G. Sogo <jgsogo@gmail.com> * Integrations refactor (conan-io#1308) * Integrations * Rename and fixed link * Fix link * Update integrations/custom.rst Co-Authored-By: Daniel <danimanzaneque@gmail.com> * Update integrations/vcs/git.rst Co-Authored-By: Javier G. Sogo <jgsogo@gmail.com> * Update integrations.rst Co-Authored-By: Javier G. Sogo <jgsogo@gmail.com> * Update integrations/build_system.rst Co-Authored-By: Javier G. Sogo <jgsogo@gmail.com> * Update integrations/cross_platform.rst Co-Authored-By: Daniel <danimanzaneque@gmail.com> * Update integrations/vcs/svn.rst Co-Authored-By: Javier G. Sogo <jgsogo@gmail.com> * Update integrations/linting/binary.rst Co-Authored-By: SSE4 <tomskside@gmail.com> * Msbuild and visual studio, clion update * Same as clion * Redirects * Fixed symlinks * Rewrite of the SCM optimization tip box (conan-io#1307) * Rewrite of the SCM optimization tip box * review * Update creating_packages/package_repo.rst Co-Authored-By: Javier G. Sogo <jgsogo@gmail.com> * WIP elastic * Improvements * elastic * sss * gather logs * json gather * Real index * Ignore failures deleting index * Deploy complete * Fix * Fix themes * Stash layot * Add all
1 parent c51aa2c commit 13df666

File tree

13 files changed

+465
-8
lines changed

13 files changed

+465
-8
lines changed

‎_elastic/indexer.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import os
2+
import json
3+
import re
4+
5+
import boto3
6+
import elasticsearch
7+
from elasticsearch import Elasticsearch, RequestsHttpConnection
8+
from elasticsearch.helpers import bulk
9+
from requests_aws4auth import AWS4Auth
10+
11+
12+
def build_documents(version, build_folder):
13+
for root, _, files in os.walk(build_folder):
14+
for filename in files:
15+
if not filename.endswith(".fjson"):
16+
continue
17+
abs_path = os.path.join(root, filename)
18+
with open(abs_path, "r") as f:
19+
data = json.load(f)
20+
if not data.get("title"):
21+
continue
22+
title = data["title"]
23+
slug = data["current_page_name"] + ".html"
24+
parent_title = data["parents"][0]["title"] if data["parents"] else ""
25+
html = data["body"]
26+
element = {"version": version, "title": title, "parent_title": parent_title,
27+
"slug": slug, "html": html}
28+
yield element
29+
30+
31+
class ElasticManager(object):
32+
33+
def __init__(self, host, region):
34+
credentials = boto3.Session().get_credentials()
35+
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, "es")
36+
37+
es = Elasticsearch(
38+
hosts=[{'host': host, 'port': 443}],
39+
http_auth=awsauth,
40+
use_ssl=True,
41+
verify_certs=True,
42+
connection_class=RequestsHttpConnection
43+
)
44+
45+
self.es = es
46+
47+
def _gendata(self, version, folder):
48+
data = build_documents(version, folder)
49+
for doc in data:
50+
doc["_index"] = "docs"
51+
doc["_type"] = "docs"
52+
yield doc
53+
54+
def create_index(self):
55+
doc = """
56+
{
57+
"settings": {
58+
"analysis": {
59+
"analyzer": {
60+
"htmlStripAnalyzer": {
61+
"type": "custom",
62+
"tokenizer": "standard",
63+
"filter": ["standard","lowercase"],
64+
"char_filter": [
65+
"html_strip"
66+
]
67+
}
68+
}
69+
}
70+
},
71+
"mappings": {
72+
"docs": {
73+
"properties": {
74+
"html": {"type": "text", "analyzer": "htmlStripAnalyzer"},
75+
"title" : { "type" : "text" },
76+
"parent_title" : { "type" : "text" },
77+
"version": {"type": "text"},
78+
"url" : { "type" : "text" }
79+
}
80+
}
81+
}
82+
}
83+
"""
84+
self.es.indices.create(index="docs", body=doc)
85+
86+
def remove_index(self):
87+
try:
88+
self.es.indices.delete(index="docs")
89+
except elasticsearch.exceptions.NotFoundError:
90+
pass
91+
92+
def index(self, version, folder):
93+
bulk(self.es, self._gendata(version, folder))
94+

‎_elastic/query_examples/index.txt

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
curl -X DELETE "localhost:9200/docs"
2+
3+
curl -X PUT "localhost:9200/docs" -H 'Content-Type: application/json' -d'
4+
{
5+
"settings": {
6+
"analysis": {
7+
"analyzer": {
8+
"htmlStripAnalyzer": {
9+
"type": "custom",
10+
"tokenizer": "standard",
11+
"filter": ["standard","lowercase"],
12+
"char_filter": [
13+
"html_strip"
14+
]
15+
}
16+
}
17+
}
18+
},
19+
"mappings": {
20+
"docs": {
21+
"properties": {
22+
"html": {"type": "text", "analyzer": "htmlStripAnalyzer"},
23+
"title" : { "type" : "text" },
24+
"parent_title" : { "type" : "text" },
25+
"version": {"type": "text"},
26+
"url" : { "type" : "text" }
27+
}
28+
}
29+
}
30+
}'
31+
32+
33+
34+

‎_elastic/query_examples/insert_example.txt

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
curl -X POST "localhost:9200/docs/_doc/integrations\/custom" -H 'Content-Type: application/json' -d'
2+
{
3+
"version": "1.15",
4+
"parent_title" : "Integrations",
5+
"url": "integrations/custom",
6+
"title" : "Custom integrations",
7+
"html" : "trying out Elasticsearch"
8+
}
9+
'
10+
11+
12+
13+
curl -X POST "localhost:9200/docs/_doc/" -H 'Content-Type: application/json' -d'
14+
{
15+
"version": "1.15",
16+
"parent_title" : "Integrations",
17+
"url": "integrations/custom",
18+
"title" : "Custom integrations",
19+
"html" : "trying out integrations"
20+
}
21+
'
22+
23+
24+
25+
curl -X POST "localhost:9200/docs/_doc/" -H 'Content-Type: application/json' -d'
26+
{
27+
"version": "1.15",
28+
"parent_title" : "Integrations",
29+
"url": "integrations/custom",
30+
"title" : "Patata",
31+
"html" : "trying out integrations"
32+
}
33+
'
34+
35+
36+
curl -X POST "localhost:9200/docs/_doc/" -H 'Content-Type: application/json' -d'
37+
{
38+
"version": "1.15",
39+
"parent_title" : "Integrations",
40+
"url": "integrations/custom",
41+
"title" : "Other title",
42+
"html" : "trying out integrations patata"
43+
}
44+
'
45+

‎_elastic/query_examples/query.txt

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
curl -X GET "localhost:9200/docs/_search" -H 'Content-Type: application/json' -d'
2+
{
3+
"from" : 0, "size" : 5,
4+
"query": {
5+
"bool": {
6+
"filter": [
7+
{ "match": { "version": "1.15"}}
8+
],
9+
"should": [
10+
{ "match": {
11+
"html": {
12+
"query": "patata",
13+
"boost": 1
14+
}
15+
}},
16+
{ "match": {
17+
"title": {
18+
"query": "patata",
19+
"boost": 4
20+
}
21+
}}
22+
]
23+
}
24+
}
25+
}
26+
'
27+
28+
29+
30+
curl -X GET "localhost:9200/docs/_search" -H 'Content-Type: application/json' -d'
31+
{
32+
"query": {
33+
"match" : {
34+
"version" : "1.15"
35+
}
36+
}
37+
}
38+
'

‎_themes/conan/breadcrumbs.html

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
{% set display_gitlab = True %}
2525
{% endif %}
2626

27-
<div role="navigation" aria-label="breadcrumbs navigation">
27+
<div role="navigation" aria-label="breadcrumbs navigation" id="breadcrumbs">
2828

2929
<ul class="wy-breadcrumbs">
3030
{% block breadcrumbs %}
@@ -78,5 +78,4 @@
7878
{% endif %}
7979
</div>
8080
{% endif %}
81-
<hr/>
8281
</div>

‎_themes/conan/layout.html

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@
4949
<link rel="stylesheet" href="{{ pathto(cssfile, 1) }}" type="text/css" />
5050
{% endfor %}
5151

52+
<link href="https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.7/css/select2.min.css" rel="stylesheet" />
53+
<link rel="stylesheet" href="{{ pathto('_static/css/' +'search.css', 1) }}" type="text/css" />
54+
5255
{%- block linktags %}
5356
{%- if hasdoc('about') %}
5457
<link rel="author" title="{{ _('About these documents') }}"
@@ -78,7 +81,6 @@
7881
{%- block extrahead %} {% endblock %}
7982

8083
{# Keep modernizr in head - http://modernizr.com/docs/#installing #}
81-
<script src="{{ pathto('_static/js/modernizr.min.js', 1) }}"></script>
8284

8385
</head>
8486

@@ -148,9 +150,14 @@
148150

149151

150152
{# PAGE CONTENT #}
153+
<script>
154+
var reldir = "{{ pathto("index") }}".replace("index.html","").replace("#", "");
155+
</script>
151156
<div class="wy-nav-content">
152157
<div class="rst-content">
153158
{% include "breadcrumbs.html" %}
159+
<select id="search" name="state" multiple="true" onchange="window.location = reldir + this.value;" style="width:90%"></select>
160+
<hr/>
154161
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
155162
<div itemprop="articleBody">
156163
{% block body %}{% endblock %}
@@ -191,6 +198,14 @@
191198
<script type="text/javascript" src="{{ pathto('_static/js/theme.js', 1) }}"></script>
192199
{% endif %}
193200

201+
<script
202+
src="https://code.jquery.com/jquery-3.4.1.min.js"
203+
integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo="
204+
crossorigin="anonymous"></script>
205+
<script src="{{ pathto('_static/js/modernizr.min.js', 1) }}"></script>
206+
<script src="{{ pathto('_static/js/search.js', 1) }}"></script>
207+
<script src="https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.7/js/select2.min.js"></script>
208+
194209
{# STICKY NAVIGATION #}
195210
{% if theme_sticky_navigation %}
196211
<script type="text/javascript">
@@ -200,6 +215,11 @@
200215
</script>
201216
{% endif %}
202217

218+
219+
<script>
220+
initSearch("{{ current_version }}");
221+
</script>
222+
203223
{%- block footer %} {% endblock %}
204224

205225
</body>

‎_themes/conan/searchbox.html

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
{%- if builder != 'singlehtml' %}
2-
<div role="search">
2+
<!--<div role="search">
33
<form id="rtd-search-form" class="wy-form" action="{{ pathto('search') }}" method="get">
44
<input type="text" name="q" placeholder="{{ _('Search docs') }}" />
55
<input type="hidden" name="check_keywords" value="yes" />
66
<input type="hidden" name="area" value="default" />
77
</form>
8-
</div>
8+
</div>-->
99
{%- endif %}

0 commit comments

Comments
 (0)