Merge branch 'master' into develop

Author: danimtb

_elastic/indexer.py

@@ -0,0 +1,95 @@
+import json
+import os
+
+import boto3
+import elasticsearch
+from elasticsearch import Elasticsearch, RequestsHttpConnection
+from elasticsearch.helpers import bulk
+from requests_aws4auth import AWS4Auth
+
+
+def build_documents(version, build_folder):
+    for root, _, files in os.walk(build_folder):
+        for filename in files:
+            if not filename.endswith(".fjson"):
+                continue
+            abs_path = os.path.join(root, filename)
+            with open(abs_path, "r") as f:
+                data = json.load(f)
+                if not data.get("title"):
+                    continue
+                title = data["title"]
+                slug = data["current_page_name"] + ".html"
+                parent_title = data["parents"][0]["title"] if data["parents"] else ""
+                html = data["body"]
+                element = {"version": version, "title": title, "parent_title": parent_title,
+                           "slug": slug, "html": html}
+                yield element
+
+
+class ElasticManager(object):
+
+    def __init__(self, host, region):
+        credentials = boto3.Session().get_credentials()
+        awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, "es")
+
+        es = Elasticsearch(
+            hosts=[{'host': host, 'port': 443}],
+            http_auth=awsauth,
+            use_ssl=True,
+            verify_certs=True,
+            connection_class=RequestsHttpConnection
+        )
+
+        self.es = es
+
+    def _gendata(self, version, folder):
+        data = build_documents(version, folder)
+        for doc in data:
+            doc["_index"] = "docs"
+            doc["_type"] = "docs"
+            yield doc
+
+    def create_index(self):
+        doc = """
+{
+    "settings": {
+        "analysis": {
+          "analyzer": {
+            "htmlStripAnalyzer": {
+              "type": "custom",
+              "tokenizer": "standard",
+              "filter": ["standard","lowercase"],
+              "char_filter": [
+                "html_strip"
+              ]
+            }
+          }
+        }
+    },
+    "mappings": {
+      "docs": {
+        "properties": {
+          "html": {"type": "text", "analyzer": "htmlStripAnalyzer"},
+          "title" : { "type" : "text" },
+          "parent_title" : { "type" : "text" },
+          "version": {"type": "text"},
+          "url" : { "type" : "text" }
+        }
+      }
+    }
+}
+"""
+        self.es.indices.create(index="docs", body=doc)
+
+    def remove_index(self):
+        try:
+            self.es.indices.delete(index="docs")
+        except elasticsearch.exceptions.NotFoundError:
+            pass
+
+    def index(self, version, folder):
+        bulk(self.es, self._gendata(version, folder))
+
+    def ping(self):
+        return self.es.info()

_elastic/query_examples/index.txt

@@ -0,0 +1,34 @@
+curl -X DELETE "localhost:9200/docs"
+
+curl -X PUT "localhost:9200/docs" -H 'Content-Type: application/json' -d'
+{
+    "settings": {
+        "analysis": {
+          "analyzer": {
+            "htmlStripAnalyzer": {
+              "type": "custom",
+              "tokenizer": "standard",
+              "filter": ["standard","lowercase"],
+              "char_filter": [
+                "html_strip"
+              ]
+            }
+          }
+        }
+    },
+    "mappings": {
+      "docs": {
+        "properties": {
+          "html": {"type": "text", "analyzer": "htmlStripAnalyzer"},
+          "title" : { "type" : "text" },
+          "parent_title" : { "type" : "text" },
+          "version": {"type": "text"},
+          "url" : { "type" : "text" }
+        }
+      }
+    }
+}'
+
+
+
+

_elastic/query_examples/insert_example.txt

@@ -0,0 +1,45 @@
+curl -X POST "localhost:9200/docs/_doc/integrations\/custom" -H 'Content-Type: application/json' -d'
+{
+    "version": "1.15",
+    "parent_title" : "Integrations",
+    "url": "integrations/custom",
+    "title" : "Custom integrations",
+    "html" : "trying out Elasticsearch"
+}
+'
+
+
+
+curl -X POST "localhost:9200/docs/_doc/" -H 'Content-Type: application/json' -d'
+{
+    "version": "1.15",
+    "parent_title" : "Integrations",
+    "url": "integrations/custom",
+    "title" : "Custom integrations",
+    "html" : "trying out integrations"
+}
+'
+
+
+
+curl -X POST "localhost:9200/docs/_doc/" -H 'Content-Type: application/json' -d'
+{
+    "version": "1.15",
+    "parent_title" : "Integrations",
+    "url": "integrations/custom",
+    "title" : "Patata",
+    "html" : "trying out integrations"
+}
+'
+
+
+curl -X POST "localhost:9200/docs/_doc/" -H 'Content-Type: application/json' -d'
+{
+    "version": "1.15",
+    "parent_title" : "Integrations",
+    "url": "integrations/custom",
+    "title" : "Other title",
+    "html" : "trying out integrations patata"
+}
+'
+

_elastic/query_examples/query.txt

@@ -0,0 +1,38 @@
+curl -X GET "localhost:9200/docs/_search" -H 'Content-Type: application/json' -d'
+{
+    "from" : 0, "size" : 5,
+    "query": {
+        "bool": {
+           "filter": [
+                { "match": { "version":  "1.15"}}
+           ],
+           "should": [
+                { "match": {
+                    "html": {
+                        "query": "patata",
+                        "boost": 1
+                    }
+                }},
+                { "match": {
+                    "title": {
+                        "query": "patata",
+                        "boost": 4
+                    }
+                }}
+           ]
+        }
+    }
+}
+'
+
+
+
+curl -X GET "localhost:9200/docs/_search" -H 'Content-Type: application/json' -d'
+{
+    "query": {
+        "match" : {
+            "version" : "1.15"
+        }
+    }
+}
+'

_themes/conan/breadcrumbs.html

@@ -24,7 +24,7 @@
 {% set display_gitlab = True %}
 {% endif %}
 
-<div role="navigation" aria-label="breadcrumbs navigation">
+<div role="navigation" aria-label="breadcrumbs navigation" id="breadcrumbs">
 
   <ul class="wy-breadcrumbs">
     {% block breadcrumbs %}
@@ -78,5 +78,4 @@
       {% endif %}
   </div>
   {% endif %}
-  <hr/>
 </div>

_themes/conan/layout.html

@@ -49,6 +49,9 @@
     <link rel="stylesheet" href="{{ pathto(cssfile, 1) }}" type="text/css" />
   {% endfor %}
 
+  <link href="https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.7/css/select2.min.css" rel="stylesheet" />
+  <link rel="stylesheet" href="{{ pathto('_static/css/' +'search.css', 1) }}" type="text/css" />
+
   {%- block linktags %}
     {%- if hasdoc('about') %}
         <link rel="author" title="{{ _('About these documents') }}"
@@ -78,7 +81,6 @@
   {%- block extrahead %} {% endblock %}
 
   {# Keep modernizr in head - http://modernizr.com/docs/#installing #}
-  <script src="{{ pathto('_static/js/modernizr.min.js', 1) }}"></script>
 
 </head>
 
@@ -148,9 +150,14 @@
 
 
       {# PAGE CONTENT #}
+      <script>
+        var reldir = "{{ pathto("index") }}".replace("index.html","").replace("#", "");
+      </script>
       <div class="wy-nav-content">
         <div class="rst-content">
           {% include "breadcrumbs.html" %}
+          <select id="search" name="state" multiple="true" onchange="window.location = reldir + this.value;" style="width:90%"></select>
+          <hr/>
           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
            <div itemprop="articleBody">
             {% block body %}{% endblock %}
@@ -191,6 +198,14 @@
     <script type="text/javascript" src="{{ pathto('_static/js/theme.js', 1) }}"></script>
   {% endif %}
 
+   <script
+  src="https://code.jquery.com/jquery-3.4.1.min.js"
+  integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo="
+  crossorigin="anonymous"></script>
+  <script src="{{ pathto('_static/js/modernizr.min.js', 1) }}"></script>
+  <script src="{{ pathto('_static/js/search.js', 1) }}"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.7/js/select2.min.js"></script>
+
   {# STICKY NAVIGATION #}
   {% if theme_sticky_navigation %}
   <script type="text/javascript">
@@ -200,6 +215,11 @@
   </script>
   {% endif %}
 
+
+  <script>
+    initSearch("{{ current_version }}");
+  </script>
+
   {%- block footer %} {% endblock %}
 
 </body>

_themes/conan/searchbox.html

@@ -1,9 +1,9 @@
 {%- if builder != 'singlehtml' %}
-<div role="search">
+<!--<div role="search">
   <form id="rtd-search-form" class="wy-form" action="{{ pathto('search') }}" method="get">
     <input type="text" name="q" placeholder="{{ _('Search docs') }}" />
     <input type="hidden" name="check_keywords" value="yes" />
     <input type="hidden" name="area" value="default" />
   </form>
-</div>
+</div>-->
 {%- endif %}