Upgrade Elasticsearch to version 6.x

readthedocs/projects/apps.py

@@ -9,5 +9,7 @@ class ProjectsConfig(AppConfig):
     def ready(self):
         from readthedocs.projects import tasks
         from readthedocs.worker import app
+        from .signals import pre_save_html_file #noqa
+
         app.tasks.register(tasks.SyncRepositoryTask)
         app.tasks.register(tasks.UpdateDocsTask)

readthedocs/projects/managers.py

@@ -0,0 +1,7 @@
+from django.db import models
+
+
+class HTMLFileManager(models.Manager):
+
+    def get_queryset(self):
+        return super(HTMLFileManager, self).get_queryset().filter(is_html=True)

readthedocs/projects/models.py

@@ -7,8 +7,8 @@
 import fnmatch
 import logging
 import os
-from builtins import object  # pylint: disable=redefined-builtin
 
+from builtins import object  # pylint: disable=redefined-builtin
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.urlresolvers import NoReverseMatch, reverse
@@ -24,6 +24,7 @@
 from readthedocs.core.utils import broadcast, slugify
 from readthedocs.projects import constants
 from readthedocs.projects.exceptions import ProjectConfigurationError
+from readthedocs.projects.managers import HTMLFileManager
 from readthedocs.projects.querysets import (
     ChildRelatedProjectQuerySet, FeatureQuerySet, ProjectQuerySet,
     RelatedProjectQuerySet)
@@ -902,6 +903,7 @@ class ImportedFile(models.Model):
     path = models.CharField(_('Path'), max_length=255)
     md5 = models.CharField(_('MD5 checksum'), max_length=255)
     commit = models.CharField(_('Commit'), max_length=255)
+    is_html = models.BooleanField(default=False)
 
     def get_absolute_url(self):
         return resolve(project=self.project, version_slug=self.version.slug, filename=self.path)
@@ -910,6 +912,20 @@ def __str__(self):
         return '%s: %s' % (self.name, self.project)
 
 
+class HTMLFile(ImportedFile):
+
+    """
+    Imported HTML file Proxy model.
+
+    This tracks only the HTML files for indexing to search.
+    """
+
+    class Meta(object):
+        proxy = True
+
+    objects = HTMLFileManager()
+
+
 class Notification(models.Model):
     project = models.ForeignKey(Project,
                                 related_name='%(class)s_notifications')

readthedocs/projects/signals.py

@@ -2,10 +2,11 @@
 
 from __future__ import absolute_import
 import django.dispatch
+from django.db.models.signals import pre_save
 from django.dispatch import receiver
 
 from readthedocs.oauth.utils import attach_webhook
-
+from .models import HTMLFile
 
 before_vcs = django.dispatch.Signal(providing_args=["version"])
 after_vcs = django.dispatch.Signal(providing_args=["version"])
@@ -25,3 +26,8 @@ def handle_project_import(sender, **kwargs):
     request = kwargs.get('request')
 
     attach_webhook(project=project, request=request)
+
+
+@receiver(pre_save, sender=HTMLFile)
+def pre_save_html_file(sender, instance, *args, **kwargs):
+    instance.is_html = True

readthedocs/projects/tasks.py

@@ -8,6 +8,7 @@
 from __future__ import absolute_import
 
 import datetime
+import fnmatch
 import hashlib
 import json
 import logging
@@ -29,7 +30,7 @@
 
 from .constants import LOG_TEMPLATE
 from .exceptions import RepositoryError
-from .models import ImportedFile, Project, Domain
+from .models import ImportedFile, Project, Domain, HTMLFile
 from .signals import before_vcs, after_vcs, before_build, after_build, files_changed
 from readthedocs.builds.constants import (LATEST,
                                           BUILD_STATE_CLONING,
@@ -943,18 +944,23 @@ def _manage_imported_files(version, path, commit):
     changed_files = set()
     for root, __, filenames in os.walk(path):
         for filename in filenames:
+            if fnmatch.fnmatch(filename, '*.html'):
+                model_class = HTMLFile
+            else:
+                model_class = ImportedFile
+
             dirpath = os.path.join(root.replace(path, '').lstrip('/'),
                                    filename.lstrip('/'))
             full_path = os.path.join(root, filename)
             md5 = hashlib.md5(open(full_path, 'rb').read()).hexdigest()
             try:
-                obj, __ = ImportedFile.objects.get_or_create(
+                obj, __ = model_class.objects.get_or_create(
                     project=version.project,
                     version=version,
                     path=dirpath,
                     name=filename,
                 )
-            except ImportedFile.MultipleObjectsReturned:
+            except model_class.MultipleObjectsReturned:
                 log.warning('Error creating ImportedFile')
                 continue
             if obj.md5 != md5:
@@ -963,6 +969,12 @@ def _manage_imported_files(version, path, commit):
             if obj.commit != commit:
                 obj.commit = commit
             obj.save()
+
+    # Delete the HTMLFile first
+    HTMLFile.objects.filter(project=version.project,
+                            version=version
+                            ).exclude(commit=commit).delete()
+
     # Delete ImportedFiles from previous versions
     ImportedFile.objects.filter(project=version.project,
                                 version=version
@@ -1173,7 +1185,7 @@ def sync_callback(_, version_pk, commit, *args, **kwargs):
     The first argument is the result from previous tasks, which we discard.
     """
     fileify(version_pk, commit=commit)
-    update_search(version_pk, commit=commit)
+    # update_search(version_pk, commit=commit)
 
 
 @app.task()

readthedocs/search/documents.py

@@ -1,6 +1,6 @@
 from django_elasticsearch_dsl import DocType, Index, fields
 
-from readthedocs.projects.models import Project
+from readthedocs.projects.models import Project, HTMLFile
 
 from readthedocs.search.faceted_search import ProjectSearch
 
@@ -43,3 +43,18 @@ def faceted_search(cls, query, language=None, using=None, index=None):
             kwargs['filters'] = {'language': language}
 
         return ProjectSearch(**kwargs)
+
+
+page_index = Index('page')
+
+page_index.settings(
+    number_of_shards=1,
+    number_of_replicas=0
+)
+
+
+@page_index.doc_type
+class PageDocument(DocType):
+
+    class Meta(object):
+        model = HTMLFile