Explicitely define the elasticsearch mapping

Dynamic mapping set to "strict" so that we don't forget to update the
mapping when adding new fields to a model.

JIRA INFRA-723 #action In Progress

app/models.py

@@ -436,10 +436,37 @@ class SearchableMixin(object):
             search.remove_from_index(index, id)
         session._changes = None
 
+    @classmethod
+    def delete_index(cls, **kwargs):
+        """Delete the index of the class"""
+        current_app.logger.info(f"Delete the {cls.__tablename__} index")
+        search.delete_index(
+            cls.__tablename__ + current_app.config["ELASTICSEARCH_INDEX_SUFFIX"],
+            **kwargs,
+        )
+
+    @classmethod
+    def create_index(cls, **kwargs):
+        """Create the index of the class"""
+        if hasattr(cls, "__mapping__"):
+            current_app.logger.info(f"Create the {cls.__tablename__} index")
+            search.create_index(
+                cls.__tablename__ + current_app.config["ELASTICSEARCH_INDEX_SUFFIX"],
+                cls.__mapping__,
+                **kwargs,
+            )
+        else:
+            current_app.logger.info(
+                f"No mapping defined for {cls.__tablename__}. No index created."
+            )
+
     @classmethod
     def reindex(cls):
         """Force to reindex all instances of the class"""
         current_app.logger.info(f"Force to re-index all {cls.__tablename__} instances")
+        # Ignore index_not_found_exception
+        cls.delete_index(ignore_unavailable=True)
+        cls.create_index()
         for obj in cls.query:
             search.add_to_index(
                 cls.__tablename__ + current_app.config["ELASTICSEARCH_INDEX_SUFFIX"],
@@ -577,6 +604,25 @@ class Item(CreatedMixin, SearchableMixin, db.Model):
             "model_id",
         ]
     }
+    __mapping__ = {
+        "created_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
+        "updated_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
+        "user": {"type": "keyword"},
+        "ics_id": {"type": "keyword"},
+        "serial_number": {"type": "keyword"},
+        "quantity": {"type": "long"},
+        "manufacturer": {"type": "keyword"},
+        "model": {"type": "keyword"},
+        "location": {"type": "keyword"},
+        "status": {"type": "keyword"},
+        "parent": {"type": "keyword"},
+        "children": {"type": "keyword"},
+        "macs": {"type": "keyword"},
+        "host": {"type": "keyword"},
+        "stack_member": {"type": "keyword"},
+        "history": {"enabled": False},
+        "comments": {"type": "text"},
+    }
 
     # WARNING! Inheriting id from CreatedMixin doesn't play well with
     # SQLAlchemy-Continuum. It has to be defined here.
@@ -1041,6 +1087,40 @@ class AnsibleGroup(CreatedMixin, db.Model):
 
 class Host(CreatedMixin, SearchableMixin, db.Model):
     __versioned__ = {}
+    __mapping__ = {
+        "created_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
+        "updated_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
+        "user": {"type": "keyword"},
+        "name": {"type": "keyword"},
+        "fqdn": {"type": "keyword"},
+        "is_ioc": {"type": "boolean"},
+        "device_type": {"type": "keyword"},
+        "model": {"type": "keyword"},
+        "description": {"type": "text"},
+        "items": {"type": "keyword"},
+        "interfaces": {
+            "properties": {
+                "id": {"enabled": False},
+                "created_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
+                "updated_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
+                "user": {"type": "keyword"},
+                "is_main": {"type": "boolean"},
+                "network": {"type": "keyword"},
+                "ip": {"type": "ip"},
+                "netmask": {"enabled": False},
+                "name": {"type": "keyword"},
+                "mac": {"type": "keyword"},
+                "host": {"type": "keyword"},
+                "cnames": {"type": "keyword"},
+                "domain": {"type": "keyword"},
+                "tags": {"type": "keyword"},
+                "device_type": {"type": "keyword"},
+                "model": {"type": "keyword"},
+            }
+        },
+        "ansible_vars": {"enabled": False},
+        "ansible_groups": {"type": "keyword"},
+    }
 
     # id shall be defined here to be used by SQLAlchemy-Continuum
     id = db.Column(db.Integer, primary_key=True)

app/search.py

@@ -12,6 +12,21 @@ This module implements the search interface.
 from flask import current_app
 
 
+def delete_index(index, **kwargs):
+    """Delete the given index"""
+    if not current_app.elasticsearch:
+        return
+    current_app.elasticsearch.indices.delete(index=index, **kwargs)
+
+
+def create_index(index, mapping, **kwargs):
+    """Create an index with the given mapping"""
+    if not current_app.elasticsearch:
+        return
+    body = {"mappings": {"_doc": {"dynamic": "strict", "properties": mapping}}}
+    current_app.elasticsearch.indices.create(index=index, body=body, **kwargs)
+
+
 def add_to_index(index, body):
     """Add a document to an index"""
     if not current_app.elasticsearch:

app/utils.py

@@ -309,15 +309,11 @@ def retrieve_data_for_datatables(values, model):
     else:
         name = values.get(f"columns[{order_column}][data]")
         order_dir = values.get("order[0][dir]", "asc")
-        # For all fields of type text, sorting should be done
-        # on .keyword in elasticsearch
-        # quantity (in items table) is of type integer
-        # It's hard-coded here for now. If needed we can try to find a
-        # generic way to find this information.
-        if name == "quantity":
-            sort = f"{name}:{order_dir}"
-        else:
-            sort = f"{name}.keyword:{order_dir}"
+        # Sorting can be done directly on all fields of type
+        # keyword/date/long
+        # If we want to sort on fields of type text, we should
+        # add an extra .keyword field!
+        sort = f"{name}:{order_dir}"
     instances, nb_filtered = model.search(
         search, page=page, per_page=per_page, sort=sort
     )

tests/functional/conftest.py

@@ -15,7 +15,7 @@ from pytest_factoryboy import register
 from flask_ldap3_login import AuthenticationResponse, AuthenticationResponseStatus
 from app.factory import create_app
 from app.extensions import db as _db
-from app.models import SearchableMixin
+from app.models import SearchableMixin, Host, Item
 from . import common, factories
 
 register(factories.UserFactory)
@@ -124,6 +124,10 @@ def session(db, request):
     db.event.listen(session(), "after_commit", SearchableMixin.after_commit)
     db.session = session
 
+    # Create the elasticsearch indices
+    Item.create_index()
+    Host.create_index()
+
     yield session
 
     # ELASTICSEARCH_INDEX_SUFFIX is set to "-test"