Fix case insensitive search

Search on keyword can be made case insensitive by using a normalizer.
But some fields with type keyword include several words.
They are only returned by entering all words. This is not what
most users expect.

It's better to use the "text" type by default (for text fields) and add
an extra "keyword" field of type "keyword" for sorting.

JIRA INFRA-770

app/models.py

@@ -607,19 +607,19 @@ class Item(CreatedMixin, SearchableMixin, db.Model):
     __mapping__ = {
         "created_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
         "updated_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
-        "user": {"type": "keyword"},
-        "ics_id": {"type": "keyword"},
-        "serial_number": {"type": "keyword"},
+        "user": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "ics_id": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "serial_number": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
         "quantity": {"type": "long"},
-        "manufacturer": {"type": "keyword"},
-        "model": {"type": "keyword"},
-        "location": {"type": "keyword"},
-        "status": {"type": "keyword"},
-        "parent": {"type": "keyword"},
-        "children": {"type": "keyword"},
-        "macs": {"type": "keyword"},
-        "host": {"type": "keyword"},
-        "stack_member": {"type": "keyword"},
+        "manufacturer": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "model": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "location": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "status": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "parent": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "children": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "macs": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "host": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "stack_member": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
         "history": {"enabled": False},
         "comments": {"type": "text"},
     }
@@ -1090,36 +1090,39 @@ class Host(CreatedMixin, SearchableMixin, db.Model):
     __mapping__ = {
         "created_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
         "updated_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
-        "user": {"type": "keyword"},
-        "name": {"type": "keyword"},
-        "fqdn": {"type": "keyword"},
+        "user": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "name": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "fqdn": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
         "is_ioc": {"type": "boolean"},
-        "device_type": {"type": "keyword"},
-        "model": {"type": "keyword"},
-        "description": {"type": "text"},
-        "items": {"type": "keyword"},
+        "device_type": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "model": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "description": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+        "items": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
         "interfaces": {
             "properties": {
                 "id": {"enabled": False},
                 "created_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
                 "updated_at": {"type": "date", "format": "yyyy-MM-dd HH:mm"},
-                "user": {"type": "keyword"},
+                "user": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
                 "is_main": {"type": "boolean"},
-                "network": {"type": "keyword"},
+                "network": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
                 "ip": {"type": "ip"},
                 "netmask": {"enabled": False},
-                "name": {"type": "keyword"},
-                "mac": {"type": "keyword"},
-                "host": {"type": "keyword"},
-                "cnames": {"type": "keyword"},
-                "domain": {"type": "keyword"},
-                "tags": {"type": "keyword"},
-                "device_type": {"type": "keyword"},
-                "model": {"type": "keyword"},
+                "name": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+                "mac": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+                "host": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+                "cnames": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+                "domain": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+                "tags": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+                "device_type": {
+                    "type": "text",
+                    "fields": {"keyword": {"type": "keyword"}},
+                },
+                "model": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
             }
         },
         "ansible_vars": {"enabled": False},
-        "ansible_groups": {"type": "keyword"},
+        "ansible_groups": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
     }
 
     # id shall be defined here to be used by SQLAlchemy-Continuum

app/utils.py

@@ -311,9 +311,11 @@ def retrieve_data_for_datatables(values, model):
         order_dir = values.get("order[0][dir]", "asc")
         # Sorting can be done directly on all fields of type
         # keyword/date/long
-        # If we want to sort on fields of type text, we should
-        # add an extra .keyword field!
-        sort = f"{name}:{order_dir}"
+        # To sort on fields of type text, we use the extra .keyword field
+        if name in ("created_at", "updated_at", "quantity"):
+            sort = f"{name}:{order_dir}"
+        else:
+            sort = f"{name}.keyword:{order_dir}"
     instances, nb_filtered = model.search(
         search, page=page, per_page=per_page, sort=sort
     )

tests/functional/test_web.py

@@ -180,6 +180,36 @@ def test_retrieve_items_sort(logged_client, item_factory):
     ]
 
 
+def test_retrieve_items_case_insensitive(logged_client, model_factory, item_factory):
+    juniper_model = model_factory(name="Juniper")
+    item_factory(serial_number="BBB001", model=juniper_model)
+    item_factory(serial_number="ABB042")
+    response = logged_client.post(
+        "/inventory/_retrieve_items",
+        data={"draw": "50", "length": 20, "start": 0, "search[value]": "juniper"},
+    )
+    r = response.get_json()
+    assert r["recordsTotal"] == 2
+    assert r["recordsFiltered"] == 1
+    assert len(r["data"]) == 1
+    assert r["data"][0]["model"] == "Juniper"
+
+
+def test_retrieve_items_one_word(logged_client, manufacturer_factory, item_factory):
+    manufacturer = manufacturer_factory(name="Concurrent Technologies")
+    item_factory(serial_number="AAA001", manufacturer=manufacturer)
+    item_factory(serial_number="ABB042")
+    response = logged_client.post(
+        "/inventory/_retrieve_items",
+        data={"draw": "50", "length": 20, "start": 0, "search[value]": "concurrent"},
+    )
+    r = response.get_json()
+    assert r["recordsTotal"] == 2
+    assert r["recordsFiltered"] == 1
+    assert len(r["data"]) == 1
+    assert r["data"][0]["manufacturer"] == "Concurrent Technologies"
+
+
 def test_generate_random_mac(logged_client):
     response = logged_client.get("/network/_generate_random_mac")
     mac = response.get_json()["data"]["mac"]