From 6847914412c2f5811b7d47325c67c217f0f3f5af Mon Sep 17 00:00:00 2001
From: Benjamin Bertrand <benjamin.bertrand@esss.se>
Date: Tue, 17 Sep 2019 15:34:47 +0200
Subject: [PATCH] Make ansible_vars in host searchable
- Use flattened datatype (require elasticsearch 7.3)
- Upgrade to elasticsearch 7
JIRA INFRA-1295 #action In Progress
---
app/models.py | 2 +-
app/search.py | 11 +++--------
docker-compose.override.yml | 2 +-
docker-compose.yml | 2 +-
docs/network.rst | 7 +++++++
requirements-to-freeze.txt | 2 +-
requirements.txt | 2 +-
tests/functional/test_search.py | 6 +++---
8 files changed, 18 insertions(+), 16 deletions(-)
diff --git a/app/models.py b/app/models.py
index ecdb0e8..e960182 100644
--- a/app/models.py
+++ b/app/models.py
@@ -1181,7 +1181,7 @@ class Host(CreatedMixin, SearchableMixin, db.Model):
"model": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
}
},
- "ansible_vars": {"enabled": False},
+ "ansible_vars": {"type": "flattened"},
"ansible_groups": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
}
diff --git a/app/search.py b/app/search.py
index 724208f..5459872 100644
--- a/app/search.py
+++ b/app/search.py
@@ -23,7 +23,7 @@ def create_index(index, mapping, **kwargs):
"""Create an index with the given mapping"""
if not current_app.elasticsearch:
return
- body = {"mappings": {"_doc": {"dynamic": "strict", "properties": mapping}}}
+ body = {"mappings": {"dynamic": "strict", "properties": mapping}}
current_app.elasticsearch.indices.create(index=index, body=body, **kwargs)
@@ -37,7 +37,6 @@ def add_to_index(index, body):
id = body.pop("id")
current_app.elasticsearch.index(
index=index,
- doc_type="_doc",
id=id,
body=body,
refresh=current_app.config["ELASTICSEARCH_REFRESH"],
@@ -49,10 +48,7 @@ def remove_from_index(index, id):
if not current_app.elasticsearch:
return
current_app.elasticsearch.delete(
- index=index,
- doc_type="_doc",
- id=id,
- refresh=current_app.config["ELASTICSEARCH_REFRESH"],
+ index=index, id=id, refresh=current_app.config["ELASTICSEARCH_REFRESH"]
)
@@ -65,7 +61,6 @@ def query_index(index, query, page=1, per_page=20, sort=None):
return [], 0
kwargs = {
"index": index,
- "doc_type": "_doc",
"q": query,
"from_": (page - 1) * per_page,
"size": per_page,
@@ -74,4 +69,4 @@ def query_index(index, query, page=1, per_page=20, sort=None):
kwargs["sort"] = sort
search = current_app.elasticsearch.search(**kwargs)
ids = [int(hit["_id"]) for hit in search["hits"]["hits"]]
- return ids, search["hits"]["total"]
+ return ids, search["hits"]["total"]["value"]
diff --git a/docker-compose.override.yml b/docker-compose.override.yml
index 79d4364..7e98c78 100644
--- a/docker-compose.override.yml
+++ b/docker-compose.override.yml
@@ -33,7 +33,7 @@ services:
ports:
- "9200:9200"
kibana:
- image: docker.elastic.co/kibana/kibana:6.4.2
+ image: docker.elastic.co/kibana/kibana:7.3.2
ports:
- "5601:5601"
environment:
diff --git a/docker-compose.yml b/docker-compose.yml
index ac0a319..c0fe71f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -30,7 +30,7 @@ services:
image: redis:4.0
container_name: csentry_redis
elasticsearch:
- image: docker.elastic.co/elasticsearch/elasticsearch:6.4.2
+ image: docker.elastic.co/elasticsearch/elasticsearch:7.3.2
container_name: csentry_elasticsearch
environment:
- cluster.name=csentry-cluster
diff --git a/docs/network.rst b/docs/network.rst
index cb1255a..9578d53 100644
--- a/docs/network.rst
+++ b/docs/network.rst
@@ -65,6 +65,11 @@ As for the inventory :ref:`inventory-search`, Elasticsearch is used as the full-
.. image:: _static/network/search_host_mac.png
+- Ansible variables are indexed using the `flattened datatype`_.
+ This allows to search on the value of any variable like for any other field.
+ To query on a specific key, object dot notation shall be used: ``ansible_vars.local_control_room_dev_workstation:true``.
+ Searching if an Ansible variable is defined can be done using ``_exists_``: ``_exists_:ansible_vars.vm_owner``.
+
The complete list of fields that can be searched is the following:
- created_at
@@ -76,6 +81,7 @@ The complete list of fields that can be searched is the following:
- description
- items
- interfaces
+ - ansible_vars
- ansible_groups
Check the elasticsearch `query string syntax`_ for more details.
@@ -267,3 +273,4 @@ And don't forget to add the **ansible-vault** credential to your template to all
.. _Ansible vault: https://docs.ansible.com/ansible/2.6/user_guide/vault.html
.. _ansible-vault encrypt_string: https://docs.ansible.com/ansible/2.6/user_guide/vault.html#use-encrypt-string-to-create-encrypted-variables-to-embed-in-yaml
.. _query string syntax: https://www.elastic.co/guide/en/elasticsearch/reference/6.4/query-dsl-query-string-query.html#query-string-syntax
+.. _flattened datatype: https://www.elastic.co/guide/en/elasticsearch/reference/current/flattened.html
diff --git a/requirements-to-freeze.txt b/requirements-to-freeze.txt
index 4b80a1d..042e8e5 100644
--- a/requirements-to-freeze.txt
+++ b/requirements-to-freeze.txt
@@ -1,4 +1,4 @@
-elasticsearch
+elasticsearch>=7.0.0,<8.0.0
flask>=1.0.0
flask-admin
flask-caching
diff --git a/requirements.txt b/requirements.txt
index 7238748..21ed0ab 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@ certifi==2018.11.29
chardet==3.0.4
Click==7.0
colorama==0.4.1
-elasticsearch==6.3.1
+elasticsearch==7.0.4
et-xmlfile==1.0.1
Flask==1.0.2
Flask-Admin==1.5.2
diff --git a/tests/functional/test_search.py b/tests/functional/test_search.py
index 8679e91..ef74a13 100644
--- a/tests/functional/test_search.py
+++ b/tests/functional/test_search.py
@@ -27,7 +27,7 @@ class MyModel:
def test_add_to_index(db):
model1 = MyModel(2, "foo", "This is a test")
search.add_to_index("index-test", model1.to_dict())
- res = db.app.elasticsearch.get(index="index-test", doc_type="_doc", id=2)
+ res = db.app.elasticsearch.get(index="index-test", id=2)
assert res["_source"] == {"name": "foo", "description": "This is a test"}
@@ -35,10 +35,10 @@ def test_remove_from_index(db):
model1 = MyModel(3, "hello world!")
search.add_to_index("index-test", model1.to_dict())
res = db.app.elasticsearch.search(index="index-test", q="*")
- assert res["hits"]["total"] == 1
+ assert res["hits"]["total"]["value"] == 1
search.remove_from_index("index-test", model1.id)
res = db.app.elasticsearch.search(index="index-test", q="*")
- assert res["hits"]["total"] == 0
+ assert res["hits"]["total"]["value"] == 0
def test_remove_from_index_non_existing():
--
GitLab