From 8cf5e1fc4bcd62561f35cee4330dcab2dfecb011 Mon Sep 17 00:00:00 2001
From: Benjamin Bertrand <benjamin.bertrand@ess.eu>
Date: Thu, 30 Jan 2020 15:19:03 +0100
Subject: [PATCH] Add sensitive field to Network class
JIRA INFRA-1671
---
app/api/network.py | 2 +
app/models.py | 32 ++++++++++
app/network/forms.py | 1 +
app/search.py | 17 ++++++
app/templates/network/create_network.html | 1 +
app/templates/network/networks.html | 2 +
app/templates/network/view_network.html | 2 +
...492f46f_add_sensitive_column_to_network.py | 27 +++++++++
tests/functional/test_api.py | 58 ++++++++-----------
tests/functional/test_models.py | 27 +++++++++
tests/functional/test_search.py | 17 ++++++
tests/functional/test_web.py | 2 +-
12 files changed, 152 insertions(+), 36 deletions(-)
create mode 100644 migrations/versions/acd72492f46f_add_sensitive_column_to_network.py
diff --git a/app/api/network.py b/app/api/network.py
index 73b9d10..0e95f45 100644
--- a/app/api/network.py
+++ b/app/api/network.py
@@ -79,6 +79,8 @@ def create_network():
:jsonparam domain_id: (optional) primary key of the domain [default: scope domain]
:jsonparam admin_only: (optional) boolean to restrict the network to admin users [default: False]
:type admin_only: bool
+ :jsonparam sensitive: hide the network and all hosts if True (for non admin) [default: False]
+ :type sensitive: bool
:jsonparam description: (optional) description
"""
return utils.create_generic_model(
diff --git a/app/models.py b/app/models.py
index f5a2601..9dffdd6 100644
--- a/app/models.py
+++ b/app/models.py
@@ -832,6 +832,7 @@ class Network(CreatedMixin, db.Model):
gateway = db.Column(postgresql.INET, nullable=False, unique=True)
description = db.Column(db.Text)
admin_only = db.Column(db.Boolean, nullable=False, default=False)
+ sensitive = db.Column(db.Boolean, nullable=False, default=False)
scope_id = db.Column(db.Integer, db.ForeignKey("network_scope.id"), nullable=False)
domain_id = db.Column(db.Integer, db.ForeignKey("domain.id"), nullable=False)
@@ -951,6 +952,7 @@ class Network(CreatedMixin, db.Model):
"gateway": self.gateway,
"description": self.description,
"admin_only": self.admin_only,
+ "sensitive": self.sensitive,
"scope": utils.format_field(self.scope),
"domain": str(self.domain),
"interfaces": [str(interface) for interface in self.interfaces],
@@ -1186,6 +1188,7 @@ class Host(CreatedMixin, SearchableMixin, db.Model):
},
"ansible_vars": {"type": "flattened"},
"ansible_groups": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+ "sensitive": {"type": "boolean"},
}
# id shall be defined here to be used by SQLAlchemy-Continuum
@@ -1270,6 +1273,14 @@ class Host(CreatedMixin, SearchableMixin, db.Model):
except AttributeError:
return None
+ @property
+ def sensitive(self):
+ """Return True if the host is on a sensitive network"""
+ try:
+ return self.main_network.sensitive
+ except AttributeError:
+ return False
+
@property
def fqdn(self):
"""Return the host fully qualified domain name
@@ -1342,6 +1353,7 @@ class Host(CreatedMixin, SearchableMixin, db.Model):
"interfaces": [str(interface) for interface in self.interfaces],
"ansible_vars": self.ansible_vars,
"ansible_groups": [str(group) for group in self.ansible_groups],
+ "sensitive": self.sensitive,
}
)
if recursive:
@@ -1856,6 +1868,26 @@ def before_flush(session, flush_context, instances):
trigger_core_services_update(session)
+@sa.event.listens_for(Network.sensitive, "set")
+def update_host_sensitive_field(target, value, oldvalue, initiator):
+ """Update the host sensitive field in elasticsearch based on the Network value
+
+ Updating the network won't trigger any update of the hosts as sensitive is just
+ a property (based on host.main_interface.network).
+ We have to force the update in elasticsearch index.
+ """
+ if value != oldvalue:
+ current_app.logger.debug(f"Network {target} sensitive value changed to {value}")
+ index = "host" + current_app.config["ELASTICSEARCH_INDEX_SUFFIX"]
+ for interface in target.interfaces:
+ current_app.logger.debug(
+ f"Update sensitive to {value} for {interface.host}"
+ )
+ # We can't use interface.host.to_dict() because the property host.sensitive
+ # doesn't have the new value yet at this time
+ search.update_document(index, interface.host.id, {"sensitive": value})
+
+
# call configure_mappers after defining all the models
# required by sqlalchemy_continuum
sa.orm.configure_mappers()
diff --git a/app/network/forms.py b/app/network/forms.py
index 26aba27..71180a6 100644
--- a/app/network/forms.py
+++ b/app/network/forms.py
@@ -116,6 +116,7 @@ class NetworkForm(CSEntryForm):
gateway = NoValidateSelectField("Gateway IP", choices=[])
domain_id = SelectField("Domain")
admin_only = BooleanField("Admin only")
+ sensitive = BooleanField("Sensitive")
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
diff --git a/app/search.py b/app/search.py
index 5459872..f2a6f01 100644
--- a/app/search.py
+++ b/app/search.py
@@ -70,3 +70,20 @@ def query_index(index, query, page=1, per_page=20, sort=None):
search = current_app.elasticsearch.search(**kwargs)
ids = [int(hit["_id"]) for hit in search["hits"]["hits"]]
return ids, search["hits"]["total"]["value"]
+
+
+def update_document(index, id, partial_doc):
+ """Update partially a document
+
+ :param index: elasticsearch index
+ :param id: document id
+ :param dict partial_doc: fields to update
+ """
+ if not current_app.elasticsearch:
+ return
+ current_app.elasticsearch.update(
+ index=index,
+ id=id,
+ body={"doc": partial_doc},
+ refresh=current_app.config["ELASTICSEARCH_REFRESH"],
+ )
diff --git a/app/templates/network/create_network.html b/app/templates/network/create_network.html
index 72cd6da..fb2e52f 100644
--- a/app/templates/network/create_network.html
+++ b/app/templates/network/create_network.html
@@ -17,6 +17,7 @@
{{ render_field(form.gateway) }}
{{ render_field(form.domain_id) }}
{{ render_field(form.admin_only) }}
+ {{ render_field(form.sensitive) }}
<div class="form-group row">
<div class="col-sm-10">
<button type="submit" class="btn btn-primary">Submit</button>
diff --git a/app/templates/network/networks.html b/app/templates/network/networks.html
index a304b74..ae98b32 100644
--- a/app/templates/network/networks.html
+++ b/app/templates/network/networks.html
@@ -33,6 +33,7 @@
<th>Network scope</th>
<th>Domain</th>
<th>Admin only</th>
+ <th>Sensitive</th>
</tr>
</thead>
<tbody>
@@ -48,6 +49,7 @@
<td>{{ link_to_scope(network.scope) }}</td>
<td>{{ network.domain }}</td>
<td>{{ network.admin_only }}</td>
+ <td>{{ network.sensitive }}</td>
</tr>
{% endfor %}
</tbody>
diff --git a/app/templates/network/view_network.html b/app/templates/network/view_network.html
index 6252479..31276ac 100644
--- a/app/templates/network/view_network.html
+++ b/app/templates/network/view_network.html
@@ -37,6 +37,8 @@
<dd class="col-sm-9">{{ network.domain }}</dd>
<dt class="col-sm-3">Admin only</dt>
<dd class="col-sm-9">{{ network.admin_only }}</dd>
+ <dt class="col-sm-3">Sensitive</dt>
+ <dd class="col-sm-9">{{ network.sensitive }}</dd>
<dt class="col-sm-3">Hosts</dt>
<dd class="col-sm-9">{{ link_to_interfaces_host(network.interfaces) }}</dd>
<dt class="col-sm-3">Created by</dt>
diff --git a/migrations/versions/acd72492f46f_add_sensitive_column_to_network.py b/migrations/versions/acd72492f46f_add_sensitive_column_to_network.py
new file mode 100644
index 0000000..4d39a30
--- /dev/null
+++ b/migrations/versions/acd72492f46f_add_sensitive_column_to_network.py
@@ -0,0 +1,27 @@
+"""Add sensitive column to Network
+
+Revision ID: acd72492f46f
+Revises: 33720bfb353a
+Create Date: 2020-01-30 14:12:54.923114
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "acd72492f46f"
+down_revision = "33720bfb353a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.add_column(
+ "network",
+ sa.Column("sensitive", sa.Boolean(), nullable=False, server_default="False"),
+ )
+
+
+def downgrade():
+ op.drop_column("network", "sensitive")
diff --git a/tests/functional/test_api.py b/tests/functional/test_api.py
index d4276e4..5e0c580 100644
--- a/tests/functional/test_api.py
+++ b/tests/functional/test_api.py
@@ -60,6 +60,7 @@ HOST_KEYS = {
"created_at",
"updated_at",
"user",
+ "sensitive",
}
INTERFACE_KEYS = {
"id",
@@ -78,6 +79,26 @@ INTERFACE_KEYS = {
"updated_at",
"user",
}
+NETWORK_KEYS = {
+ "id",
+ "vlan_name",
+ "vlan_id",
+ "address",
+ "netmask",
+ "broadcast",
+ "first_ip",
+ "last_ip",
+ "gateway",
+ "description",
+ "admin_only",
+ "sensitive",
+ "scope",
+ "domain",
+ "interfaces",
+ "created_at",
+ "updated_at",
+ "user",
+}
def get(client, url, token=None):
@@ -736,25 +757,7 @@ def test_create_network(client, admin_token, network_scope_factory):
}
response = post(client, f"{API_URL}/network/networks", data=data, token=admin_token)
assert response.status_code == 201
- assert {
- "id",
- "vlan_name",
- "vlan_id",
- "address",
- "netmask",
- "broadcast",
- "first_ip",
- "last_ip",
- "gateway",
- "description",
- "admin_only",
- "scope",
- "domain",
- "interfaces",
- "created_at",
- "updated_at",
- "user",
- } == set(response.get_json().keys())
+ assert NETWORK_KEYS == set(response.get_json().keys())
assert response.get_json()["vlan_name"] == "network1"
assert response.get_json()["vlan_id"] == 1600
assert response.get_json()["address"] == "172.16.1.0/24"
@@ -1561,22 +1564,7 @@ def test_create_host(client, device_type_factory, user_token):
data = {"name": "my-hostname", "device_type": device_type.name}
response = post(client, f"{API_URL}/network/hosts", data=data, token=user_token)
assert response.status_code == 201
- assert {
- "id",
- "name",
- "fqdn",
- "is_ioc",
- "device_type",
- "model",
- "description",
- "items",
- "interfaces",
- "ansible_vars",
- "ansible_groups",
- "created_at",
- "updated_at",
- "user",
- } == set(response.get_json().keys())
+ assert HOST_KEYS == set(response.get_json().keys())
assert response.get_json()["name"] == data["name"]
# Check that name shall be unique
diff --git a/tests/functional/test_models.py b/tests/functional/test_models.py
index 31bf839..4eadeb3 100644
--- a/tests/functional/test_models.py
+++ b/tests/functional/test_models.py
@@ -931,3 +931,30 @@ def test_network_scope_overlapping(address, network_scope_factory):
assert f"{address} overlaps {scope.name} ({scope.supernet_ip})" in str(
excinfo.value
)
+
+
+def test_host_sensitive_field_update_on_network_change(
+ network_scope_factory, network_factory, interface_factory, host_factory
+):
+ scope = network_scope_factory(
+ first_vlan=3800, last_vlan=4000, supernet="192.168.0.0/16"
+ )
+ network = network_factory(
+ vlan_id=3800,
+ address="192.168.1.0/24",
+ first_ip="192.168.1.10",
+ last_ip="192.168.1.250",
+ sensitive=False,
+ scope=scope,
+ )
+ name = "host1"
+ host = host_factory(name=name)
+ interface_factory(name=name, host=host, ip="192.168.1.20", network=network)
+ # There is no sensitive host
+ instances, nb = models.Host.search("sensitive:true")
+ assert nb == 0
+ # Updating the network should update the host in the elasticsearch index
+ network.sensitive = True
+ instances, nb = models.Host.search("sensitive:true")
+ assert nb == 1
+ assert instances[0].name == name
diff --git a/tests/functional/test_search.py b/tests/functional/test_search.py
index ef74a13..59ed0db 100644
--- a/tests/functional/test_search.py
+++ b/tests/functional/test_search.py
@@ -68,3 +68,20 @@ def test_query_index():
# Test query sort
ids, total = search.query_index("index-test", "*", sort="name.keyword")
assert ids == [2, 1]
+
+
+def test_update_document(db):
+ # Create a document
+ index = "index-test"
+ id = 4
+ name = "a name"
+ description = "just an example"
+ model = MyModel(id, name, description)
+ search.add_to_index(index, model.to_dict())
+ res = db.app.elasticsearch.get(index="index-test", id=id)
+ assert res["_source"] == {"name": name, "description": description}
+ # Update the name field (description doesn't change)
+ new_name = "new name"
+ search.update_document(index, id, {"name": new_name})
+ res = db.app.elasticsearch.get(index=index, id=id)
+ assert res["_source"] == {"name": new_name, "description": description}
diff --git a/tests/functional/test_web.py b/tests/functional/test_web.py
index dcee73e..93d38f6 100644
--- a/tests/functional/test_web.py
+++ b/tests/functional/test_web.py
@@ -245,7 +245,7 @@ def test_retrieve_hosts(logged_client, interface_factory, host_factory):
response = logged_client.post("/network/_retrieve_hosts")
hosts = response.get_json()["data"]
assert {host1.name, host2.name} == set(host["name"] for host in hosts)
- assert len(hosts[0]) == 14
+ assert len(hosts[0]) == 15
assert len(hosts[0]["interfaces"][0]) == 15
--
GitLab