app/templates/network/view_host.html

@@ -80,7 +80,7 @@
           {% if current_user.is_admin %}
           {{ render_field(form.skip_post_install_job, label_size='5', input_size='7') }}
           {% endif %}
-          {{ submit_button_with_confirmation('Create ' + vm_type, 'Do you really want to create the ' + vm_type + ' ' + host.name + '?') }}
+          {{ submit_button_with_confirmation('Create ' + vm_type, 'Do you really want to create the ' + vm_type + ' ' + host.name + '?', 'Note: If you are experiencing issues with the VM, pressing "Create VM" once more will not resolve anything and you should instead contact the INFRA team or create a JIRA ticket to resolve this.') }}
         </form>
       </div>
     {% elif host.device_type.name in config.ALLOWED_SET_BOOT_PROFILE_DEVICE_TYPES %}

app/user/views.py

@@ -29,6 +29,7 @@ bp = Blueprint("user", __name__)
 
 @bp.route("/login", methods=["GET", "POST"])
 def login():
+    """Login page"""
     form = LDAPLoginForm(request.form)
     if form.validate_on_submit():
         login_user(form.user, remember=form.remember_me.data)
@@ -39,6 +40,7 @@ def login():
 @bp.route("/logout")
 @login_required
 def logout():
+    """Logout endpoint"""
     logout_user()
     return redirect(url_for("user.login"))
 
@@ -46,6 +48,7 @@ def logout():
 @bp.route("/profile", methods=["GET", "POST"])
 @login_required
 def profile():
+    """User profile"""
     # Try to get the generated token from the session
     token = session.pop("generated_token", None)
     form = TokenForm(request.form)
@@ -70,6 +73,7 @@ def profile():
 @bp.route("/tokens/revoke", methods=["POST"])
 @login_required
 def revoke_token():
+    """Endpoint to revoke a token"""
     token_id = request.form["token_id"]
     jti = request.form["jti"]
     try:

app/validators.py

@@ -15,8 +15,10 @@ import sqlalchemy as sa
 from wtforms import ValidationError, SelectField
 
 ICS_ID_RE = re.compile(r"^[A-Z]{3}[0-9]{3}$")
-HOST_NAME_RE = re.compile(r"^[a-z0-9\-]{2,20}$")
-INTERFACE_NAME_RE = re.compile(r"^[a-z0-9\-]{2,25}$")
+HOST_NAME_RE = re.compile(r"^[a-z0-9\-]{2,24}$")
+GROUP_NAME_RE = re.compile(r"^[a-z0-9\-\_]{2,50}$")
+# Interface name needs to be at least 5 characters more than the hostname (Every interface name have to start with the hostname)
+INTERFACE_NAME_RE = re.compile(r"^[a-z0-9\-]{2,29}$")
 VLAN_NAME_RE = re.compile(r"^[A-Za-z0-9\-]{3,25}$")
 MAC_ADDRESS_RE = re.compile(r"^(?:[0-9a-fA-F]{2}[:-]?){5}[0-9a-fA-F]{2}$")
 DEVICE_TYPE_RE = re.compile(r"^[A-Za-z0-9\-]{3,25}$")

docs/development/deployment.rst

+Deployment
+==========
+
+Deployment is performed using Ansible_ and docker_. The application is made of the following containers:
+
+- csentry_web to run the main Flask_ application with uwsgi_
+- csentry_workers_<index> to run RQ_ workers (same image as the main application)
+- csentry_postgres to run PostgreSQL_
+- csentry_elasticsearch to run Elasticsearch_
+- csentry_redis to run Redis_
+
+Refer to the `CSEntry Ansible role`_ for details.
+
+All application `default settings <https://gitlab.esss.lu.se/ics-infrastructure/csentry/-/blob/master/app/settings.py>`_
+can be overridden using a local `settings.cfg` file.
+For deployment, this local file is defined in the `CSEntry Ansible playbook`_:
+
+- `config/settings-prod.cfg <https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-csentry/-/blob/master/config/settings-prod.cfg>`_ for production
+- `config/settings-test.cfg <https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-csentry/-/blob/master/config/settings-test.cfg>`_ for staging
+
+
+.. _docker: https://www.docker.com
+.. _Ansible: https://docs.ansible.com/ansible/latest/index.html
+.. _Flask: https://flask.palletsprojects.com
+.. _uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/
+.. _PostgreSQL: https://www.postgresql.org
+.. _Redis: https://redis.io
+.. _Elasticsearch: https://www.elastic.co/elasticsearch/
+.. _RQ: https://python-rq.org
+.. _CSEntry Ansible role: https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-role-csentry
+.. _CSEntry Ansible playbook: https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-csentry

docs/development/design.rst

+Design
+======
+
+CSEntry is a web application developed using Flask_, one of the most popular Python web frameworks.
+Many principles follow the concepts described in `The Flask Mega-Tutorial`_ from `Miguel Grinberg <https://blog.miguelgrinberg.com/index>`_.
+
+The application relies on:
+
+- PostgreSQL_ as main database
+- Redis_ for caching and for queuing jobs and processing them in the background with workers using RQ_
+- Elasticsearch_ for search
+
+Database
+--------
+
+PostgreSQL_ is used as the main database.
+Interactions with the database are perfomed using SQLAlchemy_ ORM, which maps SQL tables to Python classes.
+All models are defined in the :mod:`app.models` file.
+
+Database migrations are performed using alembic_.
+When the database is modified, a new migration script can be autogenerated using::
+
+    docker-compose run --rm web flask db migrate -m "revision message"
+
+Resulting script should be modified according to the desired behaviour. Looking at previously migration scripts
+under the `migration/versions <https://gitlab.esss.lu.se/ics-infrastructure/csentry/-/tree/master/migrations/versions>`_
+directory can help.
+
+
+Full-text Search
+----------------
+
+Search is performed using Elasticsearch_ which provides very powerful capabilities.
+Implementation was inspired by `The Flask Mega-Tutorial Part XVI <https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-xvi-full-text-search>`_.
+
+.. note::
+
+   Elasticsearch index has to be kept in sync with the Postgres database. This is done using SQLAlchemy event listeners.
+
+The `List items <https://csentry.esss.lu.se/inventory/items>`_ and `List hosts <https://csentry.esss.lu.se/network/hosts>`_ pages
+use Elasticsearch_ to display the paginated list of results.
+If the index is empty or not up-to-date, missing items/hosts won't be displayed (even if they are in the postgres database).
+
+To make a class searchable, it should be a subclass of the :class:`~app.models.SearchableMixin` class and define the fields to index in the `__mapping__` class attribute.
+By default, the object will be kept in sync in the index thanks to the event listeners defined in the :class:`~app.models.SearchableMixin` class.
+Note that if you define a field that isn't a database column, you have to make sure it is kept up-to-date in the Elasticsearch index.
+An example is the :attr:`~app.models.Host.sensitive` field on the :class:`~app.models.Host` class. It's a property that comes from the network.
+Updating the sensitive field on a network won't trigger an update of the Host objects.
+A specific event listener :meth:`~app.models.update_host_sensitive_field` had to be implemented in that case.
+
+
+
+Background Tasks
+----------------
+
+A web application should reply to a client as fast as possible.
+Long running tasks can't be performed in the main process and block the response.
+
+Solving this is usually done using a task queue, a mechanism to send tasks to workers.
+In CSEntry, background tasks are run using RQ_, which is backed by Redis_.
+
+A subclass of RQ_ `Worker` is used to keep tasks results in the Postgres database: :class:`~app.tasks.TaskWorker`.
+This allows the user to see the `tasks <https://csentry.esss.lu.se/task/tasks>`_ that he created.
+Admin users have access to all tasks.
+
+The workers use the same docker image as the main application.
+They are started using the `flask runworker` subcommand, giving them access to the same settings as the web application.
+
+Some tasks are triggered automatically, based on SQLAlchemy event listeners, to keep the csentry inventory in sync in AWX for example.
+Some are triggered manually, like to create a VM for example.
+
+Most tasks are used to trigger templates in AWX via the API.
+But they can be used for internal processing as well, like reindexing Ansible groups in Elasticsearch_.
+
+Permissions
+-----------
+
+Using CSEntry requires each user to login with his/her ESS username.
+Any logged user has read access to most information, except `network scopes <https://csentry.esss.lu.se/network/scopes>`_
+and sensitive `networks <https://csentry.esss.lu.se/network/networks>`_.
+
+Write access requires to be part of a specific group.
+There are several roles/groups in CSEntry.
+Each internal group has to be mapped to a list of LDAP groups.
+
+Internal groups
++++++++++++++++
+
+admin
+~~~~~
+
+Admin users have full power and access to the application.
+They can modify anything via the `admin interface <https://csentry.esss.lu.se/admin/>`_.
+They can also monitor background jobs via the `RQ Dashboard <https://csentry.esss.lu.se/rq/>`_.
+
+auditor
+~~~~~~~
+
+Auditor users have read-only access to everything in the application (including sensitive networks).
+They don't have any write access.
+
+inventory
+~~~~~~~~~
+
+Users part of the inventory group have read and write access
+to the `Inventory <https://csentry.esss.lu.se/inventory/items>`_ to register items.
+
+network
+~~~~~~~
+
+*network* is used to gather groups based on the network scope. It shouldn't be mapped to a LDAP group.
+Only network scope groups should.
+
+If a user is part of a network scope group, it is added automatically to the *network* group.
+The user will have write access to all hosts in this scope, including on sensitive networks.
+He will still have read-only access to hosts on admin only networks.
+
+Admin users have automatically access to all network scopes.
+
+Configuration
++++++++++++++
+
+The *admin*, *auditor* and *inventory* groups mapping shall be defined in the :attr:`~app.settings.CSENTRY_LDAP_GROUPS` dictionary::
+
+    CSENTRY_LDAP_GROUPS = {
+        "admin": ["LDAP admin group"],
+        "auditor": ["another group"],
+        "inventory": ["group1", "group2"],
+    }
+
+For networks, groups based on the network scope name shall be defined
+in the :attr:`~app.settings.CSENTRY_NETWORK_SCOPES_LDAP_GROUPS` dictionary::
+
+    CSENTRY_NETWORK_SCOPES_LDAP_GROUPS = {
+        "TechnicalNetwork": ["group-tn"],
+        "LabNetworks": ["group-lab", "group-tn"],
+    }
+
+With the above settings, a user part of *group-tn* will have access to both the `TechnicalNetwork` and `LabNetworks` scopes.
+While a user part of the *group-lab* will only have access to the `LabNetworks` scope.
+If a network scope isn't defined in this dictionary, only admin users will have access to it.
+
+Usage
++++++
+
+Every endpoint should be protected using either the `login_required` or :meth:`~app.decorators.login_groups_accepted` decorator.
+The first will give access to any logged user. With the second, a list of internal groups should be given.
+
+To restrict access to admin users only::
+
+    @login_groups_accepted("admin")
+    def create_domain():
+        ...
+
+
+When using the *network* group, an additional check inside the function is required to check that the current user
+has the proper access (based on the network)::
+
+    @login_groups_accepted("admin", "network")
+    def edit_interface(name):
+        interface = models.Interface.query.filter_by(name=name).first_or_404()
+        if not current_user.has_access_to_network(interface.network):
+            abort(403)
+
+Database versioning
+-------------------
+
+SQLAlchemy-Continuum_ is used to track changes and keep an history.
+
+To enable versioning on a models, the ``__versioned__`` attribute shall be added to the model class.
+The following classes are versioned:
+
+-  :class:`~app.models.Item`
+-  :class:`~app.models.Host`
+-  :class:`~app.models.AnsibleGroup`
+
+The *History* on the view host or view group page displays the list of changes performed.
+As there is a relationship between Host and AnsibleGroup, it can lead to unexpected behavior.
+
+1. Edit a group to add a host -> new host added to the group history
+2. Edit another host to add it to that group -> new group added to the host history
+
+At this point the group page will show 2 hosts but only one was added in the history.
+If the group is edited, to add a variable, the recorded change will display all current hosts in the history.
+This is correct as those hosts were present when the group was edited.
+But the information showing the hosts were added previously is missing.
+This is a limitation of the current implementation.
+
+
+
+.. _Flask: https://flask.palletsprojects.com
+.. _The Flask Mega-Tutorial: https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-i-hello-world
+.. _PostgreSQL: https://www.postgresql.org
+.. _SQLAlchemy: https://www.sqlalchemy.org
+.. _Redis: https://redis.io
+.. _Elasticsearch: https://www.elastic.co/elasticsearch/
+.. _RQ: https://python-rq.org
+.. _alembic: https://alembic.sqlalchemy.org
+.. _SQLAlchemy-Continuum: https://sqlalchemy-continuum.readthedocs.io/en/latest/

docs/development/endpoints.rst

+Endpoints
+=========
+
+Main blueprint
+--------------
+
+.. autoflask:: wsgi:app
+   :modules: app.main.views
+   :include-empty-docstring:
+   :order: path
+
+User blueprint
+--------------
+
+.. autoflask:: wsgi:app
+   :modules: app.user.views
+   :include-empty-docstring:
+   :order: path
+
+Inventory blueprint
+-------------------
+
+.. autoflask:: wsgi:app
+   :modules: app.inventory.views
+   :include-empty-docstring:
+   :order: path
+
+Network blueprint
+-----------------
+
+.. autoflask:: wsgi:app
+   :modules: app.network.views
+   :include-empty-docstring:
+   :order: path
\ No newline at end of file

docs/development/implementation/commands.rst

+.. automodule:: app.commands
+   :members:
+   :undoc-members:

docs/development/implementation/decorators.rst

+.. automodule:: app.decorators
+   :members:
+   :undoc-members:

docs/development/implementation/index.rst

+Implementation
+==============
+
+The main modules are described below.
+Full source code is available in `GitLab <https://gitlab.esss.lu.se/ics-infrastructure/csentry>`_.
+
+.. toctree::
+   :maxdepth: 2
+
+   models
+   commands
+   decorators
+   search
+   tasks
+   user_view
+   inventory_view
+   network_view

docs/development/implementation/inventory_view.rst

+.. automodule:: app.inventory.views
+   :members:
+   :undoc-members:

docs/development/implementation/models.rst

+.. automodule:: app.models
+   :members:
+   :undoc-members:

docs/development/implementation/network_view.rst

+.. automodule:: app.network.views
+   :members:
+   :undoc-members:

docs/development/implementation/search.rst

+.. automodule:: app.search
+   :members:
+   :undoc-members:

docs/development/implementation/tasks.rst

+.. automodule:: app.tasks
+   :members:
+   :undoc-members:

docs/development/implementation/user_view.rst

+.. automodule:: app.user.views
+   :members:
+   :undoc-members:

docs/index.rst

@@ -45,6 +45,7 @@ Please use the navigation sidebar on the left to begin.
 
 .. toctree::
    :hidden:
+   :caption: User documentation
    :maxdepth: 3
 
    inventory
@@ -53,3 +54,13 @@ Please use the navigation sidebar on the left to begin.
    profile
    api
    changelog
+
+.. toctree::
+   :hidden:
+   :caption: Developer documentation
+   :maxdepth: 3
+
+   development/design
+   development/deployment
+   development/endpoints
+   development/implementation/index

tests/functional/conftest.py

@@ -9,10 +9,12 @@ Pytest fixtures common to all functional tests.
 :license: BSD 2-Clause, see LICENSE for more details.
 
 """
+import redis
 import pytest
 import sqlalchemy as sa
 from pytest_factoryboy import register
 from flask_ldap3_login import AuthenticationResponse, AuthenticationResponseStatus
+from rq import push_connection, pop_connection
 from app.factory import create_app
 from app.extensions import db as _db
 from app.models import SearchableMixin, Host, Item, AnsibleGroup
@@ -45,6 +47,7 @@ def app(request):
         "TESTING": True,
         "WTF_CSRF_ENABLED": False,
         "SQLALCHEMY_DATABASE_URI": "postgresql://ics:icspwd@postgres/csentry_db_test",
+        "RQ_REDIS_URL": "redis://redis:6379/4",
         "ELASTICSEARCH_INDEX_SUFFIX": "-test",
         "ELASTICSEARCH_REFRESH": "true",
         "CACHE_TYPE": "null",
@@ -135,8 +138,17 @@ def session(db, request):
     Host.create_index()
     AnsibleGroup.create_index()
 
+    # Setup RQ redis connection
+    redis_connection = redis.from_url(db.app.config["RQ_REDIS_URL"])
+    redis_connection.flushdb()
+    push_connection(redis_connection)
+
     yield session
 
+    # Clean RQ redis connnection
+    redis_connection.flushdb()
+    pop_connection()
+
     # ELASTICSEARCH_INDEX_SUFFIX is set to "-test"
     # Delete all "*-test" indices after each test
     db.app.elasticsearch.indices.delete("*-test", ignore=404)
@@ -163,8 +175,19 @@ def patch_ldap_authenticate(monkeypatch):
             response.user_groups = [{"cn": "CSEntry Admin"}]
         elif username == "audit" and password == "auditpasswd":
             response.status = AuthenticationResponseStatus.success
-            response.user_info = {"cn": "Auditor User", "mail": "audit@example.com"}
-            response.user_groups = [{"cn": "CSEntry Auditor"}]
+            response.user_dn = "uid=audit,ou=Service accounts,dc=esss,dc=lu,dc=se"
+            response.user_info = {
+                "uid": ["audit"],
+                "cn": [],
+                "mail": [],
+                "dn": "uid=audit,ou=Service accounts,dc=esss,dc=lu,dc=se",
+            }
+            response.user_groups = [
+                {
+                    "cn": ["CSEntry Auditor"],
+                    "dn": "cn=CSEntry Auditor,ou=ICS,ou=Groups,dc=esss,dc=lu,dc=se",
+                }
+            ]
         elif username == "user_rw" and password == "userrw":
             response.status = AuthenticationResponseStatus.success
             response.user_info = {"cn": "User RW", "mail": "user_rw@example.com"}

tests/functional/test_api.py

@@ -1455,13 +1455,15 @@ def test_create_interface_without_ip(
     assert response.get_json()["ip"] == "192.168.1.10"
 
 
-def test_delete_interface_invalid_credentials(client, interface_factory, user_token):
+def test_delete_interface_normal_user(client, interface_factory, user_token):
     interface1 = interface_factory()
-    response = delete(
-        client, f"{API_URL}/network/interfaces/{interface1.id}", token=user_token
+    check_delete_success(
+        client,
+        user_token,
+        interface1,
+        "network/interfaces",
+        models.Interface,
     )
-    assert response.status_code == 403
-    assert len(models.Interface.query.all()) == 1
 
 
 def test_delete_interface_success(client, admin_token, interface):
@@ -1568,7 +1570,7 @@ def test_create_ansible_group(client, admin_token):
     response = post(client, f"{API_URL}/network/groups", data=data, token=admin_token)
     check_response_message(
         response,
-        "(psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint",
+        "Group name matches an existing group",
         422,
     )
 

tests/functional/test_models.py

@@ -1085,36 +1085,36 @@ def test_task_awx_job_url(db, task_factory):
     assert task5.awx_job_url == "https://awx.example.org/#/jobs/inventory/12"
 
 
-@pytest.mark.parametrize("length", (1, 21, 50))
+@pytest.mark.parametrize("length", (1, 25, 50))
 def test_hostname_invalid_length(db, host_factory, length):
     with pytest.raises(ValidationError) as excinfo:
         host_factory(name="x" * length)
-    assert r"Host name shall match [a-z0-9\-]{2,20}" in str(excinfo.value)
+    assert r"Host name shall match ^[a-z0-9\-]{2,24}" in str(excinfo.value)
 
 
 @pytest.mark.parametrize("name", ("my_host", "host@", "foo:bar", "U02.K02"))
 def test_hostname_invalid_characters(db, host_factory, name):
     with pytest.raises(ValidationError) as excinfo:
         host_factory(name=name)
-    assert r"Host name shall match [a-z0-9\-]{2,20}" in str(excinfo.value)
+    assert r"Host name shall match ^[a-z0-9\-]{2,24}" in str(excinfo.value)
 
 
-@pytest.mark.parametrize("length", (1, 26, 50))
+@pytest.mark.parametrize("length", (1, 30, 50))
 def test_interface_name_invalid_length(db, interface_factory, length):
     with pytest.raises(ValidationError) as excinfo:
         interface_factory(name="x" * length)
-    assert r"Interface name shall match [a-z0-9\-]{2,25}" in str(excinfo.value)
+    assert r"Interface name shall match ^[a-z0-9\-]{2,29}" in str(excinfo.value)
 
 
 def test_interface_name_length(db, host_factory, interface_factory):
-    hostname = "x" * 20
+    hostname = "x" * 24
     interface_name = hostname + "-yyyy"
     host1 = host_factory(name=hostname)
     interface_factory(name=interface_name, host=host1)
     assert host1.interfaces[0].name == interface_name
     with pytest.raises(ValidationError) as excinfo:
         interface_factory(name=interface_name + "y", host=host1)
-    assert r"Interface name shall match [a-z0-9\-]{2,25}" in str(excinfo.value)
+    assert r"Interface name shall match ^[a-z0-9\-]{2,29}" in str(excinfo.value)
 
 
 @pytest.mark.parametrize("ics_id", ("123", "AA123", "AAA1234"))
@@ -1264,3 +1264,61 @@ def test_host_sensitive_field_update_on_network_change(
     instances, nb = models.Host.search("sensitive:true")
     assert nb == 1
     assert instances[0].name == name
+
+
+@pytest.mark.parametrize(
+    "dn,username,user_info,user_groups,expected_display_name,expected_email,expected_groups",
+    [
+        (
+            "uid=johndoe,ou=Users,dc=esss,dc=lu,dc=se",
+            "johndoe",
+            {"mail": "john.doe@example.org", "cn": "John Doe"},
+            [{"cn": "group2"}, {"cn": "group1"}],
+            "John Doe",
+            "john.doe@example.org",
+            ["group1", "group2"],
+        ),
+        (
+            "uid=johndoe,ou=Users,dc=esss,dc=lu,dc=se",
+            "johndoe",
+            {"mail": ["john.doe@example.org"], "cn": ["John Doe"]},
+            [{"cn": ["group2"]}, {"cn": ["group1"]}],
+            "John Doe",
+            "john.doe@example.org",
+            ["group1", "group2"],
+        ),
+        (
+            "uid=auditor,ou=Service accounts,dc=esss,dc=lu,dc=se",
+            "auditor",
+            {
+                "uid": ["auditor"],
+                "cn": [],
+                "mail": [],
+                "dn": "uid=csentry_svc,ou=Service accounts,dc=esss,dc=lu,dc=se",
+            },
+            [
+                {
+                    "cn": ["csentry_auditors"],
+                    "dn": "cn=csentry_auditors,ou=ICS,ou=Groups,dc=esss,dc=lu,dc=se",
+                }
+            ],
+            "auditor",
+            "",
+            ["csentry_auditors"],
+        ),
+    ],
+)
+def test_save_user(
+    dn,
+    username,
+    user_info,
+    user_groups,
+    expected_display_name,
+    expected_email,
+    expected_groups,
+):
+    user = models.save_user(dn, username, user_info, user_groups)
+    assert user.username == username
+    assert user.display_name == expected_display_name
+    assert user.email == expected_email
+    assert user.groups == expected_groups

tests/functional/test_tasks.py

+import pytest
+
+
+@pytest.mark.parametrize(
+    "name, func, input_kwargs, output_args",
+    [
+        ("my task1", "my_func1", {}, ""),
+        (
+            "my task2",
+            "my_func2",
+            {"arg1": "foo", "arg2": True},
+            "arg1='foo', arg2=True",
+        ),
+        # job_timeout is used by enqueue for the job
+        ("another task", "func_to_run", {"job_timeout": 180}, ""),
+        # timeout is NOT used by enqueue for the job (deprecated in RQ >= 1.0)
+        # it's passed to the function
+        ("task4", "my_func4", {"timeout": 60}, "timeout=60"),
+    ],
+)
+def test_launch_task_kwargs(user, name, func, input_kwargs, output_args):
+    task = user.launch_task(name, func=func, **input_kwargs)
+    assert task.name == name
+    assert task.command == f"app.tasks.{func}({output_args})"