.gitlab-ci.yml

 ---
 include:
-  - 'https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/PreCommit.gitlab-ci.yml'
-
-.runner_tags: &runner_tags
-  tags:
-    - docker
+  - remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/PreCommit.gitlab-ci.yml"
+  - remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/SonarScanner.gitlab-ci.yml"
 
 variables:
   CONTAINER_TEST_IMAGE: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME"
@@ -22,11 +19,14 @@ stages:
   - release
   - deploy
 
+default:
+  tags:
+    - docker
+
 before_script:
   - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
 
 build:
-  <<: *runner_tags
   stage: build
   image: docker:latest
   script:
@@ -35,7 +35,6 @@ build:
     - docker push "$CONTAINER_TEST_IMAGE"
 
 test:
-  <<: *runner_tags
   stage: test
   image: "$CONTAINER_TEST_IMAGE"
   services:
@@ -47,7 +46,7 @@ test:
   before_script:
     - pip install -r requirements-dev.txt
   script:
-    - pytest --junitxml=junit.xml --cov-report xml:coverage.xml --cov=app -v
+    - pytest --junitxml=junit.xml --cov-report=xml:coverage.xml --cov-report=term --cov=app -v
   artifacts:
     paths:
       - junit.xml
@@ -56,16 +55,7 @@ test:
       junit: junit.xml
     expire_in: 1 hour
 
-analyse:
-  <<: *runner_tags
-  stage: analyse
-  image: registry.esss.lu.se/ics-docker/sonar-scanner:3
-  before_script: []
-  script:
-    - sonar-scanner -Dsonar.login=$SONARQUBE_TOKEN -Dsonar.projectVersion=$CI_COMMIT_REF_NAME
-
 release-image:
-  <<: *runner_tags
   stage: release
   image: docker:latest
   dependencies: []
@@ -76,8 +66,26 @@ release-image:
   only:
     - tags
 
+deploy-dev:
+  stage: deploy
+  image: registry.esss.lu.se/ics-docker/tower-cli
+  before_script: []
+  dependencies: []
+  script:
+    - >
+      tower-cli job launch
+      -h torn.tn.esss.lu.se
+      -t ${TOWER_OAUTH_TOKEN}
+      -J deploy-csentry-dev
+      -e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
+  environment:
+    name: dev
+    url: https://csentry-lab-01.cslab.esss.lu.se
+  except:
+    - master
+    - tags
+
 deploy-staging:
-  <<: *runner_tags
   stage: deploy
   image: registry.esss.lu.se/ics-docker/tower-cli
   before_script: []
@@ -97,7 +105,6 @@ deploy-staging:
     - tags
 
 pages:
-  <<: *runner_tags
   stage: deploy
   image: "$CONTAINER_RELEASE_IMAGE"
   dependencies: []
@@ -115,7 +122,6 @@ pages:
     - tags
 
 deploy-production:
-  <<: *runner_tags
   stage: deploy
   image: registry.esss.lu.se/ics-docker/tower-cli
   before_script: []

.pre-commit-config.yaml

 repos:
-  - repo: https://github.com/ambv/black
-    rev: 19.10b0
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
     hooks:
       - id: black
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
     hooks:
       - id: flake8

CHANGELOG.rst

 Changelog
 =========
 
+Version 2021.04.06
+------------------
+
+- Validate group names (INFRA-3135)
+- Add warning text for creating vm (INFRA-3292)
+- Paginate history in ansible groups and hosts (INFRA-3136)
+
+Version 2020.11.25
+------------------
+
+- Add documentation about database versioning
+- Set display_name to username when empty (INFRA-2909)
+- Improve logging (INFRA-2899)
+
+Version 2020.11.13
+------------------
+
+- Increased hostname character limit from 20 -> 24 (INFRA-2808)
+- Use SSL by default for LDAP connection (INFRA-2805)
+- Use SonarScanner.gitlab-ci.yml template (INFRA-2832)
+
+Version 2020.10.30
+------------------
+
+- Fix inventory export to excel (INFRA-2780)
+- Add developer documentation
+
+Version 2020.10.27
+------------------
+
+- Update Python and requirements (INFRA-2649)
+- Update RQ and rq-dashboard (INFRA-2649 / INFRA-2754)
+- Fix code smells (INFRA-2729)
+- Add auditor group (INFRA-2728)
+- Update Sphinx and switch to sphinx_rtd_theme (INFRA-2742)
+
+Version 2020.10.06
+------------------
+
+- Make sensitive hosts visible for members of the scope (INFRA-2508)
+- Only hide sensitive networks not part of the user scope (INFRA-2437)
+- Automatically set network_scope group as parent of network groups (INFRA-2639)
+
+Version 2020.07.08
+------------------
+
+- Add view to edit network scope (INFRA-2316)
+- Add API endpoint to patch network scope (INFRA-2316)
+
+Version 2020.06.12
+------------------
+
+- Change MTCA-IOxOS to MTCA-IFC (INFRA-2200)
+- Add description field on Interface (INFRA-2112)
+
+Version 2020.04.16
+------------------
+
+- Allow /network/networks API endpoint for normal users (INFRA-2013)
+- Update Bootstrap to 4.4.1 (INFRA-2021)
+- Update DataTables (INFRA-2021)
+- Add tooltip on search box (INFRA-1893)
+
+Version 2020.03.17
+------------------
+
+- Add extra badges to README and documentation
+- Fix get_hosts for normal users (INFRA-1888)
+
+Version 2020.03.09
+------------------
+
+- Trigger core services update on host change (INFRA-1846)
+- Add DELETE method on all API endpoints (INFRA-1853)
+- Remove create_user API endpoint (INFRA-1854)
+
 Version 2020.03.05
 ------------------
 

Dockerfile

-FROM python:3.7-slim as base
+FROM python:3.8-slim as base
+
+# Add ESS specific debian repository mirror
+RUN echo "deb https://artifactory.esss.lu.se/artifactory/debian-mirror stable main" | tee /etc/apt/sources.list.d/ess-debian-mirror.list
 
 # Install Python dependencies in an intermediate image
 # as some requires a compiler (psycopg2)
@@ -37,6 +40,7 @@ RUN apt-get update \
 COPY requirements.txt /requirements.txt
 RUN python -m venv /venv \
   && . /venv/bin/activate \
+  && pip install --no-cache-dir --upgrade pip \
   && pip install --no-cache-dir -r /requirements.txt
 
 ARG CSENTRY_BUILD
@@ -64,6 +68,8 @@ RUN apt-get update \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
+RUN rm /etc/apt/sources.list.d/ess-debian-mirror.list
+
 COPY --chown=csi:csi . /app/
 WORKDIR /app
 RUN echo "__version__ = \"$(git describe)\"" > app/_version.py

Makefile

-.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image
+.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image docs
 
 OWNER := registry.esss.lu.se/ics-infrastructure
 GIT_TAG := $(shell git describe --always)
@@ -60,3 +60,6 @@ test_image:  ## run the tests (on the latest image)
 
 run_uwsgi:  ## run the application with uwsgi (to test prod env)
 	docker-compose run -p 8000:8000 --rm web uwsgi --master --http 0.0.0.0:8000 --http-keepalive --manage-script-name --mount /csentry=wsgi.py --callable app --uid conda --processes 2 -b 16384
+
+docs:  ## run the tests (on current directory)
+	docker run --rm -v $(shell pwd):/app registry.esss.lu.se/ics-infrastructure/csentry:master sphinx-build -M html docs docs/_build

README.rst

@@ -7,6 +7,13 @@ CSEntry
 .. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=alert_status
     :target: https://sonarqube.esss.lu.se/dashboard?id=csentry
 
+.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=ncloc
+    :target: https://sonarqube.esss.lu.se/dashboard?id=csentry
+
+.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/pipeline.svg
+
+.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/coverage.svg
+
 Control System Entry web server.
 
 

app/api/inventory.py

@@ -13,7 +13,7 @@ from flask import Blueprint, jsonify, request, current_app
 from flask_login import login_required
 from .. import utils, models
 from ..decorators import login_groups_accepted
-from .utils import commit, create_generic_model, get_generic_model
+from .utils import commit, create_generic_model, get_generic_model, delete_generic_model
 
 bp = Blueprint("inventory_api", __name__)
 
@@ -140,6 +140,18 @@ def patch_item(id_):
     return jsonify(item.to_dict())
 
 
+@bp.route("/items/<int:item_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_item(item_id):
+    """Delete an item
+
+    .. :quickref: Inventory; Delete an item
+
+    :param item_id: item primary key
+    """
+    return delete_generic_model(models.Item, item_id)
+
+
 @bp.route("/items/<id_>/comments")
 @login_required
 def get_item_comments(id_):
@@ -169,6 +181,18 @@ def create_item_comment(id_):
     )
 
 
+@bp.route("/items/comments/<int:comment_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_comment(comment_id):
+    """Delete an item comment
+
+    .. :quickref: Inventory; Delete an item comment
+
+    :param comment_id: comment primary key
+    """
+    return delete_generic_model(models.ItemComment, comment_id)
+
+
 @bp.route("/actions")
 @login_required
 def get_actions():
@@ -202,6 +226,18 @@ def create_manufacturer():
     return create_generic_model(models.Manufacturer)
 
 
+@bp.route("/manufacturers/<int:manufacturer_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_manufacturer(manufacturer_id):
+    """Delete a manufacturer
+
+    .. :quickref: Inventory; Delete a manufacturer
+
+    :param manufacturer_id: manufacturer primary key
+    """
+    return delete_generic_model(models.Manufacturer, manufacturer_id)
+
+
 @bp.route("/models")
 @login_required
 def get_models():
@@ -225,6 +261,18 @@ def create_model():
     return create_generic_model(models.Model)
 
 
+@bp.route("/models/<int:model_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_model(model_id):
+    """Delete a model
+
+    .. :quickref: Inventory; Delete a model
+
+    :param model_id: model primary key
+    """
+    return delete_generic_model(models.Model, model_id)
+
+
 @bp.route("/locations")
 @login_required
 def get_locations():
@@ -248,6 +296,18 @@ def create_locations():
     return create_generic_model(models.Location)
 
 
+@bp.route("/locations/<int:location_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_location(location_id):
+    """Delete a location
+
+    .. :quickref: Inventory; Delete a location
+
+    :param location_id: location primary key
+    """
+    return delete_generic_model(models.Location, location_id)
+
+
 @bp.route("/statuses")
 @login_required
 def get_status():
@@ -271,6 +331,18 @@ def create_status():
     return create_generic_model(models.Status)
 
 
+@bp.route("/statuses/<int:status_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_status(status_id):
+    """Delete a status
+
+    .. :quickref: Inventory; Delete a status
+
+    :param status_id: status primary key
+    """
+    return delete_generic_model(models.Status, status_id)
+
+
 @bp.route("/macs")
 @login_required
 def get_macs():

app/api/network.py

@@ -21,7 +21,7 @@ bp = Blueprint("network_api", __name__)
 
 
 @bp.route("/scopes")
-@login_groups_accepted("admin")
+@login_groups_accepted("admin", "auditor")
 def get_scopes():
     """Return network scopes
 
@@ -52,14 +52,66 @@ def create_scope():
     )
 
 
-@bp.route("/networks")
+@bp.route("/scopes/<int:scope_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_scope(scope_id):
+    """Delete a network scope
+
+    .. :quickref: Network; Delete a network scope
+
+    :param scope_id: network scope primary key
+    """
+    return utils.delete_generic_model(models.NetworkScope, scope_id)
+
+
+@bp.route("/scopes/<int:scope_id>", methods=["PATCH"])
 @login_groups_accepted("admin")
+def patch_scope(scope_id):
+    r"""Patch an existing network scope
+
+    .. :quickref: Network; Update an existing network scope
+
+    :param scope_id: network scope primary key
+    :jsonparam name: network scope name
+    :jsonparam description: description
+    :jsonparam first_vlan: network scope first vlan
+    :jsonparam last_vlan: network scope last vlan
+    :jsonparam supernet: network scope supernet
+    :jsonparam domain: name of the default domain
+    """
+    allowed_fields = (
+        ("name", str, None),
+        ("description", str, None),
+        ("first_vlan", int, None),
+        ("last_vlan", int, None),
+        ("supernet", str, None),
+        ("domain", models.Domain, "name"),
+    )
+    return utils.update_generic_model(models.NetworkScope, scope_id, allowed_fields)
+
+
+@bp.route("/networks")
+@login_groups_accepted("admin", "auditor", "network")
 def get_networks():
     """Return networks
 
     .. :quickref: Network; Get networks
     """
-    return utils.get_generic_model(models.Network, order_by=models.Network.address)
+    query = models.Network.query
+    if not (current_user.is_admin or current_user.is_auditor):
+        sensitive_networks = (
+            query.filter(models.Network.sensitive.is_(True))
+            .join(models.Network.scope)
+            .filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
+        )
+        query = (
+            query.filter(models.Network.sensitive.is_(False))
+            .union(sensitive_networks)
+            .distinct(models.Network.id)
+            .from_self()
+        )
+    query = query.order_by(models.Network.address)
+    return utils.get_generic_model(model=models.Network, base_query=query)
 
 
 @bp.route("/networks", methods=["POST"])
@@ -133,6 +185,18 @@ def patch_network(network_id):
     return utils.update_generic_model(models.Network, network_id, allowed_fields)
 
 
+@bp.route("/networks/<int:network_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_network(network_id):
+    """Delete a network
+
+    .. :quickref: Network; Delete a network
+
+    :param network_id: network primary key
+    """
+    return utils.delete_generic_model(models.Network, network_id)
+
+
 @bp.route("/interfaces")
 @login_required
 def get_interfaces():
@@ -142,8 +206,19 @@ def get_interfaces():
     """
     query = models.Interface.query
     query = query.join(models.Interface.network).order_by(models.Interface.ip)
-    if not current_user.is_admin:
-        query = query.filter(models.Network.sensitive.is_(False))
+    if not (current_user.is_admin or current_user.is_auditor):
+        sensitive_interfaces = (
+            query.filter(models.Network.sensitive.is_(True))
+            .join(models.Network.scope)
+            .filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
+        )
+        query = (
+            query.filter(models.Network.sensitive.is_(False))
+            .union(sensitive_interfaces)
+            .from_self()
+            .join(models.Interface.network)
+            .order_by(models.Interface.ip)
+        )
     domain = request.args.get("domain", None)
     if domain is not None:
         query = query.join(models.Network.domain).filter(models.Domain.name == domain)
@@ -238,7 +313,7 @@ def patch_interface(interface_id):
 
 
 @bp.route("/interfaces/<int:interface_id>", methods=["DELETE"])
-@login_groups_accepted("admin")
+@login_groups_accepted("admin", "network")
 def delete_interface(interface_id):
     """Delete an interface
 
@@ -274,6 +349,18 @@ def create_ansible_groups():
     return utils.create_generic_model(models.AnsibleGroup, mandatory_fields=("name",))
 
 
+@bp.route("/groups/<int:group_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_ansible_group(group_id):
+    """Delete an Ansible group
+
+    .. :quickref: Network; Delete an Ansible group
+
+    :param group_id: Ansible group primary key
+    """
+    return utils.delete_generic_model(models.AnsibleGroup, group_id)
+
+
 @bp.route("/hosts")
 @login_required
 def get_hosts():
@@ -282,15 +369,27 @@ def get_hosts():
     .. :quickref: Network; Get hosts
     """
     query = models.Host.query
-    if not current_user.is_admin:
+    if not (current_user.is_admin or current_user.is_auditor):
         # Note that hosts without interface will be filtered out
         # by this query. This is not an issue as hosts should always
         # have an interface. They are useless otherwise.
-        query = (
+        non_sensitive_hosts = (
             query.join(models.Host.interfaces)
             .join(models.Interface.network)
             .filter(models.Network.sensitive.is_(False))
         )
+        sensitive_hosts = (
+            query.join(models.Host.interfaces)
+            .join(models.Interface.network)
+            .filter(models.Network.sensitive.is_(True))
+            .join(models.Network.scope)
+            .filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
+        )
+        query = (
+            non_sensitive_hosts.union(sensitive_hosts)
+            .distinct(models.Host.id)
+            .from_self()
+        )
     query = query.order_by(models.Host.name)
     return utils.get_generic_model(model=models.Host, base_query=query)
 
@@ -393,6 +492,18 @@ def create_domain():
     return utils.create_generic_model(models.Domain, mandatory_fields=("name",))
 
 
+@bp.route("/domains/<int:domain_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_domain(domain_id):
+    """Delete a domain
+
+    .. :quickref: Network; Delete a domain
+
+    :param domain_id: domain primary key
+    """
+    return utils.delete_generic_model(models.Domain, domain_id)
+
+
 @bp.route("/cnames")
 @login_required
 def get_cnames():
@@ -427,3 +538,15 @@ def create_cname():
     return utils.create_generic_model(
         models.Cname, mandatory_fields=("name", "interface_id")
     )
+
+
+@bp.route("/cnames/<int:cname_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_cname(cname_id):
+    """Delete a cname
+
+    .. :quickref: Network; Delete a cname
+
+    :param cname_id: cname primary key
+    """
+    return utils.delete_generic_model(models.Cname, cname_id)

app/api/user.py

@@ -15,13 +15,13 @@ from flask_login import login_required, current_user
 from ..extensions import ldap_manager
 from ..decorators import login_groups_accepted
 from .. import utils, tokens, models
-from .utils import get_generic_model, create_generic_model
+from .utils import get_generic_model
 
 bp = Blueprint("user_api", __name__)
 
 
 @bp.route("/users")
-@login_required
+@login_groups_accepted("admin", "auditor")
 def get_users():
     """Return users information
 
@@ -40,22 +40,6 @@ def get_user_profile():
     return jsonify(current_user.to_dict()), 200
 
 
-@bp.route("/users", methods=["POST"])
-@login_groups_accepted("admin")
-def create_user():
-    """Create a new user
-
-    .. :quickref: User; Create new user
-
-    :jsonparam username: User's username
-    :jsonparam display_name: User's display name
-    :jsonparam email: User's email
-    """
-    return create_generic_model(
-        models.User, mandatory_fields=("username", "display_name", "email")
-    )
-
-
 @bp.route("/login", methods=["POST"])
 def login():
     """Return a JSON Web Token

app/api/utils.py

@@ -12,6 +12,7 @@ This module implements useful functions for the API.
 import urllib.parse
 import sqlalchemy as sa
 from flask import current_app, jsonify, request
+from flask_login import current_user
 from flask_sqlalchemy import Pagination
 from ..extensions import db
 from .. import utils
@@ -124,7 +125,6 @@ def get_json_body():
     data = request.get_json()
     if data is None:
         raise utils.CSEntryError("Body should be a JSON object")
-    current_app.logger.debug(f"Received: {data}")
     if not data:
         raise utils.CSEntryError("At least one field is required", status_code=422)
     return data
@@ -154,6 +154,9 @@ def create_generic_model(model, mandatory_fields=("name",), **kwargs):
         raise utils.CSEntryError(str(e), status_code=422)
     db.session.add(instance)
     commit()
+    current_app.logger.info(
+        f"New {model.__tablename__} created by {current_user}: {instance.to_dict()}"
+    )
     return jsonify(instance.to_dict()), 201
 
 
@@ -165,7 +168,10 @@ def delete_generic_model(model, primary_key):
     """
     instance = model.query.get_or_404(primary_key)
     db.session.delete(instance)
-    db.session.commit()
+    commit()
+    current_app.logger.info(
+        f"{model.__tablename__} {instance} deleted by {current_user}"
+    )
     return jsonify(), 204
 
 
@@ -195,4 +201,7 @@ def update_generic_model(model, primary_key, allowed_fields):
     except Exception as e:
         raise utils.CSEntryError(str(e), status_code=422)
     commit()
+    current_app.logger.info(
+        f"{model.__tablename__} {primary_key} updated by {current_user}: {instance.to_dict()}"
+    )
     return jsonify(instance.to_dict()), 200

app/commands.py

@@ -54,7 +54,9 @@ def sync_user(connection, user):
             disable_user(user)
         else:
             attributes = ldap_user["attributes"]
-            user.display_name = utils.attribute_to_string(attributes["cn"])
+            user.display_name = (
+                utils.attribute_to_string(attributes["cn"]) or user.username
+            )
             user.email = utils.attribute_to_string(attributes["mail"])
             groups = ldap_manager.get_user_groups(
                 dn=ldap3.utils.conv.escape_filter_chars(ldap_user["dn"]),
@@ -139,7 +141,7 @@ def register_cli(app):
     @app.cli.command()
     def runworker():
         """Run RQ worker"""
-        redis_url = current_app.config["REDIS_URL"]
+        redis_url = current_app.config["RQ_REDIS_URL"]
         redis_connection = redis.from_url(redis_url)
         with rq.Connection(redis_connection):
             worker = TaskWorker(["high", "normal", "low"])

app/decorators.py

@@ -21,6 +21,7 @@ def login_groups_accepted(*groups):
     This can be used for users logged in using a cookie (web UI) or JWT (API).
 
     Example::
+
         @bp.route('/models', methods=['POST'])
         @login_groups_accepted('admin', 'inventory')
         def create_model():

app/defaults.py

@@ -24,7 +24,7 @@ defaults = [
     models.DeviceType(name="VirtualMachine"),
     models.DeviceType(name="Network"),
     models.DeviceType(name="MTCA-AMC"),
-    models.DeviceType(name="MTCA-IOxOS"),
+    models.DeviceType(name="MTCA-IFC"),
     models.DeviceType(name="MTCA-MCH"),
     models.DeviceType(name="MTCA-RTM"),
     models.DeviceType(name="VME"),

app/factory.py

@@ -72,19 +72,7 @@ def create_app(config=None):
             integrations=[FlaskIntegration()],
         )
 
-    if not app.debug:
-        # Log to stderr
-        handler = logging.StreamHandler()
-        handler.setFormatter(
-            logging.Formatter(
-                "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
-            )
-        )
-        # Set app logger level to DEBUG
-        # otherwise only WARNING and above are propagated
-        app.logger.setLevel(logging.DEBUG)
-        handler.setLevel(logging.DEBUG)
-        app.logger.addHandler(handler)
+    app.logger.setLevel(logging.DEBUG)
     app.logger.info("CSEntry created!")
     # Remove variables that contain a password
     settings_to_display = {
@@ -107,6 +95,9 @@ def create_app(config=None):
     )
     app.logger.info(f"Settings:\n{settings_string}")
 
+    # Force RQ_DASHBOARD_REDIS_URL to RQ_REDIS_URL
+    app.config["RQ_DASHBOARD_REDIS_URL"] = app.config["RQ_REDIS_URL"]
+
     db.init_app(app)
     migrate.init_app(app)
     login_manager.init_app(app)

app/inventory/views.py

@@ -46,7 +46,7 @@ def list_items():
 @login_required
 def _generate_excel_file():
     task = current_user.launch_task(
-        "generate_items_excel_file", func="generate_items_excel_file", timeout=180
+        "generate_items_excel_file", func="generate_items_excel_file", job_timeout=180
     )
     db.session.commit()
     return utils.redirect_to_job_status(task.id)

app/main/views.py

@@ -81,7 +81,7 @@ def modified_static_file(endpoint, values):
 def get_redis_connection():
     redis_connection = getattr(g, "_redis_connection", None)
     if redis_connection is None:
-        redis_url = current_app.config["REDIS_URL"]
+        redis_url = current_app.config["RQ_REDIS_URL"]
         redis_connection = g._redis_connection = redis.from_url(redis_url)
     return redis_connection
 
@@ -99,6 +99,7 @@ def pop_rq_connection(exception=None):
 @bp.route("/")
 @login_required
 def index():
+    """Return the application index"""
     return render_template("index.html")
 
 

app/network/forms.py

@@ -154,7 +154,7 @@ class EditNetworkForm(CSEntryForm):
 class HostForm(CSEntryForm):
     name = StringField(
         "Hostname",
-        description="hostname must be 2-20 characters long and contain only letters, numbers and dash",
+        description="hostname must be 2-24 characters long and contain only letters, numbers and dash",
         validators=[
             validators.InputRequired(),
             validators.Regexp(HOST_NAME_RE),
@@ -165,7 +165,11 @@ class HostForm(CSEntryForm):
     )
     description = TextAreaField("Description")
     device_type_id = SelectField("Device Type")
-    is_ioc = BooleanField("IOC", default=False)
+    is_ioc = BooleanField(
+        "IOC",
+        default=False,
+        description="This host will be used to run IOCs",
+    )
     ansible_vars = YAMLField(
         "Ansible vars",
         description="Enter variables in YAML format. See https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html",
@@ -199,7 +203,7 @@ class InterfaceForm(CSEntryForm):
     )
     interface_name = StringField(
         "Interface name",
-        description="name must be 2-25 characters long and contain only letters, numbers and dash",
+        description="name must be 2-29 characters long and contain only letters, numbers and dash",
         validators=[
             validators.InputRequired(),
             validators.Regexp(INTERFACE_NAME_RE),
@@ -209,6 +213,7 @@ class InterfaceForm(CSEntryForm):
         ],
         filters=[utils.lowercase_field],
     )
+    interface_description = TextAreaField("Description")
     random_mac = BooleanField("Random MAC", default=False)
     mac = StringField(
         "MAC",
@@ -220,7 +225,7 @@ class InterfaceForm(CSEntryForm):
     )
     cnames_string = StringField(
         "Cnames",
-        description="space separated list of cnames (must be 2-20 characters long and contain only letters, numbers and dash)",
+        description="space separated list of cnames (must be 2-24 characters long and contain only letters, numbers and dash)",
         validators=[
             validators.Optional(),
             RegexpList(HOST_NAME_RE),

app/network/views.py

@@ -75,6 +75,8 @@ def create_host():
     del form.host_id
     # First interface name shall be identical to host name
     del form.interface_name
+    # Interface description can only be added when adding or editing interface
+    del form.interface_description
     if form.validate_on_submit():
         device_type_id = form.device_type_id.data
         network_id = form.network_id.data
@@ -108,7 +110,7 @@ def create_host():
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
             return render_template("network/create_host.html", form=form)
-        current_app.logger.debug(f"Trying to create: {host!r}")
+        current_app.logger.debug(f"Trying to create: {host}")
         db.session.add(host)
         try:
             db.session.commit()
@@ -117,6 +119,9 @@ def create_host():
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
         else:
+            current_app.logger.info(
+                f"Host {host} created by {current_user}: {host.to_dict()}"
+            )
             flash(f"Host {host} created!", "success")
         # Save network_id and device_type_id to the session to retrieve them after the redirect
         session["network_id"] = network_id
@@ -136,6 +141,7 @@ def delete_host():
     # defined on the model
     db.session.delete(host)
     db.session.commit()
+    current_app.logger.info(f"Host {host} deleted by {current_user}")
     flash(f"Host {host.name} has been deleted", "success")
     if host.device_type.name == "VirtualMachine":
         flash(
@@ -150,8 +156,7 @@ def delete_host():
 @login_required
 def view_host(name):
     host = models.Host.query.filter_by(name=name).first_or_404()
-    if not current_user.is_admin and host.sensitive:
-        # Only admin users can see hosts on sensitive networks
+    if not current_user.can_view_host(host):
         abort(403)
     if host.main_interface is None:
         flash(f"Host {host.name} has no interface! Add one or delete it.", "warning")
@@ -183,7 +188,7 @@ def view_host(name):
         ):
             if not current_user.can_set_boot_profile(host):
                 flash(
-                    f"You don't have the proper permissions to set the boot profile. Please contact an admin user.",
+                    "You don't have the proper permissions to set the boot profile. Please contact an admin user.",
                     "warning",
                 )
                 return redirect(url_for("network.view_host", name=name))
@@ -192,17 +197,16 @@ def view_host(name):
                 task = utils.trigger_set_network_boot_profile(
                     host, boot_profile=boot_profile
                 )
-                if boot_profile != "localboot":
-                    # For localboot, there is no need to update the variable
-                    # csentry_autoinstall_boot_profile is used for DHCP options
-                    if utils.update_ansible_vars(
-                        host, {"csentry_autoinstall_boot_profile": boot_profile}
-                    ):
-                        # If a change occured, force DHCP update
-                        utils.trigger_core_services_update()
+                # For localboot, there is no need to update the variable
+                # csentry_autoinstall_boot_profile is used for DHCP options
+                if boot_profile != "localboot" and utils.update_ansible_vars(
+                    host, {"csentry_autoinstall_boot_profile": boot_profile}
+                ):
+                    # If a change occured, force DHCP update
+                    utils.trigger_core_services_update()
                 db.session.commit()
                 current_app.logger.info(
-                    f"Set network boot profile to {boot_profile} for {name} requested: task {task.id}"
+                    f"Set network boot profile to {boot_profile} for {name} requested by {current_user}: task {task.id}"
                 )
                 flash(
                     f"Set network boot profile to {boot_profile} for {name} requested! "
@@ -213,7 +217,7 @@ def view_host(name):
         else:
             if not current_user.can_create_vm(host):
                 flash(
-                    f"You don't have the proper permissions to create this VM. Please contact an admin user.",
+                    "You don't have the proper permissions to create this VM. Please contact an admin user.",
                     "warning",
                 )
                 return redirect(url_for("network.view_host", name=name))
@@ -227,7 +231,9 @@ def view_host(name):
                     skip_post_install_job=form.skip_post_install_job.data,
                 )
                 db.session.commit()
-                current_app.logger.info(f"Creation of {name} requested: task {task.id}")
+                current_app.logger.info(
+                    f"Creation of {name} requested by {current_user}: task {task.id}"
+                )
                 flash(
                     f"Creation of {name} requested! Refresh the page to update the status.",
                     "success",
@@ -267,7 +273,7 @@ def edit_host(name):
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
             return render_template("network/edit_host.html", form=form)
-        current_app.logger.debug(f"Trying to update: {host!r}")
+        current_app.logger.debug(f"Trying to update: {host}")
         try:
             db.session.commit()
         except sa.exc.IntegrityError as e:
@@ -275,6 +281,9 @@ def edit_host(name):
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
         else:
+            current_app.logger.info(
+                f"Host {name} updated by {current_user}: {host.to_dict()}"
+            )
             flash(f"Host {host} updated!", "success")
         return redirect(url_for("network.view_host", name=host.name))
     return render_template("network/edit_host.html", form=form)
@@ -309,6 +318,7 @@ def create_interface(hostname):
             interface = models.Interface(
                 host=host,
                 name=form.interface_name.data,
+                description=form.interface_description.data,
                 ip=form.ip.data,
                 mac=form.mac.data,
                 network=network,
@@ -332,6 +342,9 @@ def create_interface(hostname):
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
         else:
+            current_app.logger.info(
+                f"Interface {interface} created by {current_user}: {interface.to_dict()}"
+            )
             flash(f"Interface {interface} created!", "success")
         return redirect(url_for("network.create_interface", hostname=hostname))
     return render_template(
@@ -350,6 +363,7 @@ def edit_interface(name):
         request.form,
         obj=interface,
         interface_name=interface.name,
+        interface_description=interface.description,
         cnames_string=cnames_string,
     )
     if not current_user.is_admin and not interface.is_main:
@@ -374,6 +388,7 @@ def edit_interface(name):
             abort(403)
         try:
             interface.name = form.interface_name.data
+            interface.description = form.interface_description.data
             interface.ip = form.ip.data
             interface.mac = form.mac.data
             # Setting directly network_id doesn't update the relationship and bypass the checks
@@ -412,6 +427,9 @@ def edit_interface(name):
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
         else:
+            current_app.logger.info(
+                f"Interface {name} updated by {current_user}: {interface.to_dict()}"
+            )
             flash(f"Interface {interface} updated!", "success")
         return redirect(url_for("network.view_host", name=interface.host.name))
     return render_template(
@@ -434,6 +452,7 @@ def delete_interface():
     # defined on the model
     db.session.delete(interface)
     db.session.commit()
+    current_app.logger.info(f"Interface {interface} deleted by {current_user}")
     flash(f"Interface {interface.name} has been deleted", "success")
     return redirect(url_for("network.view_host", name=hostname))
 
@@ -457,6 +476,7 @@ def delete_ansible_group():
     group = models.AnsibleGroup.query.get_or_404(request.form["group_id"])
     db.session.delete(group)
     db.session.commit()
+    current_app.logger.info(f"Group {group} deleted by {current_user}")
     flash(f"Group {group.name} has been deleted", "success")
     return redirect(url_for("network.list_ansible_groups"))
 
@@ -485,7 +505,8 @@ def edit_ansible_group(name):
     form.hosts.default = [host.id for host in group.hosts]
     form.hosts.process(request.form)
     # Same for AnsibleGroup children
-    form.children.default = [child.id for child in group.children]
+    # WARNING: use _children to not include groups automatically added to the children property
+    form.children.default = [child.id for child in group._children]
     form.children.process(request.form)
     if form.validate_on_submit():
         try:
@@ -509,6 +530,9 @@ def edit_ansible_group(name):
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
         else:
+            current_app.logger.info(
+                f"Group {name} updated by {current_user}: {group.to_dict()}"
+            )
             flash(f"Group {group} updated!", "success")
         return redirect(url_for("network.view_ansible_group", name=group.name))
     return render_template("network/edit_group.html", form=form)
@@ -543,6 +567,9 @@ def create_ansible_group():
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
         else:
+            current_app.logger.info(
+                f"Group {group} created by {current_user}: {group.to_dict()}"
+            )
             flash(f"Group {group} created!", "success")
         return redirect(url_for("network.view_ansible_group", name=group.name))
     return render_template("network/create_group.html", form=form)
@@ -570,20 +597,23 @@ def create_domain():
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
         else:
+            current_app.logger.info(
+                f"Domain {domain} created by {current_user}: {domain.to_dict()}"
+            )
             flash(f"Domain {domain} created!", "success")
         return redirect(url_for("network.create_domain"))
     return render_template("network/create_domain.html", form=form)
 
 
 @bp.route("/scopes")
-@login_groups_accepted("admin")
+@login_groups_accepted("admin", "auditor")
 def list_scopes():
     scopes = models.NetworkScope.query.all()
     return render_template("network/scopes.html", scopes=scopes)
 
 
 @bp.route("/scopes/view/<name>")
-@login_groups_accepted("admin")
+@login_groups_accepted("admin", "auditor")
 def view_scope(name):
     scope = models.NetworkScope.query.filter_by(name=name).first_or_404()
     return render_template("network/view_scope.html", scope=scope)
@@ -617,11 +647,51 @@ def create_scope():
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
         else:
+            current_app.logger.info(
+                f"Network scope {scope} created by {current_user}: {scope.to_dict()}"
+            )
             flash(f"Network Scope {scope} created!", "success")
         return redirect(url_for("network.create_scope"))
     return render_template("network/create_scope.html", form=form)
 
 
+@bp.route("/scopes/edit/<name>", methods=("GET", "POST"))
+@login_groups_accepted("admin")
+def edit_scope(name):
+    scope = models.NetworkScope.query.filter_by(name=name).first_or_404()
+    form = NetworkScopeForm(request.form, obj=scope)
+    if form.validate_on_submit():
+        try:
+            for field in (
+                "name",
+                "description",
+                "first_vlan",
+                "last_vlan",
+                "supernet",
+                "domain_id",
+            ):
+                setattr(scope, field, getattr(form, field).data)
+        except ValidationError as e:
+            # Check for error raised by model validation (not implemented in form validation)
+            current_app.logger.warning(f"{e}")
+            flash(f"{e}", "error")
+            return render_template("network/edit_scope.html", form=form)
+        current_app.logger.debug(f"Trying to update: {scope!r}")
+        try:
+            db.session.commit()
+        except sa.exc.IntegrityError as e:
+            db.session.rollback()
+            current_app.logger.warning(f"{e}")
+            flash(f"{e}", "error")
+        else:
+            current_app.logger.info(
+                f"Network scope {name} updated by {current_user}: {scope.to_dict()}"
+            )
+            flash(f"Network Scope {scope} updated!", "success")
+        return redirect(url_for("network.view_scope", name=scope.name))
+    return render_template("network/edit_scope.html", form=form)
+
+
 @bp.route("/_retrieve_first_available_ip/<int:network_id>")
 @login_required
 def retrieve_first_available_ip(network_id):
@@ -636,23 +706,21 @@ def retrieve_first_available_ip(network_id):
 
 
 @bp.route("/networks")
-@login_groups_accepted("admin", "network")
+@login_groups_accepted("admin", "auditor", "network")
 def list_networks():
     networks = models.Network.query.all()
-    if not current_user.is_admin:
+    if not (current_user.is_admin or current_user.is_auditor):
         networks = [
-            network
-            for network in networks
-            if current_user.has_access_to_network(network)
+            network for network in networks if current_user.can_view_network(network)
         ]
     return render_template("network/networks.html", networks=networks)
 
 
 @bp.route("/networks/view/<vlan_name>")
-@login_groups_accepted("admin", "network")
+@login_groups_accepted("admin", "auditor", "network")
 def view_network(vlan_name):
     network = models.Network.query.filter_by(vlan_name=vlan_name).first_or_404()
-    if not current_user.has_access_to_network(network):
+    if not current_user.can_view_network(network):
         abort(403)
     return render_template("network/view_network.html", network=network)
 
@@ -698,6 +766,9 @@ def create_network():
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
         else:
+            current_app.logger.info(
+                f"Network {network} created by {current_user}: {network.to_dict()}"
+            )
             flash(f"Network {network} created!", "success")
         # Save scope_id to the session to retrieve it after the redirect
         session["scope_id"] = scope_id
@@ -740,6 +811,9 @@ def edit_network(vlan_name):
             current_app.logger.warning(f"{e}")
             flash(f"{e}", "error")
         else:
+            current_app.logger.info(
+                f"Network {vlan_name} updated by {current_user}: {network.to_dict()}"
+            )
             flash(f"Network {network} updated!", "success")
         return redirect(url_for("network.view_network", vlan_name=network.vlan_name))
     return render_template("network/edit_network.html", form=form)

app/settings.py

@@ -32,7 +32,7 @@ SESSION_TYPE = "redis"
 SESSION_REDIS_URL = "redis://redis:6379/0"
 CACHE_TYPE = "redis"
 CACHE_REDIS_URL = "redis://redis:6379/1"
-REDIS_URL = "redis://redis:6379/2"
+RQ_REDIS_URL = "redis://redis:6379/2"
 
 ELASTICSEARCH_URL = "http://elasticsearch:9200"
 ELASTICSEARCH_INDEX_SUFFIX = "-dev"
@@ -42,8 +42,8 @@ ELASTICSEARCH_INDEX_SUFFIX = "-dev"
 ELASTICSEARCH_REFRESH = "false"
 
 LDAP_HOST = os.environ.get("LDAP_HOST", "esss.lu.se")
-LDAP_PORT = int(os.environ.get("LDAP_PORT", 389))
-LDAP_USE_SSL = os.environ.get("LDAP_USE_SSL", "false").lower() == "true"
+LDAP_PORT = int(os.environ.get("LDAP_PORT", 636))
+LDAP_USE_SSL = os.environ.get("LDAP_USE_SSL", "true").lower() == "true"
 LDAP_BASE_DN = os.environ.get("LDAP_BASE_DN", "DC=esss,DC=lu,DC=se")
 LDAP_USER_DN = os.environ.get("LDAP_USER_DN", "")
 LDAP_GROUP_DN = os.environ.get("LDAP_GROUP_DN", "")
@@ -73,6 +73,7 @@ LDAP_GET_GROUP_ATTRIBUTES = os.environ.get("LDAP_GET_GROUP_ATTRIBUTES", "cn").sp
 # on all CSENTRY_NETWORK_SCOPES_LDAP_GROUPS
 CSENTRY_LDAP_GROUPS = {
     "admin": ["ICS Control System Infrastructure group"],
+    "auditor": ["ICS Control System Infrastructure group"],
     "inventory": ["ICS Employees", "ICS Consultants"],
 }
 # Network scopes the user has access to based on LDAP groups