app/tasks.py

@@ -10,7 +10,6 @@ This module implements tasks to run.
 
 """
 import time
-import traceback
 import tower_cli
 from datetime import datetime
 from flask import current_app
@@ -26,18 +25,19 @@ class TaskWorker(Worker):
     the task status and end time in the CSEntry database
     """
 
-    def save_exception(self, job, *exc_info):
+    @staticmethod
+    def save_exception(job, exc_string):
         """Save the exception to the database
 
         The exception is only saved if it occured before the AWX job was triggered.
         If the AWX job failed, we can refer to the logs on AWX.
         """
         task = models.Task.query.get(job.id)
+        if task is None:
+            return
         if task.awx_job_id is None:
             # No AWX job was triggered. An exception occured before. Save it.
-            task.exception = self._get_safe_exception_string(
-                traceback.format_exception(*exc_info)
-            )
+            task.exception = exc_string
             db.session.commit()
 
     def update_task_attributes(self, job, attributes):
@@ -62,23 +62,23 @@ class TaskWorker(Worker):
             setattr(task, name, value)
         db.session.commit()
 
-    def update_reverse_dependencies(self, job):
+    @staticmethod
+    def update_reverse_dependencies(job):
         task = models.Task.query.get(job.id)
         if task is None:
             return
         task.update_reverse_dependencies()
         db.session.commit()
 
-    def move_to_failed_queue(self, job, *exc_info):
-        # This could be achieved by passing a custom exception handler
-        # when initializing the worker. As we already subclass it, it's
-        # easier to override the default handler in case of failure
+    def handle_job_failure(self, job, queue, started_job_registry=None, exc_string=""):
         self.update_task_attributes(
             job, {"ended_at": job.ended_at, "status": models.JobStatus.FAILED}
         )
         self.update_reverse_dependencies(job)
-        self.save_exception(job, *exc_info)
-        super().move_to_failed_queue(job, *exc_info)
+        self.save_exception(job, exc_string)
+        super().handle_job_failure(
+            job, queue, started_job_registry=started_job_registry, exc_string=exc_string
+        )
 
     def handle_job_success(self, job, queue, started_job_registry):
         self.update_task_attributes(
@@ -86,18 +86,18 @@ class TaskWorker(Worker):
         )
         super().handle_job_success(job, queue, started_job_registry)
 
-    def prepare_job_execution(self, job):
+    def prepare_job_execution(self, job, heartbeat_ttl=None):
         self.update_task_attributes(job, {"status": models.JobStatus.STARTED})
-        super().prepare_job_execution(job)
+        super().prepare_job_execution(job, heartbeat_ttl)
 
 
 def launch_awx_job(resource="job", **kwargs):
-    """Launch an AWX job
+    r"""Launch an AWX job
 
     job_template or inventory_source shall be passed as keyword argument
 
     :param resource: job|workflow_job|inventory_source
-    :param **kwargs: keyword arguments passed to launch the job
+    :param \*\*kwargs: keyword arguments passed to launch the job
     :returns: A dictionary with information from resource.monitor
     """
     rq_job = get_current_job()

app/templates/_helpers.html

@@ -136,7 +136,12 @@
       {{ field(class_="form-check-input", **kwargs) }}
       {{ field.label(class_="form-check-label") }}
     </div>
-  </div>
+    {% if field.description %}
+    <small class="form-text text-muted">
+      {{ field.description|safe }}
+    </small>
+    {% endif %}
+</div>
   {% else %}
   {{ field.label(class_="col-sm-" + label_size + " col-form-label") }}
   <div class="col-sm-{{ input_size }}">
@@ -184,7 +189,7 @@
 </div>
 {%- endmacro %}
 
-{% macro submit_button_with_confirmation(title, message) -%}
+{% macro submit_button_with_confirmation(title, message, message2) -%}
 <button type="button" class="btn btn-primary" data-toggle="modal" data-target="#submitModal" }}>
   {{ title }}
 </button>
@@ -194,7 +199,7 @@
   <div class="modal-dialog" role="document">
     <div class="modal-content">
       <div class="modal-header">
-        <h6 class="modal-title">{{ message }}</h6>
+        <h6 class="modal-title">{{ message }} {% if message2 is defined %}<br/><br/><br/> {{ message2 }} {% endif %}</h6>
         <button type="button" class="close" data-dismiss="modal" aria-label="Close">
           <span aria-hidden="true">&times;</span>
         </button>
@@ -277,4 +282,4 @@
   </div>
   <div class="card-body item-comment">{{ comment.body }}</div>
 </div>
-{%- endmacro %}
\ No newline at end of file
+{%- endmacro %}

app/templates/base.html

@@ -78,6 +78,7 @@
   <script type=text/javascript>
     $SCRIPT_ROOT = {{ request.script_root|tojson|safe }};
   </script>
+  <script src="{{ url_for('static', filename='js/history.js') }}"></script>
   <script src="{{ url_for('static', filename='js/csentry.js') }}"></script>
   {% block csentry_scripts %}{% endblock %}
 {% endblock %}

app/templates/network/create_interface.html

@@ -21,6 +21,7 @@
     {{ form.hidden_tag() }}
     {{ render_field(form.host_id, disabled=True) }}
     {{ render_field(form.interface_name, class_="text-lowercase") }}
+    {{ render_field(form.interface_description) }}
     {{ render_field(form.network_id, class_="selectize-default") }}
     {{ render_field(form.ip) }}
     {{ render_field(form.random_mac) }}

app/templates/network/edit_interface.html

@@ -24,6 +24,7 @@
     {{ form.hidden_tag() }}
     {{ render_field(form.host_id, disabled=True) }}
     {{ render_field(form.interface_name, class_="text-lowercase") }}
+    {{ render_field(form.interface_description) }}
     {{ render_field(form.network_id, class_="selectize-default") }}
     {{ render_field(form.ip) }}
     {{ render_field(form.mac) }}

app/templates/network/edit_scope.html

+{% extends "network/scopes.html" %}
+{% from "_helpers.html" import render_field %}
+
+{% block title %}Edit {{ form.name.data }} network Scope{% endblock %}
+
+{% block scopes_nav %}
+    <li class="nav-item">
+      <a class="nav-link" href="{{ url_for('network.view_scope', name=form.name.data) }}">View network scope</a>
+    </li>
+    <li class="nav-item">
+      <a class="nav-link active" href="{{ url_for('network.edit_scope', name=form.name.data) }}">Edit network scope</a>
+    </li>
+{% endblock %}
+
+{% block scopes_main %}
+  <form id="editScopeForm" method="POST">
+    {{ form.hidden_tag() }}
+    {{ render_field(form.name) }}
+    {{ render_field(form.description) }}
+    {{ render_field(form.first_vlan) }}
+    {{ render_field(form.last_vlan) }}
+    {{ render_field(form.supernet) }}
+    {{ render_field(form.domain_id) }}
+    <div class="form-group row">
+      <div class="col-sm-10">
+        <button type="submit" class="btn btn-primary">Submit</button>
+      </div>
+    </div>
+  </form>
+{%- endblock %}

app/templates/network/view_host.html

@@ -80,7 +80,7 @@
           {% if current_user.is_admin %}
           {{ render_field(form.skip_post_install_job, label_size='5', input_size='7') }}
           {% endif %}
-          {{ submit_button_with_confirmation('Create ' + vm_type, 'Do you really want to create the ' + vm_type + ' ' + host.name + '?') }}
+          {{ submit_button_with_confirmation('Create ' + vm_type, 'Do you really want to create the ' + vm_type + ' ' + host.name + '?', 'Note: If you are experiencing issues with the VM, pressing "Create VM" once more will not resolve anything and you should instead contact the INFRA team or create a JIRA ticket to resolve this.') }}
         </form>
       </div>
     {% elif host.device_type.name in config.ALLOWED_SET_BOOT_PROFILE_DEVICE_TYPES %}
@@ -99,6 +99,7 @@
       <tr>
         <th width="5%"></th>
         <th>Name</th>
+        <th>Description</th>
         <th>Cnames</th>
         <th>IP</th>
         <th>MAC</th>
@@ -119,6 +120,7 @@
           </form>
         </td>
         <td>{{ interface.name }}</td>
+        <td>{{ interface.description if interface.description }}</td>
         <td>{{ interface.cnames | join(' ') }}</td>
         <td>{{ interface.ip }}</td>
         <td>{{ interface.mac }}</td>

app/templates/network/view_scope.html

@@ -7,6 +7,9 @@
 <li class="nav-item">
   <a class="nav-link active" href="{{ url_for('network.view_scope', name=scope.name) }}">View network scope</a>
 </li>
+<li class="nav-item">
+  <a class="nav-link" href="{{ url_for('network.edit_scope', name=scope.name) }}">Edit network scope</a>
+</li>
 {% endblock %}
 
 {% block scopes_main %}

app/user/views.py

@@ -29,6 +29,7 @@ bp = Blueprint("user", __name__)
 
 @bp.route("/login", methods=["GET", "POST"])
 def login():
+    """Login page"""
     form = LDAPLoginForm(request.form)
     if form.validate_on_submit():
         login_user(form.user, remember=form.remember_me.data)
@@ -39,6 +40,7 @@ def login():
 @bp.route("/logout")
 @login_required
 def logout():
+    """Logout endpoint"""
     logout_user()
     return redirect(url_for("user.login"))
 
@@ -46,6 +48,7 @@ def logout():
 @bp.route("/profile", methods=["GET", "POST"])
 @login_required
 def profile():
+    """User profile"""
     # Try to get the generated token from the session
     token = session.pop("generated_token", None)
     form = TokenForm(request.form)
@@ -70,6 +73,7 @@ def profile():
 @bp.route("/tokens/revoke", methods=["POST"])
 @login_required
 def revoke_token():
+    """Endpoint to revoke a token"""
     token_id = request.form["token_id"]
     jti = request.form["jti"]
     try:

app/utils.py

@@ -186,7 +186,7 @@ def get_query(query, model, **kwargs):
         # Always apply filtering on the given model
         for key, value in kwargs.items():
             query = query.filter(getattr(model, key) == value)
-    except (sa.exc.InvalidRequestError, AttributeError) as e:
+    except (sa.exc.InvalidRequestError, sa.exc.ArgumentError, AttributeError) as e:
         current_app.logger.warning(f"Invalid query arguments: {e}")
         raise CSEntryError("Invalid query arguments", status_code=422)
     return query
@@ -346,7 +346,9 @@ def trigger_ansible_groups_reindex():
     Make sure that we don't have more than one in queue.
     """
     return trigger_job_once(
-        "reindex_ansible_groups", queue_name="low", func="reindex_ansible_groups",
+        "reindex_ansible_groups",
+        queue_name="low",
+        func="reindex_ansible_groups",
     )
 
 
@@ -553,11 +555,10 @@ def validate_ip(ip, network):
         is_admin = current_user.is_admin
     except AttributeError:
         is_admin = False
-    if not is_admin:
-        if addr < network.first or addr > network.last:
-            raise ValidationError(
-                f"IP address {ip} is not in range {network.first} - {network.last}"
-            )
+    if (not is_admin) and (addr < network.first or addr > network.last):
+        raise ValidationError(
+            f"IP address {ip} is not in range {network.first} - {network.last}"
+        )
 
 
 def overlaps(subnet, subnets):

app/validators.py

@@ -15,8 +15,10 @@ import sqlalchemy as sa
 from wtforms import ValidationError, SelectField
 
 ICS_ID_RE = re.compile(r"^[A-Z]{3}[0-9]{3}$")
-HOST_NAME_RE = re.compile(r"^[a-z0-9\-]{2,20}$")
-INTERFACE_NAME_RE = re.compile(r"^[a-z0-9\-]{2,25}$")
+HOST_NAME_RE = re.compile(r"^[a-z0-9\-]{2,24}$")
+GROUP_NAME_RE = re.compile(r"^[a-z0-9\-\_]{2,50}$")
+# Interface name needs to be at least 5 characters more than the hostname (Every interface name have to start with the hostname)
+INTERFACE_NAME_RE = re.compile(r"^[a-z0-9\-]{2,29}$")
 VLAN_NAME_RE = re.compile(r"^[A-Za-z0-9\-]{3,25}$")
 MAC_ADDRESS_RE = re.compile(r"^(?:[0-9a-fA-F]{2}[:-]?){5}[0-9a-fA-F]{2}$")
 DEVICE_TYPE_RE = re.compile(r"^[A-Za-z0-9\-]{3,25}$")

docs/Makefile

 # Minimal makefile for Sphinx documentation
 #
 
-# You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-SPHINXPROJ    = CSEntry
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
 
@@ -17,4 +17,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/CS_entry_v02_blue.svg

-<?xml version="1.0" encoding="utf-8"?>
-<!-- Generator: Adobe Illustrator 23.0.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 2560 854" style="enable-background:new 0 0 2560 854;" xml:space="preserve">
-<style type="text/css">
-	.st0{fill:#0094CA;}
-</style>
-<g id="Layer_1">
-	<g>
-		<path class="st0" d="M886.8,303.7c-67.8-15.7-130.9-36.1-130.9-66.2c0-18.1,18.5-44.6,72.4-44.6c75.5,0,78.5,36.1,80.1,50.6h251
-			c-21.6-175.8-215.6-210.8-328-210.8c-82.9,0-160.7,19.3-220.1,53.9c2,1.5,3.9,3,5.9,4.6c67.8,54.1,114.8,131,132.4,216.5l6.2,30.1
-			H527.8c35.5,45.9,101.4,77.4,175.7,95.9c163.2,49.4,226.4,55.4,226.4,101.2c0,32.5-46.2,51.8-89.3,51.8c-10.8,0-86.2,0-97-61.4
-			h-8.6c-23.9,61.6-63.3,116.7-114.1,158.7c-4.1,3.4-8.4,6.7-12.6,10c70.8,39.7,159.6,52.9,229.2,52.9c189.4,0,348-86.7,348-233.6
-			C1185.5,368.8,1031.5,335,886.8,303.7z"/>
-		<g>
-			<path class="st0" d="M726.2,469.6c-36.2,152-170.1,276.2-351,276.2C172.6,745.8,17,586.6,17,387.6C17,191,170.2,31.8,370.4,31.8
-				c177.3,0,320.8,117,354.6,281H522.3c-21.7-45.8-61.5-94.1-144.7-94.1c-47-2.4-86.8,15.7-115.8,47c-27.7,31.4-43.4,74.8-43.4,123
-				c0,97.7,63.9,170.1,159.2,170.1c83.2,0,123-48.2,144.7-89.2H726.2z"/>
-		</g>
-		<g>
-			<path class="st0" d="M1312.9,628.4c3.5,29.7,31.8,49.4,64.5,49.4c26.7,0,40.8-11.6,50.9-26.2h85.7c-13.6,31.3-33.3,55.5-57,71.6
-				c-23.2,16.6-50.9,25.2-79.7,25.2c-80.2,0-148.2-65-148.2-148.2c0-78.2,61.5-150.3,146.7-150.3c42.9,0,79.7,16.6,105.9,44.4
-				c35.3,37.8,45.9,82.7,39.3,134.1H1312.9z M1438.9,564.9c-2-13.1-19.2-44.4-62.5-44.4c-43.4,0-60.5,31.3-62.5,44.4H1438.9z"/>
-			<path class="st0" d="M1559.9,459.5h78.2v29.7c9.6-13.6,27.2-39.3,77.1-39.3c94.3,0,103.9,76.6,103.9,114.5v175h-83.7V586.6
-				c0-30.8-6.6-58-43.9-58c-41.3,0-47.9,29.7-47.9,58.5v152.3h-83.7V459.5z"/>
-			<path class="st0" d="M1895.2,523.1h-41.3v-63.5h41.3v-93.3h83.7v93.3h40.3v63.5h-40.3v216.3h-83.7V523.1z"/>
-			<path class="st0" d="M2055.1,459.5h78.7v31.3c8.1-15.1,22.7-40.8,71.1-40.8v84.2h-3c-42.9,0-63,15.6-63,55v150.3h-83.7V459.5z"/>
-			<path class="st0" d="M2332.9,725.2L2229,459.5h89.2l57,162.4l53.4-162.4h88.7l-140.2,373.1h-88.7L2332.9,725.2z"/>
-		</g>
-	</g>
-</g>
-<g id="lines">
-</g>
-<g id="rest">
-</g>
-</svg>

docs/conf.py

-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
+# Configuration file for the Sphinx documentation builder.
 #
-# CSEntry documentation build configuration file, created by
-# sphinx-quickstart on Sun Feb  4 20:26:41 2018.
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -23,11 +16,22 @@ import sys
 sys.path.insert(0, os.path.abspath(".."))
 
 
-# -- General configuration ------------------------------------------------
+# -- Project information -----------------------------------------------------
 
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
+project = "CSEntry"
+copyright = "2020, European Spallation Source ERIC"
+author = "Benjamin Bertrand"
+
+try:
+    # CI_COMMIT_REF_NAME is defined by GitLab Runner
+    # The branch or tag name for which project is built
+    release = os.environ["CI_COMMIT_REF_NAME"]
+except KeyError:
+    # dev mode
+    release = os.popen("git describe").read().strip()
+
+
+# -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -44,68 +48,21 @@ extensions = [
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = ".rst"
-
-# The master toctree document.
-master_doc = "index"
-
-# General information about the project.
-project = "CSEntry"
-copyright = "2018, Benjamin Bertrand"
-author = "Benjamin Bertrand"
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-try:
-    # CI_COMMIT_TAG is defined by GitLab Runner when building tags
-    version = os.environ["CI_COMMIT_TAG"]
-except KeyError:
-    # dev mode
-    version = os.popen("git describe").read().strip()
-# The full version, including alpha/beta/rc tags.
-release = version
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-# This patterns also effect to html_static_path and html_extra_path
+# This pattern also affects html_static_path and html_extra_path.
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = "sphinx"
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = False
-
 
-# -- Options for HTML output ----------------------------------------------
+# -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = "alabaster"
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
+html_theme = "sphinx_rtd_theme"
+html_logo = "_static/CS_entry_v02_white_512.png"
 html_theme_options = {
-    "logo": "CS_entry_v02_blue.svg",
-    "description": "Control System Entry",
-    "fixed_sidebar": True,
+    "logo_only": True,
 }
 
 # Add any paths that contain custom static files (such as style sheets) here,
@@ -113,76 +70,6 @@ html_theme_options = {
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
 
-# Custom sidebar templates, must be a dictionary that maps document names
-# to template names.
-#
-# This is required for the alabaster theme
-# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
-html_sidebars = {
-    "**": [
-        "about.html",
-        "navigation.html",
-        "relations.html",  # needs 'show_related': True theme option to display
-        "searchbox.html",
-    ]
-}
-
-
-# -- Options for HTMLHelp output ------------------------------------------
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = "CSEntrydoc"
-
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    (master_doc, "CSEntry.tex", "CSEntry Documentation", "Benjamin Bertrand", "manual")
-]
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [(master_doc, "csentry", "CSEntry Documentation", [author], 1)]
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-    (
-        master_doc,
-        "CSEntry",
-        "CSEntry Documentation",
-        author,
-        "CSEntry",
-        "One line description of project.",
-        "Miscellaneous",
-    )
-]
-
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {

docs/development/deployment.rst

+Deployment
+==========
+
+Deployment is performed using Ansible_ and docker_. The application is made of the following containers:
+
+- csentry_web to run the main Flask_ application with uwsgi_
+- csentry_workers_<index> to run RQ_ workers (same image as the main application)
+- csentry_postgres to run PostgreSQL_
+- csentry_elasticsearch to run Elasticsearch_
+- csentry_redis to run Redis_
+
+Refer to the `CSEntry Ansible role`_ for details.
+
+All application `default settings <https://gitlab.esss.lu.se/ics-infrastructure/csentry/-/blob/master/app/settings.py>`_
+can be overridden using a local `settings.cfg` file.
+For deployment, this local file is defined in the `CSEntry Ansible playbook`_:
+
+- `config/settings-prod.cfg <https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-csentry/-/blob/master/config/settings-prod.cfg>`_ for production
+- `config/settings-test.cfg <https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-csentry/-/blob/master/config/settings-test.cfg>`_ for staging
+
+
+.. _docker: https://www.docker.com
+.. _Ansible: https://docs.ansible.com/ansible/latest/index.html
+.. _Flask: https://flask.palletsprojects.com
+.. _uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/
+.. _PostgreSQL: https://www.postgresql.org
+.. _Redis: https://redis.io
+.. _Elasticsearch: https://www.elastic.co/elasticsearch/
+.. _RQ: https://python-rq.org
+.. _CSEntry Ansible role: https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-role-csentry
+.. _CSEntry Ansible playbook: https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-csentry

docs/development/design.rst

+Design
+======
+
+CSEntry is a web application developed using Flask_, one of the most popular Python web frameworks.
+Many principles follow the concepts described in `The Flask Mega-Tutorial`_ from `Miguel Grinberg <https://blog.miguelgrinberg.com/index>`_.
+
+The application relies on:
+
+- PostgreSQL_ as main database
+- Redis_ for caching and for queuing jobs and processing them in the background with workers using RQ_
+- Elasticsearch_ for search
+
+Database
+--------
+
+PostgreSQL_ is used as the main database.
+Interactions with the database are perfomed using SQLAlchemy_ ORM, which maps SQL tables to Python classes.
+All models are defined in the :mod:`app.models` file.
+
+Database migrations are performed using alembic_.
+When the database is modified, a new migration script can be autogenerated using::
+
+    docker-compose run --rm web flask db migrate -m "revision message"
+
+Resulting script should be modified according to the desired behaviour. Looking at previously migration scripts
+under the `migration/versions <https://gitlab.esss.lu.se/ics-infrastructure/csentry/-/tree/master/migrations/versions>`_
+directory can help.
+
+
+Full-text Search
+----------------
+
+Search is performed using Elasticsearch_ which provides very powerful capabilities.
+Implementation was inspired by `The Flask Mega-Tutorial Part XVI <https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-xvi-full-text-search>`_.
+
+.. note::
+
+   Elasticsearch index has to be kept in sync with the Postgres database. This is done using SQLAlchemy event listeners.
+
+The `List items <https://csentry.esss.lu.se/inventory/items>`_ and `List hosts <https://csentry.esss.lu.se/network/hosts>`_ pages
+use Elasticsearch_ to display the paginated list of results.
+If the index is empty or not up-to-date, missing items/hosts won't be displayed (even if they are in the postgres database).
+
+To make a class searchable, it should be a subclass of the :class:`~app.models.SearchableMixin` class and define the fields to index in the `__mapping__` class attribute.
+By default, the object will be kept in sync in the index thanks to the event listeners defined in the :class:`~app.models.SearchableMixin` class.
+Note that if you define a field that isn't a database column, you have to make sure it is kept up-to-date in the Elasticsearch index.
+An example is the :attr:`~app.models.Host.sensitive` field on the :class:`~app.models.Host` class. It's a property that comes from the network.
+Updating the sensitive field on a network won't trigger an update of the Host objects.
+A specific event listener :meth:`~app.models.update_host_sensitive_field` had to be implemented in that case.
+
+
+
+Background Tasks
+----------------
+
+A web application should reply to a client as fast as possible.
+Long running tasks can't be performed in the main process and block the response.
+
+Solving this is usually done using a task queue, a mechanism to send tasks to workers.
+In CSEntry, background tasks are run using RQ_, which is backed by Redis_.
+
+A subclass of RQ_ `Worker` is used to keep tasks results in the Postgres database: :class:`~app.tasks.TaskWorker`.
+This allows the user to see the `tasks <https://csentry.esss.lu.se/task/tasks>`_ that he created.
+Admin users have access to all tasks.
+
+The workers use the same docker image as the main application.
+They are started using the `flask runworker` subcommand, giving them access to the same settings as the web application.
+
+Some tasks are triggered automatically, based on SQLAlchemy event listeners, to keep the csentry inventory in sync in AWX for example.
+Some are triggered manually, like to create a VM for example.
+
+Most tasks are used to trigger templates in AWX via the API.
+But they can be used for internal processing as well, like reindexing Ansible groups in Elasticsearch_.
+
+Permissions
+-----------
+
+Using CSEntry requires each user to login with his/her ESS username.
+Any logged user has read access to most information, except `network scopes <https://csentry.esss.lu.se/network/scopes>`_
+and sensitive `networks <https://csentry.esss.lu.se/network/networks>`_.
+
+Write access requires to be part of a specific group.
+There are several roles/groups in CSEntry.
+Each internal group has to be mapped to a list of LDAP groups.
+
+Internal groups
++++++++++++++++
+
+admin
+~~~~~
+
+Admin users have full power and access to the application.
+They can modify anything via the `admin interface <https://csentry.esss.lu.se/admin/>`_.
+They can also monitor background jobs via the `RQ Dashboard <https://csentry.esss.lu.se/rq/>`_.
+
+auditor
+~~~~~~~
+
+Auditor users have read-only access to everything in the application (including sensitive networks).
+They don't have any write access.
+
+inventory
+~~~~~~~~~
+
+Users part of the inventory group have read and write access
+to the `Inventory <https://csentry.esss.lu.se/inventory/items>`_ to register items.
+
+network
+~~~~~~~
+
+*network* is used to gather groups based on the network scope. It shouldn't be mapped to a LDAP group.
+Only network scope groups should.
+
+If a user is part of a network scope group, it is added automatically to the *network* group.
+The user will have write access to all hosts in this scope, including on sensitive networks.
+He will still have read-only access to hosts on admin only networks.
+
+Admin users have automatically access to all network scopes.
+
+Configuration
++++++++++++++
+
+The *admin*, *auditor* and *inventory* groups mapping shall be defined in the :attr:`~app.settings.CSENTRY_LDAP_GROUPS` dictionary::
+
+    CSENTRY_LDAP_GROUPS = {
+        "admin": ["LDAP admin group"],
+        "auditor": ["another group"],
+        "inventory": ["group1", "group2"],
+    }
+
+For networks, groups based on the network scope name shall be defined
+in the :attr:`~app.settings.CSENTRY_NETWORK_SCOPES_LDAP_GROUPS` dictionary::
+
+    CSENTRY_NETWORK_SCOPES_LDAP_GROUPS = {
+        "TechnicalNetwork": ["group-tn"],
+        "LabNetworks": ["group-lab", "group-tn"],
+    }
+
+With the above settings, a user part of *group-tn* will have access to both the `TechnicalNetwork` and `LabNetworks` scopes.
+While a user part of the *group-lab* will only have access to the `LabNetworks` scope.
+If a network scope isn't defined in this dictionary, only admin users will have access to it.
+
+Usage
++++++
+
+Every endpoint should be protected using either the `login_required` or :meth:`~app.decorators.login_groups_accepted` decorator.
+The first will give access to any logged user. With the second, a list of internal groups should be given.
+
+To restrict access to admin users only::
+
+    @login_groups_accepted("admin")
+    def create_domain():
+        ...
+
+
+When using the *network* group, an additional check inside the function is required to check that the current user
+has the proper access (based on the network)::
+
+    @login_groups_accepted("admin", "network")
+    def edit_interface(name):
+        interface = models.Interface.query.filter_by(name=name).first_or_404()
+        if not current_user.has_access_to_network(interface.network):
+            abort(403)
+
+Database versioning
+-------------------
+
+SQLAlchemy-Continuum_ is used to track changes and keep an history.
+
+To enable versioning on a models, the ``__versioned__`` attribute shall be added to the model class.
+The following classes are versioned:
+
+-  :class:`~app.models.Item`
+-  :class:`~app.models.Host`
+-  :class:`~app.models.AnsibleGroup`
+
+The *History* on the view host or view group page displays the list of changes performed.
+As there is a relationship between Host and AnsibleGroup, it can lead to unexpected behavior.
+
+1. Edit a group to add a host -> new host added to the group history
+2. Edit another host to add it to that group -> new group added to the host history
+
+At this point the group page will show 2 hosts but only one was added in the history.
+If the group is edited, to add a variable, the recorded change will display all current hosts in the history.
+This is correct as those hosts were present when the group was edited.
+But the information showing the hosts were added previously is missing.
+This is a limitation of the current implementation.
+
+
+
+.. _Flask: https://flask.palletsprojects.com
+.. _The Flask Mega-Tutorial: https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-i-hello-world
+.. _PostgreSQL: https://www.postgresql.org
+.. _SQLAlchemy: https://www.sqlalchemy.org
+.. _Redis: https://redis.io
+.. _Elasticsearch: https://www.elastic.co/elasticsearch/
+.. _RQ: https://python-rq.org
+.. _alembic: https://alembic.sqlalchemy.org
+.. _SQLAlchemy-Continuum: https://sqlalchemy-continuum.readthedocs.io/en/latest/

docs/development/endpoints.rst

+Endpoints
+=========
+
+Main blueprint
+--------------
+
+.. autoflask:: wsgi:app
+   :modules: app.main.views
+   :include-empty-docstring:
+   :order: path
+
+User blueprint
+--------------
+
+.. autoflask:: wsgi:app
+   :modules: app.user.views
+   :include-empty-docstring:
+   :order: path
+
+Inventory blueprint
+-------------------
+
+.. autoflask:: wsgi:app
+   :modules: app.inventory.views
+   :include-empty-docstring:
+   :order: path
+
+Network blueprint
+-----------------
+
+.. autoflask:: wsgi:app
+   :modules: app.network.views
+   :include-empty-docstring:
+   :order: path
\ No newline at end of file

docs/development/implementation/commands.rst

+.. automodule:: app.commands
+   :members:
+   :undoc-members:

docs/development/implementation/decorators.rst

+.. automodule:: app.decorators
+   :members:
+   :undoc-members: