.dockerignore

-.git
 **/*.swp
 **/__pycache__
 settings*.cfg
@@ -8,3 +7,7 @@ Jenkinsfile
 Makefile
 docker-compose*.yml
 .gitlab-ci.yml
+.cache
+.coverage
+.pytest_cache
+docs/_build

.env

 POSTGRES_USER=ics
 POSTGRES_PASSWORD=icspwd
 POSTGRES_DB=csentry_db
-PGDATA_VOLUME=./data
+PGDATA_VOLUME=./data/postgres
+ELASTIC_DATA_VOLUME=./data/elastic

.gitignore

@@ -12,3 +12,7 @@ coverage.xml
 junit.xml
 .pytest_cache
 .tower_cli.cfg
+/app/static/files/*
+!/app/static/files/.empty
+.vscode
+.mypy_cache

.gitlab-ci.yml

-image: docker:latest
-
-.runner_tags: &runner_tags
-  tags:
-    - docker
+---
+include:
+  - remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/PreCommit.gitlab-ci.yml"
+  - remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/SonarScanner.gitlab-ci.yml"
 
 variables:
   CONTAINER_TEST_IMAGE: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME"
@@ -20,52 +19,46 @@ stages:
   - release
   - deploy
 
+default:
+  tags:
+    - docker
+
 before_script:
   - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
 
-check:
-  <<: *runner_tags
-  stage: check
-  image: registry.esss.lu.se/ics-docker/pre-commit:latest
-  before_script: []
-  script:
-    - pre-commit run --all-files
-
 build:
-  <<: *runner_tags
   stage: build
+  image: docker:latest
   script:
     - docker pull "$CONTAINER_CACHE_IMAGE" || true
     - docker build --pull --cache-from "$CONTAINER_CACHE_IMAGE" -t "$CONTAINER_TEST_IMAGE" .
     - docker push "$CONTAINER_TEST_IMAGE"
 
 test:
-  <<: *runner_tags
   stage: test
   image: "$CONTAINER_TEST_IMAGE"
   services:
     - postgres:10
     - redis:4.0
-  before_script: []
+    - name: docker.elastic.co/elasticsearch/elasticsearch:7.3.2
+      alias: elasticsearch
+      command: ["bin/elasticsearch", "-Ediscovery.type=single-node"]
+  before_script:
+    - pip install -r requirements-dev.txt
   script:
-    - pytest --junitxml=junit.xml --cov-report xml:coverage.xml --cov=app -v
+    - pytest --junitxml=junit.xml --cov-report=xml:coverage.xml --cov-report=term --cov=app -v
   artifacts:
     paths:
       - junit.xml
       - coverage.xml
+    reports:
+      junit: junit.xml
     expire_in: 1 hour
 
-analyse:
-  <<: *runner_tags
-  stage: analyse
-  image: registry.esss.lu.se/ics-docker/sonar-scanner:3
-  before_script: []
-  script:
-    - sonar-scanner -Dsonar.login=$SONARQUBE_TOKEN -Dsonar.projectVersion=$CI_COMMIT_REF_NAME
-
 release-image:
-  <<: *runner_tags
   stage: release
+  image: docker:latest
+  dependencies: []
   script:
     - docker pull "$CONTAINER_TEST_IMAGE"
     - docker tag "$CONTAINER_TEST_IMAGE" "$CONTAINER_RELEASE_IMAGE"
@@ -73,17 +66,35 @@ release-image:
   only:
     - tags
 
+deploy-dev:
+  stage: deploy
+  image: registry.esss.lu.se/ics-docker/tower-cli
+  before_script: []
+  dependencies: []
+  script:
+    - >
+      tower-cli job launch
+      -h torn.tn.esss.lu.se
+      -t ${TOWER_OAUTH_TOKEN}
+      -J deploy-csentry-dev
+      -e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
+  environment:
+    name: dev
+    url: https://csentry-lab-01.cslab.esss.lu.se
+  except:
+    - master
+    - tags
+
 deploy-staging:
-  <<: *runner_tags
   stage: deploy
-  image: registry.esss.lu.se/ics-docker/tower-cli:3.2
+  image: registry.esss.lu.se/ics-docker/tower-cli
   before_script: []
+  dependencies: []
   script:
     - >
       tower-cli job launch
       -h torn.tn.esss.lu.se
-      -u ${ANSIBLE_AWX_USERNAME}
-      -p ${ANSIBLE_AWX_PASSWORD}
+      -t ${TOWER_OAUTH_TOKEN}
       -J deploy-csentry-staging
       -e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
   environment:
@@ -94,10 +105,11 @@ deploy-staging:
     - tags
 
 pages:
-  <<: *runner_tags
   stage: deploy
   image: "$CONTAINER_RELEASE_IMAGE"
-  before_script: []
+  dependencies: []
+  before_script:
+    - pip install -r requirements-dev.txt
   script:
     - sphinx-build -M html docs docs/_build
     - mv docs/_build/html public
@@ -110,16 +122,15 @@ pages:
     - tags
 
 deploy-production:
-  <<: *runner_tags
   stage: deploy
-  image: registry.esss.lu.se/ics-docker/tower-cli:3.2
+  image: registry.esss.lu.se/ics-docker/tower-cli
   before_script: []
+  dependencies: []
   script:
     - >
       tower-cli job launch
       -h torn.tn.esss.lu.se
-      -u ${ANSIBLE_AWX_USERNAME}
-      -p ${ANSIBLE_AWX_PASSWORD}
+      -t ${TOWER_OAUTH_TOKEN}
       -J deploy-csentry
       -e "csentry_tag=$CI_COMMIT_TAG" --monitor
   environment:

.pre-commit-config.yaml

 repos:
-  - repo: https://github.com/ambv/black
-    rev: stable
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
     hooks:
       - id: black
-        language_version: python3.6
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v1.3.0
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
     hooks:
       - id: flake8

CHANGELOG.rst

 Changelog
 =========
 
+Version 2021.04.06
+------------------
+
+- Validate group names (INFRA-3135)
+- Add warning text for creating vm (INFRA-3292)
+- Paginate history in ansible groups and hosts (INFRA-3136)
+
+Version 2020.11.25
+------------------
+
+- Add documentation about database versioning
+- Set display_name to username when empty (INFRA-2909)
+- Improve logging (INFRA-2899)
+
+Version 2020.11.13
+------------------
+
+- Increased hostname character limit from 20 -> 24 (INFRA-2808)
+- Use SSL by default for LDAP connection (INFRA-2805)
+- Use SonarScanner.gitlab-ci.yml template (INFRA-2832)
+
+Version 2020.10.30
+------------------
+
+- Fix inventory export to excel (INFRA-2780)
+- Add developer documentation
+
+Version 2020.10.27
+------------------
+
+- Update Python and requirements (INFRA-2649)
+- Update RQ and rq-dashboard (INFRA-2649 / INFRA-2754)
+- Fix code smells (INFRA-2729)
+- Add auditor group (INFRA-2728)
+- Update Sphinx and switch to sphinx_rtd_theme (INFRA-2742)
+
+Version 2020.10.06
+------------------
+
+- Make sensitive hosts visible for members of the scope (INFRA-2508)
+- Only hide sensitive networks not part of the user scope (INFRA-2437)
+- Automatically set network_scope group as parent of network groups (INFRA-2639)
+
+Version 2020.07.08
+------------------
+
+- Add view to edit network scope (INFRA-2316)
+- Add API endpoint to patch network scope (INFRA-2316)
+
+Version 2020.06.12
+------------------
+
+- Change MTCA-IOxOS to MTCA-IFC (INFRA-2200)
+- Add description field on Interface (INFRA-2112)
+
+Version 2020.04.16
+------------------
+
+- Allow /network/networks API endpoint for normal users (INFRA-2013)
+- Update Bootstrap to 4.4.1 (INFRA-2021)
+- Update DataTables (INFRA-2021)
+- Add tooltip on search box (INFRA-1893)
+
+Version 2020.03.17
+------------------
+
+- Add extra badges to README and documentation
+- Fix get_hosts for normal users (INFRA-1888)
+
+Version 2020.03.09
+------------------
+
+- Trigger core services update on host change (INFRA-1846)
+- Add DELETE method on all API endpoints (INFRA-1853)
+- Remove create_user API endpoint (INFRA-1854)
+
+Version 2020.03.05
+------------------
+
+- Allow users to view networks (INFRA-1809)
+- Add HOSTNAME Ansible group type (INFRA-1844)
+- Remove IOC git repository creation (INFRA-1839)
+- Make IP optional when creating interface via the API (INFRA-1833)
+- Introduce new device types for MTCA (INFRA-1835)
+
+
+Version 2020.02.25
+------------------
+
+- Fix inventory update (INFRA-1792)
+
+Version 2020.02.24
+------------------
+
+- Add app version in navbar (INFRA-1789)
+- Allow to pass LDAP settings as env variables (INFRA-1788)
+- Implement server side processing for Ansible groups (INFRA-1790)
+
+Version 0.29.0 (2020-02-03)
+---------------------------
+
+- Add MAC address column to hosts table (INFRA-1669)
+- Restrict view of networks and scopes to admin only (INFRA-1671)
+- Add sensitive field to Network class (INFRA-1671)
+- Filter out sensitive hosts for non admin users (INFRA-1671)
+- Add API endpoint and view to update network (INFRA-1724)
+- Make pages title more explicit (INFRA-1726)
+
+Version 0.28.0 (2019-12-19)
+---------------------------
+
+- Fix exception when setting new stack_member (INFRA-1648)
+- Prevent recursive dependency loop in Ansible groups (INFRA-1622)
+- Prevent network and scope overlapping (INFRA-1627)
+- Add netmask to network view (INFRA-1660)
+
+Version 0.27.1 (2019-12-04)
+---------------------------
+
+- Fix regexp for ICS id validation (INFRA-1569)
+- Allow normal users to edit hosts with no interface (INFRA-1604)
+
+Version 0.27.0 (2019-10-30)
+---------------------------
+
+- Increase interface name max length (INFRA-1324)
+- Add endpoints to patch host and interface (INFRA-1406)
+- Limit networks to same scope for extra interfaces (INFRA-1297)
+
+Version 0.26.1 (2019-09-18)
+---------------------------
+
+- Fix inventory update not triggered (INFRA-1290)
+
+Version 0.26.0 (2019-09-18)
+---------------------------
+
+- Decouple inventory and core services update (INFRA-1290)
+- Add broadcast address to network view (INFRA-1291)
+- Add view network scope page (INFRA-1292)
+- Make ansible_vars in host searchable (INFRA-1295)
+- Remove Generate ZTP configuration function (INFRA-1299)
+
+Version 0.25.1 (2019-08-30)
+---------------------------
+
+- Prevent input of invalid Ansible variables (INFRA-1221)
+- Update black and pre-commit config
+- Add pytest.ini to speed test discovery
+
+Version 0.25.0 (2019-08-22)
+---------------------------
+
+- Implement user access on network scope instead of domain (INFRA-1228)
+- Create IOC repositories with same path as name (INFRA-1230)
+
+Version 0.24.1 (2019-07-10)
+---------------------------
+
+- Fix sorting of items with some null stack_member (INFRA-1112)
+
+Version 0.24.0 (2019-07-10)
+---------------------------
+
+- Add serial_number and stack_member to hosts json model (INFRA-1111)
+
+Version 0.23.0 (2019-06-14)
+---------------------------
+
+- Increase RQ default timeout (INFRA-1051)
+
+Version 0.22.0 (2019-06-14)
+---------------------------
+
+- Make vlan optional in NetworkScope and Network (INFRA-1049)
+- Update reverse dependencies of failed tasks (INFRA-1051)
+
+Version 0.21.1 (2019-05-13)
+---------------------------
+
+- Fix users synchronization (INFRA-1026)
+- Remove caching of user retrieval (INFRA-1025)
+
+Version 0.21.0 (2019-05-10)
+---------------------------
+
+- Improve documentation about ansible-vault
+- Fix IOC host indexation (INFRA-1015)
+- Allow users to delete their host (INFRA-1018)
+- Fix job.status DeprecationWarning (INFRA-1021)
+- Allow service users to login (INFRA-1022)
+
+Version 0.20.2 (2019-05-08)
+---------------------------
+
+- Fix IOC repository creation (INFRA-1015)
+- Save ansible var when setting boot profile (INFRA-1016)
+
+Version 0.20.1 (2019-04-25)
+---------------------------
+
+- Save the device_type_id in the session (INFRA-987)
+- Fix exception when no network selected (INFRA-810)
+- Remove graylog support in uwsgi (INFRA-981)
+- Replace raven with sentry-sdk (INFRA-979)
+
+Version 0.20.0 (2019-04-05)
+---------------------------
+
+- Add API endpoint to search hosts (INFRA-931)
+- Initialize IOC repository on GitLab (INFRA-932)
+- Limit the max number of elements returned per page (INFRA-942)
+- Allow to set the boot profile from CSEntry (INFRA-943)
+
+Version 0.19.2 (2019-03-27)
+---------------------------
+
+- Fix model update in admin interface (INFRA-908)
+- Increase timeout for core services update (INFRA-895)
+- Add depends_on field on Task (INFRA-895)
+
+Version 0.19.1 (2019-03-18)
+---------------------------
+
+- Fix excel file download (INFRA-890)
+- Skip post install for windows VM creation (INFRA-877)
+
+Version 0.19.0 (2019-03-18)
+---------------------------
+
+- Add osversion field to CreateVMForm (INFRA-877)
+- Fix link to AWX job id for workflow jobs (INFRA-886)
+- Allow users to create VM/VIOC (INFRA-775)
+- Trigger AWX inventory update on database change (INFRA-887)
+
+Version 0.18.2 (2019-03-07)
+---------------------------
+
+- Fix post install job trigger (INFRA-870)
+
+Version 0.18.1 (2019-03-07)
+---------------------------
+
+- Clean extra vars passed for VM and VIOC creation (INFRA-870)
+
+Version 0.18.0 (2019-03-05)
+---------------------------
+
+- Add gateway field to network table (INFRA-809)
+- Add view network page (INFRA-860)
+- Remove tags and add is_ioc field to host table (INFRA-862)
+- Allow to launch a workflow job (INFRA-867)
+
+Version 0.17.1 (2019-02-07)
+---------------------------
+
+- Catch exception raised by Unique Validator (INFRA-777)
+- Only check the main interface in dynamic groups (INFRA-784)
+- Make sure the first interface is the main one (INFRA-785)
+
+Version 0.17.0 (2019-01-28)
+---------------------------
+
+- Implement new layout (INFRA-763)
+- Fix case insensitive search (INFRA-770)
+- Add post install job after VM creation (INFRA-769)
+
+Version 0.16.0 (2019-01-23)
+---------------------------
+
+- Explicitely define the elasticsearch mapping (INFRA-723)
+- Add favicon (INFRA-725)
+- Pass disk size in inventory when creating VM (INFRA-759)
+
+Version 0.15.1 (2018-12-10)
+---------------------------
+
+- Fix extra interface creation (INFRA-697)
+
+Version 0.15.0 (2018-11-30)
+---------------------------
+
+- Disable DHCP & DNS update on host modification (INFRA-673)
+- Use different groups for inventory and network (INFRA-578)
+- Add network permissions per domain (INFRA-578)
+- Move mac API to inventory endpoint (INFRA-674)
+- Add button to delete Ansible group (INFRA-678)
+- Save VM memory and cores as ansible variables (INFRA-640)
+- Return host fqdn in Ansible groups (INFRA-640)
+- Add extra fields to the API (INFRA-640)
+- Update Python to 3.7.1 and dependencies (INFRA-684)
+- Fix WhiteNoise used static directory (INFRA-683)
+- Fix flask subcommand name (INFRA-687)
+
+Version 0.14.0 (2018-11-08)
+---------------------------
+
+- Fix TypeError in after_commit (INFRA-613)
+- Add zabbix plugin to uwsgi (INFRA-614)
+- Allow to edit and delete comments (INFRA-618)
+- Make mac addresses unique by interface (INFRA-639)
+- Replace bootstrap-select with selectize (INFRA-644)
+- Make host/interface/cname unique (INFRA-245)
+- Switch to new template to deploy VM in proxmox (INFRA-651)
+
+Version 0.13.0 (2018-10-12)
+---------------------------
+
+- Implement full text search for items (INFRA-575)
+- Implement server side processing and full text search for network hosts (INFRA-595)
+- Enable datatables state saving (INFRA-608)
+- Compile uwsgi with graylog2 plugin (INFRA-576)
+- Fix javascript error: Cannot read property '_buttons' of undefined (INFRA-584)
+- Fix LDAPInvalidFilterErrorldap3 (INFRA-550)
+- Fix IndexError when mail is set to an empty list (INFRA-610)
+
+Version 0.12.0 (2018-09-28)
+---------------------------
+
+- Update DataTables with bootstrap 4 theme (INFRA-545)
+- Implement alert flashing from javascript (INFRA-545)
+- Add button to export items to excel file (INFRA-545)
+- Add button to delete hosts (INFRA-557)
+- Allow to delete hosts and interfaces via the API (INFRA-570)
+- Display hosts with no interface on list hosts page (INFRA-569)
+- Ignore old "lost" deferred tasks (INFRA-559)
+
+Version 0.11.4 (2018-09-24)
+---------------------------
+
+- Replace AWX username/password with oauth token for deployment
+- Allow to resize the CodeMirror editor (INFRA-543)
+- Add yaml constructor to support "!vault" tag (INFRA-544)
+- Fix TowerCLIError when ztp_stack is empty (INFRA-547)
+- Fix AttributeError when creating host with group (INFRA-548)
+- Enable sentry user feedback for crash reports
+
+Version 0.11.3 (2018-09-14)
+---------------------------
+
+- Add sentry integration (INFRA-525)
+- Pass sensitive information via environment variables
+
+Version 0.11.2 (2018-09-13)
+---------------------------
+
+- Add versioning on Host and AnsibleGroup (INFRA-500)
+- Disable artifact downloads in release and deploy stages
+
+Version 0.11.1 (2018-08-21)
+---------------------------
+
+- Update tower-cli version for deployment
+- Add host FQDN in host view (INFRA-459)
+- Pass ztp_stack var to trigger_ztp_configuration (INFRA-458)
+- Update Ansible groups documentation
+
+Version 0.11.0 (2018-08-16)
+---------------------------
+
+- Add Ansible parent - child groups
+- Add Ansible dynamic groups
+- Remove flask-bootstrap dependency
+- Update to bootstrap 4.1.3
+- Switch to python-slim image and update requirements
+- Replace jwt decorators with flask-login
+
+Version 0.10.2 (2018-07-23)
+---------------------------
+
+- Allow retrieving interfaces by network name (INFRA-424)
+
+Version 0.10.1 (2018-07-17)
+---------------------------
+
+- Wrap long lines in Ansible groups table
+- Add model field to /networks/hosts endpoint (INFRA-414)
+- Add model field to /networks/interfaces endpoint (INFRA-414)
+- Add trigger ZTP configuration task (INFRA-414)
+
+Version 0.10.0 (2018-07-16)
+---------------------------
+
+- Use sqlalchemy events hook to trigger tasks
+- Add Ansible variables on the host table (INFRA-412)
+- Add Ansible groups table and views (INFRA-412)
+- Add new api endpoint /network/groups (INFRA-412)
+
+
+Version 0.9.0 (2018-07-06)
+--------------------------
+
+- Redirect to the view host page when creating a host (INFRA-402)
+- Save tasks (background jobs) to the database (INFRA-403)
+- Add blueprint to view tasks (INFRA-403)
+- Redirect to the task view page when creating a VM
+- Display host creation date and user on view host page
+- Use black for source code formatting
+- Add pre-commit hooks for code formatting and linting
+- Update documentation
+
 Version 0.8.1
 -------------
 

Dockerfile

-FROM registry.esss.lu.se/ics-docker/miniconda:4.5
+FROM python:3.8-slim as base
 
-USER root
-WORKDIR /app
-RUN chown conda:users /app
+# Add ESS specific debian repository mirror
+RUN echo "deb https://artifactory.esss.lu.se/artifactory/debian-mirror stable main" | tee /etc/apt/sources.list.d/ess-debian-mirror.list
+
+# Install Python dependencies in an intermediate image
+# as some requires a compiler (psycopg2)
+FROM base as builder
+
+# Install dependencies required to compile some Python packages
+# Taken from https://github.com/docker-library/python/blob/master/3.6/stretch/slim/Dockerfile
+# For psycopg2: libpq-dev
+# For pillow: libjpeg-dev libpng-dev libtiff-dev
+RUN apt-get update \
+  && apt-get install -yq --no-install-recommends \
+    gcc \
+    libbz2-dev \
+    libc6-dev \
+    libexpat1-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libgdbm-dev \
+    liblzma-dev \
+    libncursesw5-dev \
+    libpcre3-dev \
+    libpng-dev \
+    libpq-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    libtiff-dev \
+    make \
+    tk-dev \
+    wget \
+    xz-utils \
+    zlib1g-dev \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /requirements.txt
+RUN python -m venv /venv \
+  && . /venv/bin/activate \
+  && pip install --no-cache-dir --upgrade pip \
+  && pip install --no-cache-dir -r /requirements.txt
+
+ARG CSENTRY_BUILD
+COPY requirements-dev.txt /requirements-dev.txt
+RUN if [ "$CSENTRY_BUILD" = "DEV" ] ; then /venv/bin/pip install --no-cache-dir -r /requirements-dev.txt ; fi
 
-USER conda
+FROM base
 
-# Install CSEntry requirements
-COPY environment.yml /app/environment.yml
-RUN conda env create -n csentry -f environment.yml \
-    && rm -rf /opt/conda/pkgs/*
+RUN groupadd -r -g 1000 csi \
+  && useradd --no-log-init -r -g csi -u 1000 csi
+
+COPY --chown=csi:csi --from=builder /venv /venv
+
+# Install libraries for psycopg2 and pillow
+# Shall be the same as the one linked to when compiling in builder image!
+RUN apt-get update \
+  && apt-get install -yq --no-install-recommends \
+    libjpeg62-turbo \
+    libpng16-16 \
+    libtiff5 \
+    libpq5 \
+    libpcre3 \
+    zlib1g \
+    git \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN rm /etc/apt/sources.list.d/ess-debian-mirror.list
+
+COPY --chown=csi:csi . /app/
+WORKDIR /app
+RUN echo "__version__ = \"$(git describe)\"" > app/_version.py
 
-# Install the app
-COPY . /app/
+ENV PATH /venv/bin:$PATH
+EXPOSE 8000
+CMD ["flask", "run", "--host", "0.0.0.0", "--port", "8000"]
 
-# activate the csentry environment
-ENV PATH /opt/conda/envs/csentry/bin:$PATH
+USER csi

Makefile

-.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image
+.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image docs
 
 OWNER := registry.esss.lu.se/ics-infrastructure
 GIT_TAG := $(shell git describe --always)
@@ -38,21 +38,21 @@ release: refresh \
 release: ## build, tag, and push all stacks
 
 db: ## start postgres and redis for development
-	docker-compose up -d postgres redis worker
+	docker-compose up -d postgres redis elasticsearch worker
 
 init_db: ## initialize the dev database
 	docker-compose run --rm web flask db upgrade head
-	docker-compose run --rm web flask create_defaults
+	docker-compose run --rm web flask create-defaults
 
 upgrade_db: ## upgrade the dev database
 	docker-compose run --rm web flask db upgrade head
 
 test:  ## run the tests (on current directory)
-	docker-compose run --rm web pytest --cov=app -v
+	docker-compose -f docker-compose.yml run --rm web pytest --cov=app -v
 
-db_image: ## start postgres and redis to test the latest image
-# Pass docker-compose.yml to skip docker-compose.override.yml
-	docker-compose -f docker-compose.yml up -d postgres redis
+db_test: ## start required containers for test
+# Pass docker-compose.yml to skip docker-compose.override.yml (db not mounted as volume for speed)
+	docker-compose -f docker-compose.yml up -d postgres redis elasticsearch worker
 
 test_image:  ## run the tests (on the latest image)
 # Pass docker-compose.yml to skip docker-compose.override.yml
@@ -60,3 +60,6 @@ test_image:  ## run the tests (on the latest image)
 
 run_uwsgi:  ## run the application with uwsgi (to test prod env)
 	docker-compose run -p 8000:8000 --rm web uwsgi --master --http 0.0.0.0:8000 --http-keepalive --manage-script-name --mount /csentry=wsgi.py --callable app --uid conda --processes 2 -b 16384
+
+docs:  ## run the tests (on current directory)
+	docker run --rm -v $(shell pwd):/app registry.esss.lu.se/ics-infrastructure/csentry:master sphinx-build -M html docs docs/_build

README.rst

@@ -4,6 +4,16 @@ CSEntry
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
     :target: https://github.com/ambv/black
 
+.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=alert_status
+    :target: https://sonarqube.esss.lu.se/dashboard?id=csentry
+
+.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=ncloc
+    :target: https://sonarqube.esss.lu.se/dashboard?id=csentry
+
+.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/pipeline.svg
+
+.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/coverage.svg
+
 Control System Entry web server.
 
 
@@ -80,16 +90,3 @@ To restore the database::
 
     $ gunzip -c csentry_db.sql.gz | docker run --rm --link csentry_postgres:postgres --net csentry_default
       -e PGPASSWORD="<csentry_password>" -i postgres:10 psql -h postgres -U ics csentry_db
-
-
-Dependencies
-------------
-
-The initial dependencies were generated using::
-
-    $ docker run --rm -it -v $(pwd):/app continuumio/miniconda3:latest bash
-    $ conda config --add channels conda-forge
-    $ conda create -n csentry python=3.6 flask alembic flask-debugtoolbar flask-login flask-sqlalchemy flask-wtf pillow psycopg2 pytest pytest-cov qrcode whitenoise factory_boy flask-admin pyjwt ldap3 flask-mail flask-migrate flask-jwt-extended
-    $ source activate csentry
-    $ pip install flask-ldap3-login sqlalchemy-citext sqlalchemy-continuum pytest-factoryboy git+https://github.com/beenje/flask-bootstrap@4.0.0-beta.1.dev1
-    $ conda env export > /app/environment.yml

app/_version.py

+__version__ = "dev"

app/api/inventory.py

@@ -10,10 +10,10 @@ This module implements the inventory API.
 
 """
 from flask import Blueprint, jsonify, request, current_app
-from flask_jwt_extended import jwt_required
+from flask_login import login_required
 from .. import utils, models
-from ..decorators import jwt_groups_accepted
-from .utils import commit, create_generic_model, get_generic_model
+from ..decorators import login_groups_accepted
+from .utils import commit, create_generic_model, get_generic_model, delete_generic_model
 
 bp = Blueprint("inventory_api", __name__)
 
@@ -33,7 +33,7 @@ def get_item_by_id_or_ics_id(id_):
 
 
 @bp.route("/items")
-@jwt_required
+@login_required
 def get_items():
     """Return items
 
@@ -43,9 +43,9 @@ def get_items():
 
 
 @bp.route("/items/<id_>")
-@jwt_required
+@login_required
 def get_item(id_):
-    """Retrieve item by id or ICS id
+    r"""Retrieve item by id or ICS id
 
     .. :quickref: Inventory; Get item by id or ICS id
 
@@ -56,8 +56,7 @@ def get_item(id_):
 
 
 @bp.route("/items", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin", "create")
+@login_groups_accepted("admin", "inventory")
 def create_item():
     """Register a new item
 
@@ -82,10 +81,9 @@ def create_item():
 
 
 @bp.route("/items/<id_>", methods=["PATCH"])
-@jwt_required
-@jwt_groups_accepted("admin", "create")
+@login_groups_accepted("admin", "inventory")
 def patch_item(id_):
-    """Patch an existing item
+    r"""Patch an existing item
 
     .. :quickref: Inventory; Update existing item
 
@@ -142,10 +140,22 @@ def patch_item(id_):
     return jsonify(item.to_dict())
 
 
+@bp.route("/items/<int:item_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_item(item_id):
+    """Delete an item
+
+    .. :quickref: Inventory; Delete an item
+
+    :param item_id: item primary key
+    """
+    return delete_generic_model(models.Item, item_id)
+
+
 @bp.route("/items/<id_>/comments")
-@jwt_required
+@login_required
 def get_item_comments(id_):
-    """Get item comments
+    r"""Get item comments
 
     .. :quickref: Inventory; Get item comments
 
@@ -156,10 +166,9 @@ def get_item_comments(id_):
 
 
 @bp.route("/items/<id_>/comments", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin", "create")
+@login_groups_accepted("admin", "inventory")
 def create_item_comment(id_):
-    """Create a comment on item
+    r"""Create a comment on item
 
     .. :quickref: Inventory; Create comment on item
 
@@ -172,8 +181,20 @@ def create_item_comment(id_):
     )
 
 
+@bp.route("/items/comments/<int:comment_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_comment(comment_id):
+    """Delete an item comment
+
+    .. :quickref: Inventory; Delete an item comment
+
+    :param comment_id: comment primary key
+    """
+    return delete_generic_model(models.ItemComment, comment_id)
+
+
 @bp.route("/actions")
-@jwt_required
+@login_required
 def get_actions():
     """Get actions
 
@@ -183,7 +204,7 @@ def get_actions():
 
 
 @bp.route("/manufacturers")
-@jwt_required
+@login_required
 def get_manufacturers():
     """Get manufacturers
 
@@ -193,8 +214,7 @@ def get_manufacturers():
 
 
 @bp.route("/manufacturers", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin", "create")
+@login_groups_accepted("admin", "inventory")
 def create_manufacturer():
     """Create a new manufacturer
 
@@ -206,8 +226,20 @@ def create_manufacturer():
     return create_generic_model(models.Manufacturer)
 
 
+@bp.route("/manufacturers/<int:manufacturer_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_manufacturer(manufacturer_id):
+    """Delete a manufacturer
+
+    .. :quickref: Inventory; Delete a manufacturer
+
+    :param manufacturer_id: manufacturer primary key
+    """
+    return delete_generic_model(models.Manufacturer, manufacturer_id)
+
+
 @bp.route("/models")
-@jwt_required
+@login_required
 def get_models():
     """Get models
 
@@ -217,8 +249,7 @@ def get_models():
 
 
 @bp.route("/models", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin", "create")
+@login_groups_accepted("admin", "inventory")
 def create_model():
     """Create a new model
 
@@ -230,8 +261,20 @@ def create_model():
     return create_generic_model(models.Model)
 
 
+@bp.route("/models/<int:model_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_model(model_id):
+    """Delete a model
+
+    .. :quickref: Inventory; Delete a model
+
+    :param model_id: model primary key
+    """
+    return delete_generic_model(models.Model, model_id)
+
+
 @bp.route("/locations")
-@jwt_required
+@login_required
 def get_locations():
     """Get locations
 
@@ -241,8 +284,7 @@ def get_locations():
 
 
 @bp.route("/locations", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin", "create")
+@login_groups_accepted("admin", "inventory")
 def create_locations():
     """Create a new location
 
@@ -254,8 +296,20 @@ def create_locations():
     return create_generic_model(models.Location)
 
 
+@bp.route("/locations/<int:location_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_location(location_id):
+    """Delete a location
+
+    .. :quickref: Inventory; Delete a location
+
+    :param location_id: location primary key
+    """
+    return delete_generic_model(models.Location, location_id)
+
+
 @bp.route("/statuses")
-@jwt_required
+@login_required
 def get_status():
     """Get statuses
 
@@ -265,8 +319,7 @@ def get_status():
 
 
 @bp.route("/statuses", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin", "create")
+@login_groups_accepted("admin", "inventory")
 def create_status():
     """Create a new status
 
@@ -276,3 +329,38 @@ def create_status():
     :jsonparam description: (optional) description
     """
     return create_generic_model(models.Status)
+
+
+@bp.route("/statuses/<int:status_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_status(status_id):
+    """Delete a status
+
+    .. :quickref: Inventory; Delete a status
+
+    :param status_id: status primary key
+    """
+    return delete_generic_model(models.Status, status_id)
+
+
+@bp.route("/macs")
+@login_required
+def get_macs():
+    """Return mac addresses
+
+    .. :quickref: Inventory; Get mac addresses
+    """
+    return get_generic_model(models.Mac, order_by=models.Mac.address)
+
+
+@bp.route("/macs", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
+def create_macs():
+    """Create a new mac address
+
+    .. :quickref: Inventory; Create new mac address
+
+    :jsonparam address: MAC address
+    :jsonparam item_id: (optional) linked item primary key
+    """
+    return create_generic_model(models.Mac, mandatory_fields=("address",))

app/api/network.py

@@ -10,27 +10,30 @@ This module implements the network API.
 
 """
 from flask import Blueprint, request
-from flask_jwt_extended import jwt_required
+from flask_login import login_required, current_user
+from wtforms import ValidationError
 from .. import models
-from ..decorators import jwt_groups_accepted
-from .utils import get_generic_model, create_generic_model
+from ..decorators import login_groups_accepted
+from ..utils import CSEntryError, validate_ip
+from . import utils
 
 bp = Blueprint("network_api", __name__)
 
 
 @bp.route("/scopes")
-@jwt_required
+@login_groups_accepted("admin", "auditor")
 def get_scopes():
     """Return network scopes
 
     .. :quickref: Network; Get network scopes
     """
-    return get_generic_model(models.NetworkScope, order_by=models.NetworkScope.name)
+    return utils.get_generic_model(
+        models.NetworkScope, order_by=models.NetworkScope.name
+    )
 
 
 @bp.route("/scopes", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin")
+@login_groups_accepted("admin")
 def create_scope():
     """Create a new network scope
 
@@ -43,25 +46,76 @@ def create_scope():
     :jsonparam domain_id: primary key of the default domain
     :jsonparam description: (optional) description
     """
-    return create_generic_model(
+    return utils.create_generic_model(
         models.NetworkScope,
         mandatory_fields=("name", "first_vlan", "last_vlan", "supernet", "domain_id"),
     )
 
 
+@bp.route("/scopes/<int:scope_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_scope(scope_id):
+    """Delete a network scope
+
+    .. :quickref: Network; Delete a network scope
+
+    :param scope_id: network scope primary key
+    """
+    return utils.delete_generic_model(models.NetworkScope, scope_id)
+
+
+@bp.route("/scopes/<int:scope_id>", methods=["PATCH"])
+@login_groups_accepted("admin")
+def patch_scope(scope_id):
+    r"""Patch an existing network scope
+
+    .. :quickref: Network; Update an existing network scope
+
+    :param scope_id: network scope primary key
+    :jsonparam name: network scope name
+    :jsonparam description: description
+    :jsonparam first_vlan: network scope first vlan
+    :jsonparam last_vlan: network scope last vlan
+    :jsonparam supernet: network scope supernet
+    :jsonparam domain: name of the default domain
+    """
+    allowed_fields = (
+        ("name", str, None),
+        ("description", str, None),
+        ("first_vlan", int, None),
+        ("last_vlan", int, None),
+        ("supernet", str, None),
+        ("domain", models.Domain, "name"),
+    )
+    return utils.update_generic_model(models.NetworkScope, scope_id, allowed_fields)
+
+
 @bp.route("/networks")
-@jwt_required
+@login_groups_accepted("admin", "auditor", "network")
 def get_networks():
     """Return networks
 
     .. :quickref: Network; Get networks
     """
-    return get_generic_model(models.Network, order_by=models.Network.address)
+    query = models.Network.query
+    if not (current_user.is_admin or current_user.is_auditor):
+        sensitive_networks = (
+            query.filter(models.Network.sensitive.is_(True))
+            .join(models.Network.scope)
+            .filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
+        )
+        query = (
+            query.filter(models.Network.sensitive.is_(False))
+            .union(sensitive_networks)
+            .distinct(models.Network.id)
+            .from_self()
+        )
+    query = query.order_by(models.Network.address)
+    return utils.get_generic_model(model=models.Network, base_query=query)
 
 
 @bp.route("/networks", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin")
+@login_groups_accepted("admin")
 def create_network():
     """Create a new network
 
@@ -72,13 +126,16 @@ def create_network():
     :jsonparam address: vlan address
     :jsonparam first_ip: first IP of the allowed range
     :jsonparam last_ip: last IP of the allowed range
+    :jsonparam gateway: gateway IP
     :jsonparam scope: network scope name
     :jsonparam domain_id: (optional) primary key of the domain [default: scope domain]
     :jsonparam admin_only: (optional) boolean to restrict the network to admin users [default: False]
     :type admin_only: bool
+    :jsonparam sensitive: hide the network and all hosts if True (for non admin) [default: False]
+    :type sensitive: bool
     :jsonparam description: (optional) description
     """
-    return create_generic_model(
+    return utils.create_generic_model(
         models.Network,
         mandatory_fields=(
             "vlan_name",
@@ -86,66 +143,271 @@ def create_network():
             "address",
             "first_ip",
             "last_ip",
+            "gateway",
             "scope",
         ),
     )
 
 
+@bp.route("/networks/<int:network_id>", methods=["PATCH"])
+@login_groups_accepted("admin")
+def patch_network(network_id):
+    r"""Patch an existing network
+
+    .. :quickref: Network; Update an existing network
+
+    :param network_id: network primary key
+    :jsonparam vlan_name: vlan name
+    :jsonparam address: vlan address
+    :jsonparam first_ip: first IP of the allowed range
+    :jsonparam last_ip: last IP of the allowed range
+    :jsonparam gateway: gateway IP
+    :jsonparam domain: domain name
+    :jsonparam admin_only: boolean to restrict the network to admin users
+    :type admin_only: bool
+    :jsonparam sensitive: hide the network and all hosts if True (for non admin)
+    :type sensitive: bool
+    :jsonparam description: description
+    """
+    # The method currently doesn't allow to update the network_scope
+    # as this will have an impact on the address
+    allowed_fields = (
+        ("vlan_name", str, None),
+        ("address", str, None),
+        ("first_ip", str, None),
+        ("last_ip", str, None),
+        ("gateway", str, None),
+        ("domain", models.Domain, "name"),
+        ("admin_only", bool, None),
+        ("sensitive", bool, None),
+        ("description", str, None),
+    )
+    return utils.update_generic_model(models.Network, network_id, allowed_fields)
+
+
+@bp.route("/networks/<int:network_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_network(network_id):
+    """Delete a network
+
+    .. :quickref: Network; Delete a network
+
+    :param network_id: network primary key
+    """
+    return utils.delete_generic_model(models.Network, network_id)
+
+
 @bp.route("/interfaces")
-@jwt_required
+@login_required
 def get_interfaces():
     """Return interfaces
 
     .. :quickref: Network; Get interfaces
     """
-    domain = request.args.get("domain", None)
-    if domain is not None:
-        query = models.Interface.query
+    query = models.Interface.query
+    query = query.join(models.Interface.network).order_by(models.Interface.ip)
+    if not (current_user.is_admin or current_user.is_auditor):
+        sensitive_interfaces = (
+            query.filter(models.Network.sensitive.is_(True))
+            .join(models.Network.scope)
+            .filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
+        )
         query = (
-            query.join(models.Interface.network)
-            .join(models.Network.domain)
-            .filter(models.Domain.name == domain)
+            query.filter(models.Network.sensitive.is_(False))
+            .union(sensitive_interfaces)
+            .from_self()
+            .join(models.Interface.network)
+            .order_by(models.Interface.ip)
         )
-        query = query.order_by(models.Interface.ip)
-        return get_generic_model(model=None, query=query)
-    return get_generic_model(models.Interface, order_by=models.Interface.ip)
+    domain = request.args.get("domain", None)
+    if domain is not None:
+        query = query.join(models.Network.domain).filter(models.Domain.name == domain)
+        return utils.get_generic_model(model=None, query=query)
+    network = request.args.get("network", None)
+    if network is not None:
+        query = query.filter(models.Network.vlan_name == network)
+        return utils.get_generic_model(model=None, query=query)
+    return utils.get_generic_model(model=models.Interface, base_query=query)
 
 
 @bp.route("/interfaces", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin", "create")
+@login_groups_accepted("admin", "network")
 def create_interface():
     """Create a new interface
 
     .. :quickref: Network; Create new interface
 
     :jsonparam network: network name
-    :jsonparam ip: interface IP
+    :jsonparam ip: (optional) interface IP - IP will be assigned automatically if not given
     :jsonparam name: interface name
-    :jsonparam mac_id: (optional) linked MAC address primary key
-    :jsonparam host_id: (optional) linked host primary key
+    :jsonparam host: host name
+    :jsonparam mac: (optional) MAC address
     """
     # The validate_interfaces method from the Network class is called when
     # setting interface.network. This is why we don't pass network_id here
     # but network (as vlan_name string)
-    return create_generic_model(
-        models.Interface, mandatory_fields=("network", "ip", "name")
+    # Same for host
+    data = request.get_json()
+    # Check that the user has the permissions to create an interface on this network
+    try:
+        network = models.Network.query.filter_by(
+            vlan_name=data["network"]
+        ).first_or_404()
+    except Exception:
+        # If we can't get a network, an error will be raised in create_generic_model
+        pass
+    else:
+        if not current_user.has_access_to_network(network):
+            raise CSEntryError("User doesn't have the required group", status_code=403)
+    return utils.create_generic_model(
+        models.Interface, mandatory_fields=("network", "name", "host")
     )
 
 
+@bp.route("/interfaces/<int:interface_id>", methods=["PATCH"])
+@login_groups_accepted("admin", "network")
+def patch_interface(interface_id):
+    r"""Patch an existing interface
+
+    .. :quickref: Network; Update an existing interface
+
+    :param interface_id: interface primary key
+    :jsonparam ip: interface IP
+    :jsonparam name: interface name
+    :jsonparam network: network name
+    :jsonparam mac: MAC address
+    """
+    interface = models.Interface.query.get_or_404(interface_id)
+    # User shall have access to both the current and new network (if provided)
+    if not current_user.has_access_to_network(interface.network):
+        raise CSEntryError("User doesn't have the required group", status_code=403)
+    data = utils.get_json_body()
+    if "network" in data:
+        new_network = models.Network.query.filter_by(
+            vlan_name=data["network"]
+        ).first_or_404()
+        if not current_user.has_access_to_network(new_network):
+            raise CSEntryError("User doesn't have the required group", status_code=403)
+    # Ensure that the IP is in the network (current one or new one if passed)
+    # This is required because validate_interfaces is only called when a new network
+    # is assigned. So the check is needed even when new_network == interface.network
+    # (same network given in argument)
+    if "ip" in data:
+        try:
+            if "network" in data:
+                validate_ip(data["ip"], new_network)
+            else:
+                validate_ip(data["ip"], interface.network)
+        except ValidationError as e:
+            raise CSEntryError(str(e), status_code=422)
+    # The method doesn't allow to update the interface host
+    # I don't think it makes much sense (and an interface shall start by the host name)
+    # To add a new cname, use create_cname
+    allowed_fields = (
+        ("ip", str, None),
+        ("name", str, None),
+        ("network", models.Network, "vlan_name"),
+        ("mac", str, None),
+    )
+    return utils.update_generic_model(models.Interface, interface_id, allowed_fields)
+
+
+@bp.route("/interfaces/<int:interface_id>", methods=["DELETE"])
+@login_groups_accepted("admin", "network")
+def delete_interface(interface_id):
+    """Delete an interface
+
+    .. :quickref: Network; Delete an interface
+
+    :param interface_id: interface primary key
+    """
+    return utils.delete_generic_model(models.Interface, interface_id)
+
+
+@bp.route("/groups")
+@login_required
+def get_ansible_groups():
+    """Return ansible groups
+
+    .. :quickref: Network; Get Ansible groups
+    """
+    return utils.get_generic_model(
+        models.AnsibleGroup, order_by=models.AnsibleGroup.name
+    )
+
+
+@bp.route("/groups", methods=["POST"])
+@login_groups_accepted("admin")
+def create_ansible_groups():
+    """Create a new Ansible group
+
+    .. :quickref: Network; Create new Ansible group
+
+    :jsonparam name: group name
+    :jsonparam vars: (optional) Ansible variables
+    """
+    return utils.create_generic_model(models.AnsibleGroup, mandatory_fields=("name",))
+
+
+@bp.route("/groups/<int:group_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_ansible_group(group_id):
+    """Delete an Ansible group
+
+    .. :quickref: Network; Delete an Ansible group
+
+    :param group_id: Ansible group primary key
+    """
+    return utils.delete_generic_model(models.AnsibleGroup, group_id)
+
+
 @bp.route("/hosts")
-@jwt_required
+@login_required
 def get_hosts():
     """Return hosts
 
     .. :quickref: Network; Get hosts
     """
-    return get_generic_model(models.Host, order_by=models.Host.name)
+    query = models.Host.query
+    if not (current_user.is_admin or current_user.is_auditor):
+        # Note that hosts without interface will be filtered out
+        # by this query. This is not an issue as hosts should always
+        # have an interface. They are useless otherwise.
+        non_sensitive_hosts = (
+            query.join(models.Host.interfaces)
+            .join(models.Interface.network)
+            .filter(models.Network.sensitive.is_(False))
+        )
+        sensitive_hosts = (
+            query.join(models.Host.interfaces)
+            .join(models.Interface.network)
+            .filter(models.Network.sensitive.is_(True))
+            .join(models.Network.scope)
+            .filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
+        )
+        query = (
+            non_sensitive_hosts.union(sensitive_hosts)
+            .distinct(models.Host.id)
+            .from_self()
+        )
+    query = query.order_by(models.Host.name)
+    return utils.get_generic_model(model=models.Host, base_query=query)
+
+
+@bp.route("/hosts/search")
+@login_required
+def search_hosts():
+    """Search hosts
+
+    .. :quickref: Network; Search hosts
+
+    :query q: the search query
+    """
+    return utils.search_generic_model(models.Host, filter_sensitive=True)
 
 
 @bp.route("/hosts", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin", "create")
+@login_groups_accepted("admin", "network")
 def create_host():
     """Create a new host
 
@@ -153,49 +415,73 @@ def create_host():
 
     :jsonparam name: hostname
     :jsonparam device_type: Physical|Virtual|...
+    :jsonparam is_ioc: True|False (optional)
     :jsonparam description: (optional) description
     :jsonparam items: (optional) list of items ICS id linked to the host
+    :jsonparam ansible_vars: (optional) Ansible variables
+    :jsonparam ansible_groups: (optional) list of Ansible groups names
     """
-    return create_generic_model(models.Host, mandatory_fields=("name", "device_type"))
+    return utils.create_generic_model(
+        models.Host, mandatory_fields=("name", "device_type")
+    )
+
 
+@bp.route("/hosts/<int:host_id>", methods=["PATCH"])
+@login_groups_accepted("admin", "network")
+def patch_host(host_id):
+    r"""Patch an existing host
 
-@bp.route("/macs")
-@jwt_required
-def get_macs():
-    """Return mac addresses
+    .. :quickref: Network; Update an existing host
 
-    .. :quickref: Network; Get mac addresses
+    :param host_id: host primary key
+    :jsonparam device_type: Physical|Virtual|...
+    :jsonparam is_ioc: True|False
+    :jsonparam description: description
+    :jsonparam items: list of items ICS id linked to the host
+    :jsonparam ansible_vars: Ansible variables
+    :jsonparam ansible_groups: list of Ansible groups names
     """
-    return get_generic_model(models.Mac, order_by=models.Mac.address)
+    host = models.Host.query.get_or_404(host_id)
+    if not current_user.has_access_to_network(host.main_network):
+        raise CSEntryError("User doesn't have the required group", status_code=403)
+    # The method currently doesn't allow to update the host name
+    # If we do, we have to update all the linked interface name as well!
+    # Interfaces shall always start by the host name
+    allowed_fields = (
+        ("device_type", models.DeviceType, "name"),
+        ("is_ioc", bool, None),
+        ("description", str, None),
+        ("items", [models.Item], "ics_id"),
+        ("ansible_vars", dict, None),
+        ("ansible_groups", [models.AnsibleGroup], "name"),
+    )
+    return utils.update_generic_model(models.Host, host_id, allowed_fields)
 
 
-@bp.route("/macs", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin", "create")
-def create_macs():
-    """Create a new mac address
+@bp.route("/hosts/<int:host_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_host(host_id):
+    """Delete a host
 
-    .. :quickref: Network; Create new mac address
+    .. :quickref: Network; Delete a host
 
-    :jsonparam address: MAC address
-    :jsonparam item_id: (optional) linked item primary key
+    :param host_id: host primary key
     """
-    return create_generic_model(models.Mac, mandatory_fields=("address",))
+    return utils.delete_generic_model(models.Host, host_id)
 
 
 @bp.route("/domains")
-@jwt_required
+@login_required
 def get_domains():
     """Return domains
 
     .. :quickref: Network; Get domains
     """
-    return get_generic_model(models.Domain, order_by=models.Domain.name)
+    return utils.get_generic_model(models.Domain, order_by=models.Domain.name)
 
 
 @bp.route("/domains", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin")
+@login_groups_accepted("admin")
 def create_domain():
     """Create a new domain
 
@@ -203,11 +489,23 @@ def create_domain():
 
     :jsonparam name: domain name
     """
-    return create_generic_model(models.Domain, mandatory_fields=("name",))
+    return utils.create_generic_model(models.Domain, mandatory_fields=("name",))
+
+
+@bp.route("/domains/<int:domain_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_domain(domain_id):
+    """Delete a domain
+
+    .. :quickref: Network; Delete a domain
+
+    :param domain_id: domain primary key
+    """
+    return utils.delete_generic_model(models.Domain, domain_id)
 
 
 @bp.route("/cnames")
-@jwt_required
+@login_required
 def get_cnames():
     """Return cnames
 
@@ -223,13 +521,12 @@ def get_cnames():
             .filter(models.Domain.name == domain)
         )
         query = query.order_by(models.Cname.name)
-        return get_generic_model(model=None, query=query)
-    return get_generic_model(models.Cname, order_by=models.Cname.name)
+        return utils.get_generic_model(model=None, query=query)
+    return utils.get_generic_model(models.Cname, order_by=models.Cname.name)
 
 
 @bp.route("/cnames", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin")
+@login_groups_accepted("admin")
 def create_cname():
     """Create a new cname
 
@@ -238,4 +535,18 @@ def create_cname():
     :jsonparam name: full cname
     :jsonparam interface_id: primary key of the associated interface
     """
-    return create_generic_model(models.Cname, mandatory_fields=("name", "interface_id"))
+    return utils.create_generic_model(
+        models.Cname, mandatory_fields=("name", "interface_id")
+    )
+
+
+@bp.route("/cnames/<int:cname_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_cname(cname_id):
+    """Delete a cname
+
+    .. :quickref: Network; Delete a cname
+
+    :param cname_id: cname primary key
+    """
+    return utils.delete_generic_model(models.Cname, cname_id)

app/api/user.py

@@ -11,17 +11,17 @@ This module implements the user API.
 """
 from flask import current_app, Blueprint, jsonify, request
 from flask_ldap3_login import AuthenticationResponseStatus
-from flask_jwt_extended import jwt_required, get_current_user
+from flask_login import login_required, current_user
 from ..extensions import ldap_manager
-from ..decorators import jwt_groups_accepted
+from ..decorators import login_groups_accepted
 from .. import utils, tokens, models
-from .utils import get_generic_model, create_generic_model
+from .utils import get_generic_model
 
 bp = Blueprint("user_api", __name__)
 
 
 @bp.route("/users")
-@jwt_required
+@login_groups_accepted("admin", "auditor")
 def get_users():
     """Return users information
 
@@ -31,31 +31,13 @@ def get_users():
 
 
 @bp.route("/profile")
-@jwt_required
+@login_required
 def get_user_profile():
     """Return the current user profile
 
     .. :quickref: User; Get current user profile
     """
-    user = get_current_user()
-    return jsonify(user.to_dict()), 200
-
-
-@bp.route("/users", methods=["POST"])
-@jwt_required
-@jwt_groups_accepted("admin")
-def create_user():
-    """Create a new user
-
-    .. :quickref: User; Create new user
-
-    :jsonparam username: User's username
-    :jsonparam display_name: User's display name
-    :jsonparam email: User's email
-    """
-    return create_generic_model(
-        models.User, mandatory_fields=("username", "display_name", "email")
-    )
+    return jsonify(current_user.to_dict()), 200
 
 
 @bp.route("/login", methods=["POST"])

app/api/utils.py

@@ -12,6 +12,8 @@ This module implements useful functions for the API.
 import urllib.parse
 import sqlalchemy as sa
 from flask import current_app, jsonify, request
+from flask_login import current_user
+from flask_sqlalchemy import Pagination
 from ..extensions import db
 from .. import utils
 
@@ -19,7 +21,7 @@ from .. import utils
 def commit():
     try:
         db.session.commit()
-    except (sa.exc.IntegrityError, sa.exc.DataError) as e:
+    except sa.exc.SQLAlchemyError as e:
         db.session.rollback()
         raise utils.CSEntryError(str(e), status_code=422)
 
@@ -59,33 +61,84 @@ def build_pagination_header(pagination, base_url, **kwargs):
     return header
 
 
-def get_generic_model(model, order_by=None, query=None):
+def get_generic_model(model, order_by=None, base_query=None, query=None):
     """Return data from model as json
 
     :param model: model class
-    :param order_by: column to order the result by
+    :param order_by: column to order the result by (not used if base_query or query is passed)
+    :param query: optional base_query to use [default: model.query]
     :param query: optional query to use (for more complex queries)
     :returns: data from model as json
     """
     kwargs = request.args.to_dict()
     page = int(kwargs.pop("page", 1))
     per_page = int(kwargs.pop("per_page", 20))
+    # Remove recursive from kwargs so that it doesn't get passed
+    # to query.filter_by in get_query
+    recursive = kwargs.pop("recursive", "false").lower() == "true"
     if query is None:
-        query = utils.get_query(model.query, **kwargs)
-        if order_by is None:
-            order_by = getattr(model, "name")
-        query = query.order_by(order_by)
-    pagination = query.paginate(page, per_page)
-    data = [item.to_dict() for item in pagination.items]
+        if base_query is None:
+            base_query = model.query
+            if order_by is None:
+                order_by = getattr(model, "name")
+            base_query = base_query.order_by(order_by)
+        query = utils.get_query(base_query, model, **kwargs)
+    pagination = query.paginate(
+        page, per_page, max_per_page=current_app.config["MAX_PER_PAGE"]
+    )
+    data = [item.to_dict(recursive=recursive) for item in pagination.items]
+    # Re-add recursive to kwargs so that it's part of the pagination url
+    kwargs["recursive"] = recursive
     header = build_pagination_header(pagination, request.base_url, **kwargs)
     return jsonify(data), 200, header
 
 
-def create_generic_model(model, mandatory_fields=("name",), **kwargs):
+def search_generic_model(model, filter_sensitive=False):
+    """Return filtered data from model as json
+
+    :param model: model class
+    :param bool filter_sensitive: filter out sensitive data if set to True
+    :returns: filtered data from model as json
+    """
+    kwargs = request.args.to_dict()
+    page = int(kwargs.pop("page", 1))
+    per_page = int(kwargs.pop("per_page", 20))
+    per_page = min(per_page, current_app.config["MAX_PER_PAGE"])
+    search = kwargs.get("q", "*")
+    instances, nb_filtered = model.search(
+        search, page=page, per_page=per_page, filter_sensitive=filter_sensitive
+    )
+    current_app.logger.debug(
+        f'Found {nb_filtered} {model.__tablename__}(s) when searching "{search}"'
+    )
+    data = [instance.to_dict(recursive=True) for instance in instances]
+    pagination = Pagination(None, page, per_page, nb_filtered, None)
+    header = build_pagination_header(pagination, request.base_url, **kwargs)
+    return jsonify(data), 200, header
+
+
+def get_json_body():
+    """Return the json body from the current request
+
+    Raise a CSEntryError if the body is not a json object
+    """
     data = request.get_json()
     if data is None:
         raise utils.CSEntryError("Body should be a JSON object")
-    current_app.logger.debug(f"Received: {data}")
+    if not data:
+        raise utils.CSEntryError("At least one field is required", status_code=422)
+    return data
+
+
+def create_generic_model(model, mandatory_fields=("name",), **kwargs):
+    """Create an instance of the model
+
+    :param model: model class
+    :param mandatory_fields: list of fields that shall be passed in the body
+    :param kwargs: extra fields to use
+    :returns: representation of the created instance as json
+    """
+    data = get_json_body()
     data.update(kwargs)
     for mandatory_field in mandatory_fields:
         if mandatory_field not in data:
@@ -101,4 +154,54 @@ def create_generic_model(model, mandatory_fields=("name",), **kwargs):
         raise utils.CSEntryError(str(e), status_code=422)
     db.session.add(instance)
     commit()
+    current_app.logger.info(
+        f"New {model.__tablename__} created by {current_user}: {instance.to_dict()}"
+    )
     return jsonify(instance.to_dict()), 201
+
+
+def delete_generic_model(model, primary_key):
+    """Delete the model based on the primary_key
+
+    :param model: model class
+    :param primary_key: primary key of the instance to delete
+    """
+    instance = model.query.get_or_404(primary_key)
+    db.session.delete(instance)
+    commit()
+    current_app.logger.info(
+        f"{model.__tablename__} {instance} deleted by {current_user}"
+    )
+    return jsonify(), 204
+
+
+def update_generic_model(model, primary_key, allowed_fields):
+    """Update the model based on the primary_key
+
+    :param model: model class
+    :param primary_key: primary key of the instance to update
+    :param allowed_fields: list of fields that can be updated
+    :returns: representation of the updated model as json
+    """
+    data = get_json_body()
+    # Use a list and not a set because the order is important
+    allowed_keys = [field[0] for field in allowed_fields]
+    for key in data:
+        if key not in allowed_keys:
+            raise utils.CSEntryError(f"Invalid field '{key}'", status_code=422)
+    instance = model.query.get_or_404(primary_key)
+    data = utils.convert_to_models(data, allowed_fields)
+    try:
+        # Loop on allowed_keys and not data to respect
+        # the order in which the fields are set
+        # setting ip before network is important for Interface
+        for key in allowed_keys:
+            if key in data:
+                setattr(instance, key, data[key])
+    except Exception as e:
+        raise utils.CSEntryError(str(e), status_code=422)
+    commit()
+    current_app.logger.info(
+        f"{model.__tablename__} {primary_key} updated by {current_user}: {instance.to_dict()}"
+    )
+    return jsonify(instance.to_dict()), 200

app/commands.py

@@ -9,16 +9,27 @@ This module defines extra flask commands.
 :license: BSD 2-Clause, see LICENSE for more details.
 
 """
+import click
 import ldap3
 import redis
 import rq
 import sqlalchemy as sa
+import sentry_sdk
 from flask import current_app
+from sentry_sdk.integrations.rq import RqIntegration
 from .extensions import db, ldap_manager
 from .defaults import defaults
-from .models import User
 from .tasks import TaskWorker
-from . import utils, tokens
+from . import models, utils, tokens
+
+
+def disable_user(user):
+    """Clear users'groups, email and tokens"""
+    user.groups = []
+    user.email = ""
+    # Revoke all user's tokens
+    for token in user.tokens:
+        db.session.delete(token)
 
 
 def sync_user(connection, user):
@@ -32,25 +43,35 @@ def sync_user(connection, user):
         search_scope=getattr(ldap3, current_app.config.get("LDAP_USER_SEARCH_SCOPE")),
         attributes=current_app.config.get("LDAP_GET_USER_ATTRIBUTES"),
     )
-    if len(connection.response) == 1:
-        ldap_user = connection.response[0]
-        attributes = ldap_user["attributes"]
-        user.display_name = utils.attribute_to_string(attributes["cn"])
-        user.email = utils.attribute_to_string(attributes["mail"])
-        groups = ldap_manager.get_user_groups(
-            dn=ldap_user["dn"], _connection=connection
-        )
-        user.groups = sorted(
-            [utils.attribute_to_string(group["cn"]) for group in groups]
-        )
-        current_app.logger.info(f"{user} updated")
+    results = [
+        result for result in connection.response if result["type"] == "searchResEntry"
+    ]
+    if len(results) == 1:
+        ldap_user = results[0]
+        # OU=InActiveUsers is specific to ESS AD
+        if "OU=InActiveUsers" in ldap_user["dn"]:
+            current_app.logger.info(f"{user} is inactive. User disabled.")
+            disable_user(user)
+        else:
+            attributes = ldap_user["attributes"]
+            user.display_name = (
+                utils.attribute_to_string(attributes["cn"]) or user.username
+            )
+            user.email = utils.attribute_to_string(attributes["mail"])
+            groups = ldap_manager.get_user_groups(
+                dn=ldap3.utils.conv.escape_filter_chars(ldap_user["dn"]),
+                _connection=connection,
+            )
+            user.groups = sorted(
+                [utils.attribute_to_string(group["cn"]) for group in groups]
+            )
+            current_app.logger.info(f"{user} updated")
+    elif len(results) == 0:
+        current_app.logger.warning(f"{user} not found! User disabled.")
+        disable_user(user)
     else:
-        # Clear user's groups
-        user.groups = []
-        # Revoke all user's tokens
-        for token in user.tokens:
-            db.session.delete(token)
-        current_app.logger.info(f"{user} disabled")
+        current_app.logger.warning(f"Too many results for {user}!")
+        current_app.logger.warning(f"results: {results}")
     return user
 
 
@@ -64,11 +85,26 @@ def sync_users():
     except ldap3.core.exceptions.LDAPException as e:
         current_app.logger.warning(f"Failed to connect to the LDAP server: {e}")
         return
-    for user in User.query.all():
+    for user in models.User.query.all():
         sync_user(connection, user)
     db.session.commit()
 
 
+def clean_deferred_tasks():
+    """Set all deferred tasks to failed"""
+    for task in (
+        models.Task.query.filter_by(status=models.JobStatus.DEFERRED)
+        .order_by(models.Task.created_at)
+        .all()
+    ):
+        if task.depends_on is None or task.depends_on.status == models.JobStatus.FAILED:
+            current_app.logger.info(
+                f"Set deferred task {task.id} ({task.name}) to failed"
+            )
+            task.status = models.JobStatus.FAILED
+    db.session.commit()
+
+
 def register_cli(app):
     @app.cli.command()
     def create_defaults():
@@ -77,10 +113,15 @@ def register_cli(app):
             db.session.add(instance)
             try:
                 db.session.commit()
-            except sa.exc.IntegrityError as e:
+            except sa.exc.IntegrityError:
                 db.session.rollback()
                 app.logger.debug(f"{instance} already exists")
 
+    @app.cli.command()
+    def clean_deferred():
+        """Set deferred tasks to failed if the task it depends on failed"""
+        clean_deferred_tasks()
+
     @app.cli.command()
     def syncusers():
         """Synchronize all users from the database with information the LDAP server"""
@@ -100,8 +141,37 @@ def register_cli(app):
     @app.cli.command()
     def runworker():
         """Run RQ worker"""
-        redis_url = current_app.config["REDIS_URL"]
+        redis_url = current_app.config["RQ_REDIS_URL"]
         redis_connection = redis.from_url(redis_url)
         with rq.Connection(redis_connection):
-            worker = TaskWorker(current_app.config["QUEUES"])
+            worker = TaskWorker(["high", "normal", "low"])
+            if current_app.config["SENTRY_DSN"]:
+                sentry_sdk.init(
+                    current_app.config["SENTRY_DSN"],
+                    environment=current_app.config["CSENTRY_ENVIRONMENT"],
+                    integrations=[RqIntegration()],
+                )
             worker.work()
+
+    @app.cli.command()
+    @click.option(
+        "--delete/--no-delete",
+        default=True,
+        help="Delete and recreate the index [default: True]",
+    )
+    @click.option(
+        "--name", default="all", help="name of the class to reindex [default: all]"
+    )
+    def reindex(delete, name):
+        """Initialize the elasticsearch index"""
+        if delete:
+            current_app.elasticsearch.indices.delete("*", ignore=404)
+        if name == "all":
+            models.Item.reindex(delete)
+            models.Host.reindex(delete)
+            models.AnsibleGroup.reindex(delete)
+            return
+        try:
+            getattr(models, name).reindex(delete)
+        except AttributeError:
+            click.echo(f"The class {name} can't be indexed")

app/decorators.py

@@ -10,60 +10,26 @@ This module defines some useful decorators.
 
 """
 from functools import wraps
-from flask import current_app, abort
+from flask import current_app, abort, g
 from flask_login import current_user
-from flask_jwt_extended import get_current_user
 from .utils import CSEntryError
 
 
-def jwt_groups_accepted(*groups):
-    """Decorator which specifies that a user must have at least one of the specified groups.
-
-    This shall be used for users logged in using a JWT (API).
-
-    Example::
-        @bp.route('/models', methods=['POST'])
-        @jwt_required
-        @jwt_groups_accepted('admin', 'create')
-        def create_model():
-            return create()
-
-    The current user must be in either 'admin' or 'create' group
-    to access this route.
-
-    :param groups: accepted groups
-    """
-
-    def wrapper(fn):
-        @wraps(fn)
-        def decorated_view(*args, **kwargs):
-            user = get_current_user()
-            if user is None:
-                raise CSEntryError("Invalid indentity", status_code=403)
-            if not user.is_member_of_one_group(groups):
-                raise CSEntryError(
-                    "User doesn't have the required group", status_code=403
-                )
-            return fn(*args, **kwargs)
-
-        return decorated_view
-
-    return wrapper
-
-
 def login_groups_accepted(*groups):
     """Decorator which specifies that a user must have at least one of the specified groups.
 
-    This shall be used for users logged in using a cookie (web UI).
+    This can be used for users logged in using a cookie (web UI) or JWT (API).
 
     Example::
+
         @bp.route('/models', methods=['POST'])
-        @login_groups_accepted('admin', 'create')
+        @login_groups_accepted('admin', 'inventory')
         def create_model():
             return create()
 
-    The current user must be in either 'admin' or 'create' group
-    to access this route.
+        The current user must be in either 'admin' or 'create' group
+        to access the /models route.
+
     This checks that the user is logged in. There is no need to
     use the @login_required decorator.
 
@@ -73,10 +39,19 @@ def login_groups_accepted(*groups):
     def wrapper(fn):
         @wraps(fn)
         def decorated_view(*args, **kwargs):
+            # LOGIN_DISABLED can be set to True to turn off authentication check when testing.
+            # This variable is also used by flask-login
+            if current_app.config.get("LOGIN_DISABLED"):
+                return fn(*args, **kwargs)
             if not current_user.is_authenticated:
                 return current_app.login_manager.unauthorized()
             if not current_user.is_member_of_one_group(groups):
-                abort(403)
+                if g.get("login_via_request"):
+                    raise CSEntryError(
+                        "User doesn't have the required group", status_code=403
+                    )
+                else:
+                    abort(403)
             return fn(*args, **kwargs)
 
         return decorated_view

app/defaults.py

@@ -23,8 +23,10 @@ defaults = [
     models.DeviceType(name="PhysicalMachine"),
     models.DeviceType(name="VirtualMachine"),
     models.DeviceType(name="Network"),
-    models.DeviceType(name="MicroTCA"),
+    models.DeviceType(name="MTCA-AMC"),
+    models.DeviceType(name="MTCA-IFC"),
+    models.DeviceType(name="MTCA-MCH"),
+    models.DeviceType(name="MTCA-RTM"),
     models.DeviceType(name="VME"),
     models.DeviceType(name="PLC"),
-    models.Tag(name="IOC", admin_only=False),
 ]

app/extensions.py

@@ -14,7 +14,6 @@ from flask_login import LoginManager
 from flask_ldap3_login import LDAP3LoginManager
 from flask_sqlalchemy import SQLAlchemy
 from flask_migrate import Migrate
-from flask_bootstrap import Bootstrap
 from flask_admin import Admin
 from flask_mail import Mail
 from flask_jwt_extended import JWTManager
@@ -37,7 +36,6 @@ db = SQLAlchemy(metadata=metadata, session_options={"autoflush": False})
 migrate = Migrate(db=db)
 login_manager = LoginManager()
 ldap_manager = LDAP3LoginManager()
-bootstrap = Bootstrap()
 admin = Admin(template_mode="bootstrap3")
 mail = Mail()
 jwt = JWTManager()

app/factory.py

@@ -9,17 +9,20 @@ Create the WSGI application.
 :license: BSD 2-Clause, see LICENSE for more details.
 
 """
+import logging
 import sqlalchemy as sa
 import rq_dashboard
+import sentry_sdk
 from flask import Flask
 from whitenoise import WhiteNoise
+from elasticsearch import Elasticsearch
+from sentry_sdk.integrations.flask import FlaskIntegration
 from . import settings, models
 from .extensions import (
     db,
     migrate,
     login_manager,
     ldap_manager,
-    bootstrap,
     admin,
     mail,
     jwt,
@@ -45,6 +48,7 @@ from .api.user import bp as user_api
 from .api.inventory import bp as inventory_api
 from .api.network import bp as network_api
 from .commands import register_cli
+from ._version import __version__
 from . import utils
 
 
@@ -57,70 +61,43 @@ def create_app(config=None):
     app.config.update(config or {})
 
     app.jinja_env.filters["datetimeformat"] = utils.format_datetime
-
-    if not app.debug:
-        import logging
-
-        # Send ERROR via mail
-        from logging.handlers import SMTPHandler
-
-        mail_handler = SMTPHandler(
-            app.config["MAIL_SERVER"],
-            fromaddr=app.config["EMAIL_SENDER"],
-            toaddrs=app.config["ADMIN_EMAILS"],
-            subject="CSEntry: ERROR raised",
-            credentials=app.config["MAIL_CREDENTIALS"],
-        )
-        mail_handler.setFormatter(
-            logging.Formatter(
-                """
-            Message type:       %(levelname)s
-            Location:           %(pathname)s:%(lineno)d
-            Module:             %(module)s
-            Function:           %(funcName)s
-            Time:               %(asctime)s
-
-            Message:
-
-            %(message)s
-        """
-            )
-        )
-        mail_handler.setLevel(logging.ERROR)
-        app.logger.addHandler(mail_handler)
-        # Log to stderr
-        handler = logging.StreamHandler()
-        handler.setFormatter(
-            logging.Formatter(
-                "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
-            )
+    app.jinja_env.filters["toyaml"] = utils.pretty_yaml
+    app.jinja_env.globals["__version__"] = __version__
+
+    if app.config["SENTRY_DSN"]:
+        sentry_sdk.init(
+            dsn=app.config["SENTRY_DSN"],
+            environment=app.config["CSENTRY_ENVIRONMENT"],
+            send_default_pii=True,
+            integrations=[FlaskIntegration()],
         )
-        # Set app logger level to DEBUG
-        # otherwise only WARNING and above are propagated
-        app.logger.setLevel(logging.DEBUG)
-        handler.setLevel(logging.DEBUG)
-        app.logger.addHandler(handler)
+
+    app.logger.setLevel(logging.DEBUG)
     app.logger.info("CSEntry created!")
     # Remove variables that contain a password
-    settings_to_display = [
-        f"{key}: {value}"
+    settings_to_display = {
+        key: value
         for key, value in app.config.items()
         if key
         not in (
+            "SENTRY_DSN",
             "SECRET_KEY",
             "LDAP_BIND_USER_PASSWORD",
-            "MAIL_CREDENTIALS",
             "SQLALCHEMY_DATABASE_URI",
         )
-    ]
+    }
     # The repr() of make_url hides the password
-    settings_to_display.append(
-        f'SQLALCHEMY_DATABASE_URI: {sa.engine.url.make_url(app.config["SQLALCHEMY_DATABASE_URI"])!r}'
+    settings_to_display["SQLALCHEMY_DATABASE_URI"] = repr(
+        sa.engine.url.make_url(app.config["SQLALCHEMY_DATABASE_URI"])
+    )
+    settings_string = "\n".join(
+        [f"{key}: {settings_to_display[key]}" for key in sorted(settings_to_display)]
     )
-    settings_string = "\n".join(settings_to_display)
     app.logger.info(f"Settings:\n{settings_string}")
 
-    bootstrap.init_app(app)
+    # Force RQ_DASHBOARD_REDIS_URL to RQ_REDIS_URL
+    app.config["RQ_DASHBOARD_REDIS_URL"] = app.config["RQ_REDIS_URL"]
+
     db.init_app(app)
     migrate.init_app(app)
     login_manager.init_app(app)
@@ -133,6 +110,11 @@ def create_app(config=None):
     app.config["SESSION_REDIS"] = session_redis_store
     fsession.init_app(app)
     cache.init_app(app)
+    app.elasticsearch = (
+        Elasticsearch([app.config["ELASTICSEARCH_URL"]])
+        if app.config["ELASTICSEARCH_URL"]
+        else None
+    )
 
     admin.init_app(app)
     admin.add_view(UserAdmin(models.User, db.session, endpoint="users"))
@@ -148,11 +130,11 @@ def create_app(config=None):
     admin.add_view(AdminModelView(models.NetworkScope, db.session))
     admin.add_view(NetworkAdmin(models.Network, db.session, endpoint="networks"))
     admin.add_view(AdminModelView(models.DeviceType, db.session))
+    admin.add_view(AdminModelView(models.AnsibleGroup, db.session))
     admin.add_view(AdminModelView(models.Host, db.session))
     admin.add_view(AdminModelView(models.Interface, db.session))
     admin.add_view(AdminModelView(models.Mac, db.session))
     admin.add_view(AdminModelView(models.Cname, db.session))
-    admin.add_view(AdminModelView(models.Tag, db.session))
     admin.add_view(TaskAdmin(models.Task, db.session, endpoint="tasks"))
 
     app.register_blueprint(main)
@@ -165,11 +147,7 @@ def create_app(config=None):
     app.register_blueprint(network_api, url_prefix="/api/v1/network")
     app.register_blueprint(rq_dashboard.blueprint, url_prefix="/rq")
 
-    app.wsgi_app = WhiteNoise(app.wsgi_app, root="static/")
-    app.wsgi_app.add_files(
-        root="/opt/conda/envs/csentry/lib/python3.6/site-packages/flask_bootstrap/static/",
-        prefix="bootstrap/",
-    )
+    app.wsgi_app = WhiteNoise(app.wsgi_app, root=app.config["CSENTRY_STATIC_DIR"])
 
     register_cli(app)
 

app/fields.py

+# -*- coding: utf-8 -*-
+"""
+app.fields
+~~~~~~~~~~
+
+This module defines extra WTForms fields
+
+:copyright: (c) 2018 European Spallation Source ERIC
+:license: BSD 2-Clause, see LICENSE for more details.
+
+"""
+import yaml
+from wtforms import TextAreaField
+
+
+# String can be encoded with ansible-vault and stored in yaml files
+# using the "!vault" tag.
+# To be used in a dynamic inventory, it shall be converted to the mapping
+# {"__ansible_vault": value} as it needs to be returned as JSON.
+def vault_constructor(loader, node):
+    value = loader.construct_scalar(node)
+    return {"__ansible_vault": value}
+
+
+yaml.SafeLoader.add_constructor("!vault", vault_constructor)
+
+
+class YAMLField(TextAreaField):
+    """This field represents an HTML ``<textarea>`` used to input YAML"""
+
+    def _value(self):
+        return yaml.safe_dump(self.data, default_flow_style=False) if self.data else ""
+
+    def process_formdata(self, valuelist):
+        if valuelist and valuelist[0].strip() != "":
+            try:
+                self.data = yaml.safe_load(valuelist[0])
+            except yaml.YAMLError:
+                raise ValueError("This field contains invalid YAML")
+            if not isinstance(self.data, dict):
+                raise ValueError("This field shall only contain key-value-pairs")
+        else:
+            self.data = None
+
+    def pre_validate(self, form):
+        super().pre_validate(form)
+        if self.data:
+            try:
+                yaml.safe_dump(self.data)
+            except yaml.YAMLError:
+                raise ValueError("This field contains invalid YAML")