.dockerignore

-.git
 **/*.swp
 **/__pycache__
 settings*.cfg
@@ -8,3 +7,7 @@ Jenkinsfile
 Makefile
 docker-compose*.yml
 .gitlab-ci.yml
+.cache
+.coverage
+.pytest_cache
+docs/_build

.env

 POSTGRES_USER=ics
 POSTGRES_PASSWORD=icspwd
 POSTGRES_DB=csentry_db
-PGDATA_VOLUME=./data
+PGDATA_VOLUME=./data/postgres
+ELASTIC_DATA_VOLUME=./data/elastic

.flake8

+[flake8]
+# E501: let black handle line length
+# W503 is incompatible with PEP 8
+ignore = E501,W503

.gitignore

@@ -12,3 +12,7 @@ coverage.xml
 junit.xml
 .pytest_cache
 .tower_cli.cfg
+/app/static/files/*
+!/app/static/files/.empty
+.vscode
+.mypy_cache

.gitlab-ci.yml

-image: docker:latest
-
-.runner_tags: &runner_tags
-  tags:
-    - docker
+---
+include:
+  - remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/PreCommit.gitlab-ci.yml"
+  - remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/SonarScanner.gitlab-ci.yml"
 
 variables:
   CONTAINER_TEST_IMAGE: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME"
@@ -13,50 +12,53 @@ variables:
   POSTGRES_DB: csentry_db_test
 
 stages:
+  - check
   - build
   - test
   - analyse
   - release
   - deploy
 
+default:
+  tags:
+    - docker
+
 before_script:
   - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
 
 build:
-  <<: *runner_tags
   stage: build
+  image: docker:latest
   script:
     - docker pull "$CONTAINER_CACHE_IMAGE" || true
     - docker build --pull --cache-from "$CONTAINER_CACHE_IMAGE" -t "$CONTAINER_TEST_IMAGE" .
     - docker push "$CONTAINER_TEST_IMAGE"
 
 test:
-  <<: *runner_tags
   stage: test
   image: "$CONTAINER_TEST_IMAGE"
   services:
     - postgres:10
     - redis:4.0
-  before_script: []
+    - name: docker.elastic.co/elasticsearch/elasticsearch:7.3.2
+      alias: elasticsearch
+      command: ["bin/elasticsearch", "-Ediscovery.type=single-node"]
+  before_script:
+    - pip install -r requirements-dev.txt
   script:
-    - pytest --junitxml=junit.xml --cov-report xml:coverage.xml --cov=app -v
+    - pytest --junitxml=junit.xml --cov-report=xml:coverage.xml --cov-report=term --cov=app -v
   artifacts:
     paths:
       - junit.xml
       - coverage.xml
+    reports:
+      junit: junit.xml
     expire_in: 1 hour
 
-analyse:
-  <<: *runner_tags
-  stage: analyse
-  image: registry.esss.lu.se/ics-docker/sonar-scanner:3
-  before_script: []
-  script:
-    - sonar-scanner -Dsonar.login=$SONARQUBE_TOKEN -Dsonar.projectVersion=$CI_COMMIT_REF_NAME
-
 release-image:
-  <<: *runner_tags
   stage: release
+  image: docker:latest
+  dependencies: []
   script:
     - docker pull "$CONTAINER_TEST_IMAGE"
     - docker tag "$CONTAINER_TEST_IMAGE" "$CONTAINER_RELEASE_IMAGE"
@@ -64,17 +66,35 @@ release-image:
   only:
     - tags
 
+deploy-dev:
+  stage: deploy
+  image: registry.esss.lu.se/ics-docker/tower-cli
+  before_script: []
+  dependencies: []
+  script:
+    - >
+      tower-cli job launch
+      -h torn.tn.esss.lu.se
+      -t ${TOWER_OAUTH_TOKEN}
+      -J deploy-csentry-dev
+      -e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
+  environment:
+    name: dev
+    url: https://csentry-lab-01.cslab.esss.lu.se
+  except:
+    - master
+    - tags
+
 deploy-staging:
-  <<: *runner_tags
   stage: deploy
-  image: registry.esss.lu.se/ics-docker/tower-cli:3.2
+  image: registry.esss.lu.se/ics-docker/tower-cli
   before_script: []
+  dependencies: []
   script:
     - >
       tower-cli job launch
-      -h icsv-awx01.esss.lu.se
-      -u ${ANSIBLE_AWX_USERNAME}
-      -p ${ANSIBLE_AWX_PASSWORD}
+      -h torn.tn.esss.lu.se
+      -t ${TOWER_OAUTH_TOKEN}
       -J deploy-csentry-staging
       -e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
   environment:
@@ -85,10 +105,11 @@ deploy-staging:
     - tags
 
 pages:
-  <<: *runner_tags
   stage: deploy
   image: "$CONTAINER_RELEASE_IMAGE"
-  before_script: []
+  dependencies: []
+  before_script:
+    - pip install -r requirements-dev.txt
   script:
     - sphinx-build -M html docs docs/_build
     - mv docs/_build/html public
@@ -101,16 +122,15 @@ pages:
     - tags
 
 deploy-production:
-  <<: *runner_tags
   stage: deploy
-  image: registry.esss.lu.se/ics-docker/tower-cli:3.2
+  image: registry.esss.lu.se/ics-docker/tower-cli
   before_script: []
+  dependencies: []
   script:
     - >
       tower-cli job launch
-      -h icsv-awx01.esss.lu.se
-      -u ${ANSIBLE_AWX_USERNAME}
-      -p ${ANSIBLE_AWX_PASSWORD}
+      -h torn.tn.esss.lu.se
+      -t ${TOWER_OAUTH_TOKEN}
       -J deploy-csentry
       -e "csentry_tag=$CI_COMMIT_TAG" --monitor
   environment:

.pre-commit-config.yaml

+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8

CHANGELOG.rst

 Changelog
 =========
 
+Version 2021.04.06
+------------------
+
+- Validate group names (INFRA-3135)
+- Add warning text for creating vm (INFRA-3292)
+- Paginate history in ansible groups and hosts (INFRA-3136)
+
+Version 2020.11.25
+------------------
+
+- Add documentation about database versioning
+- Set display_name to username when empty (INFRA-2909)
+- Improve logging (INFRA-2899)
+
+Version 2020.11.13
+------------------
+
+- Increased hostname character limit from 20 -> 24 (INFRA-2808)
+- Use SSL by default for LDAP connection (INFRA-2805)
+- Use SonarScanner.gitlab-ci.yml template (INFRA-2832)
+
+Version 2020.10.30
+------------------
+
+- Fix inventory export to excel (INFRA-2780)
+- Add developer documentation
+
+Version 2020.10.27
+------------------
+
+- Update Python and requirements (INFRA-2649)
+- Update RQ and rq-dashboard (INFRA-2649 / INFRA-2754)
+- Fix code smells (INFRA-2729)
+- Add auditor group (INFRA-2728)
+- Update Sphinx and switch to sphinx_rtd_theme (INFRA-2742)
+
+Version 2020.10.06
+------------------
+
+- Make sensitive hosts visible for members of the scope (INFRA-2508)
+- Only hide sensitive networks not part of the user scope (INFRA-2437)
+- Automatically set network_scope group as parent of network groups (INFRA-2639)
+
+Version 2020.07.08
+------------------
+
+- Add view to edit network scope (INFRA-2316)
+- Add API endpoint to patch network scope (INFRA-2316)
+
+Version 2020.06.12
+------------------
+
+- Change MTCA-IOxOS to MTCA-IFC (INFRA-2200)
+- Add description field on Interface (INFRA-2112)
+
+Version 2020.04.16
+------------------
+
+- Allow /network/networks API endpoint for normal users (INFRA-2013)
+- Update Bootstrap to 4.4.1 (INFRA-2021)
+- Update DataTables (INFRA-2021)
+- Add tooltip on search box (INFRA-1893)
+
+Version 2020.03.17
+------------------
+
+- Add extra badges to README and documentation
+- Fix get_hosts for normal users (INFRA-1888)
+
+Version 2020.03.09
+------------------
+
+- Trigger core services update on host change (INFRA-1846)
+- Add DELETE method on all API endpoints (INFRA-1853)
+- Remove create_user API endpoint (INFRA-1854)
+
+Version 2020.03.05
+------------------
+
+- Allow users to view networks (INFRA-1809)
+- Add HOSTNAME Ansible group type (INFRA-1844)
+- Remove IOC git repository creation (INFRA-1839)
+- Make IP optional when creating interface via the API (INFRA-1833)
+- Introduce new device types for MTCA (INFRA-1835)
+
+
+Version 2020.02.25
+------------------
+
+- Fix inventory update (INFRA-1792)
+
+Version 2020.02.24
+------------------
+
+- Add app version in navbar (INFRA-1789)
+- Allow to pass LDAP settings as env variables (INFRA-1788)
+- Implement server side processing for Ansible groups (INFRA-1790)
+
+Version 0.29.0 (2020-02-03)
+---------------------------
+
+- Add MAC address column to hosts table (INFRA-1669)
+- Restrict view of networks and scopes to admin only (INFRA-1671)
+- Add sensitive field to Network class (INFRA-1671)
+- Filter out sensitive hosts for non admin users (INFRA-1671)
+- Add API endpoint and view to update network (INFRA-1724)
+- Make pages title more explicit (INFRA-1726)
+
+Version 0.28.0 (2019-12-19)
+---------------------------
+
+- Fix exception when setting new stack_member (INFRA-1648)
+- Prevent recursive dependency loop in Ansible groups (INFRA-1622)
+- Prevent network and scope overlapping (INFRA-1627)
+- Add netmask to network view (INFRA-1660)
+
+Version 0.27.1 (2019-12-04)
+---------------------------
+
+- Fix regexp for ICS id validation (INFRA-1569)
+- Allow normal users to edit hosts with no interface (INFRA-1604)
+
+Version 0.27.0 (2019-10-30)
+---------------------------
+
+- Increase interface name max length (INFRA-1324)
+- Add endpoints to patch host and interface (INFRA-1406)
+- Limit networks to same scope for extra interfaces (INFRA-1297)
+
+Version 0.26.1 (2019-09-18)
+---------------------------
+
+- Fix inventory update not triggered (INFRA-1290)
+
+Version 0.26.0 (2019-09-18)
+---------------------------
+
+- Decouple inventory and core services update (INFRA-1290)
+- Add broadcast address to network view (INFRA-1291)
+- Add view network scope page (INFRA-1292)
+- Make ansible_vars in host searchable (INFRA-1295)
+- Remove Generate ZTP configuration function (INFRA-1299)
+
+Version 0.25.1 (2019-08-30)
+---------------------------
+
+- Prevent input of invalid Ansible variables (INFRA-1221)
+- Update black and pre-commit config
+- Add pytest.ini to speed test discovery
+
+Version 0.25.0 (2019-08-22)
+---------------------------
+
+- Implement user access on network scope instead of domain (INFRA-1228)
+- Create IOC repositories with same path as name (INFRA-1230)
+
+Version 0.24.1 (2019-07-10)
+---------------------------
+
+- Fix sorting of items with some null stack_member (INFRA-1112)
+
+Version 0.24.0 (2019-07-10)
+---------------------------
+
+- Add serial_number and stack_member to hosts json model (INFRA-1111)
+
+Version 0.23.0 (2019-06-14)
+---------------------------
+
+- Increase RQ default timeout (INFRA-1051)
+
+Version 0.22.0 (2019-06-14)
+---------------------------
+
+- Make vlan optional in NetworkScope and Network (INFRA-1049)
+- Update reverse dependencies of failed tasks (INFRA-1051)
+
+Version 0.21.1 (2019-05-13)
+---------------------------
+
+- Fix users synchronization (INFRA-1026)
+- Remove caching of user retrieval (INFRA-1025)
+
+Version 0.21.0 (2019-05-10)
+---------------------------
+
+- Improve documentation about ansible-vault
+- Fix IOC host indexation (INFRA-1015)
+- Allow users to delete their host (INFRA-1018)
+- Fix job.status DeprecationWarning (INFRA-1021)
+- Allow service users to login (INFRA-1022)
+
+Version 0.20.2 (2019-05-08)
+---------------------------
+
+- Fix IOC repository creation (INFRA-1015)
+- Save ansible var when setting boot profile (INFRA-1016)
+
+Version 0.20.1 (2019-04-25)
+---------------------------
+
+- Save the device_type_id in the session (INFRA-987)
+- Fix exception when no network selected (INFRA-810)
+- Remove graylog support in uwsgi (INFRA-981)
+- Replace raven with sentry-sdk (INFRA-979)
+
+Version 0.20.0 (2019-04-05)
+---------------------------
+
+- Add API endpoint to search hosts (INFRA-931)
+- Initialize IOC repository on GitLab (INFRA-932)
+- Limit the max number of elements returned per page (INFRA-942)
+- Allow to set the boot profile from CSEntry (INFRA-943)
+
+Version 0.19.2 (2019-03-27)
+---------------------------
+
+- Fix model update in admin interface (INFRA-908)
+- Increase timeout for core services update (INFRA-895)
+- Add depends_on field on Task (INFRA-895)
+
+Version 0.19.1 (2019-03-18)
+---------------------------
+
+- Fix excel file download (INFRA-890)
+- Skip post install for windows VM creation (INFRA-877)
+
+Version 0.19.0 (2019-03-18)
+---------------------------
+
+- Add osversion field to CreateVMForm (INFRA-877)
+- Fix link to AWX job id for workflow jobs (INFRA-886)
+- Allow users to create VM/VIOC (INFRA-775)
+- Trigger AWX inventory update on database change (INFRA-887)
+
+Version 0.18.2 (2019-03-07)
+---------------------------
+
+- Fix post install job trigger (INFRA-870)
+
+Version 0.18.1 (2019-03-07)
+---------------------------
+
+- Clean extra vars passed for VM and VIOC creation (INFRA-870)
+
+Version 0.18.0 (2019-03-05)
+---------------------------
+
+- Add gateway field to network table (INFRA-809)
+- Add view network page (INFRA-860)
+- Remove tags and add is_ioc field to host table (INFRA-862)
+- Allow to launch a workflow job (INFRA-867)
+
+Version 0.17.1 (2019-02-07)
+---------------------------
+
+- Catch exception raised by Unique Validator (INFRA-777)
+- Only check the main interface in dynamic groups (INFRA-784)
+- Make sure the first interface is the main one (INFRA-785)
+
+Version 0.17.0 (2019-01-28)
+---------------------------
+
+- Implement new layout (INFRA-763)
+- Fix case insensitive search (INFRA-770)
+- Add post install job after VM creation (INFRA-769)
+
+Version 0.16.0 (2019-01-23)
+---------------------------
+
+- Explicitely define the elasticsearch mapping (INFRA-723)
+- Add favicon (INFRA-725)
+- Pass disk size in inventory when creating VM (INFRA-759)
+
+Version 0.15.1 (2018-12-10)
+---------------------------
+
+- Fix extra interface creation (INFRA-697)
+
+Version 0.15.0 (2018-11-30)
+---------------------------
+
+- Disable DHCP & DNS update on host modification (INFRA-673)
+- Use different groups for inventory and network (INFRA-578)
+- Add network permissions per domain (INFRA-578)
+- Move mac API to inventory endpoint (INFRA-674)
+- Add button to delete Ansible group (INFRA-678)
+- Save VM memory and cores as ansible variables (INFRA-640)
+- Return host fqdn in Ansible groups (INFRA-640)
+- Add extra fields to the API (INFRA-640)
+- Update Python to 3.7.1 and dependencies (INFRA-684)
+- Fix WhiteNoise used static directory (INFRA-683)
+- Fix flask subcommand name (INFRA-687)
+
+Version 0.14.0 (2018-11-08)
+---------------------------
+
+- Fix TypeError in after_commit (INFRA-613)
+- Add zabbix plugin to uwsgi (INFRA-614)
+- Allow to edit and delete comments (INFRA-618)
+- Make mac addresses unique by interface (INFRA-639)
+- Replace bootstrap-select with selectize (INFRA-644)
+- Make host/interface/cname unique (INFRA-245)
+- Switch to new template to deploy VM in proxmox (INFRA-651)
+
+Version 0.13.0 (2018-10-12)
+---------------------------
+
+- Implement full text search for items (INFRA-575)
+- Implement server side processing and full text search for network hosts (INFRA-595)
+- Enable datatables state saving (INFRA-608)
+- Compile uwsgi with graylog2 plugin (INFRA-576)
+- Fix javascript error: Cannot read property '_buttons' of undefined (INFRA-584)
+- Fix LDAPInvalidFilterErrorldap3 (INFRA-550)
+- Fix IndexError when mail is set to an empty list (INFRA-610)
+
+Version 0.12.0 (2018-09-28)
+---------------------------
+
+- Update DataTables with bootstrap 4 theme (INFRA-545)
+- Implement alert flashing from javascript (INFRA-545)
+- Add button to export items to excel file (INFRA-545)
+- Add button to delete hosts (INFRA-557)
+- Allow to delete hosts and interfaces via the API (INFRA-570)
+- Display hosts with no interface on list hosts page (INFRA-569)
+- Ignore old "lost" deferred tasks (INFRA-559)
+
+Version 0.11.4 (2018-09-24)
+---------------------------
+
+- Replace AWX username/password with oauth token for deployment
+- Allow to resize the CodeMirror editor (INFRA-543)
+- Add yaml constructor to support "!vault" tag (INFRA-544)
+- Fix TowerCLIError when ztp_stack is empty (INFRA-547)
+- Fix AttributeError when creating host with group (INFRA-548)
+- Enable sentry user feedback for crash reports
+
+Version 0.11.3 (2018-09-14)
+---------------------------
+
+- Add sentry integration (INFRA-525)
+- Pass sensitive information via environment variables
+
+Version 0.11.2 (2018-09-13)
+---------------------------
+
+- Add versioning on Host and AnsibleGroup (INFRA-500)
+- Disable artifact downloads in release and deploy stages
+
+Version 0.11.1 (2018-08-21)
+---------------------------
+
+- Update tower-cli version for deployment
+- Add host FQDN in host view (INFRA-459)
+- Pass ztp_stack var to trigger_ztp_configuration (INFRA-458)
+- Update Ansible groups documentation
+
+Version 0.11.0 (2018-08-16)
+---------------------------
+
+- Add Ansible parent - child groups
+- Add Ansible dynamic groups
+- Remove flask-bootstrap dependency
+- Update to bootstrap 4.1.3
+- Switch to python-slim image and update requirements
+- Replace jwt decorators with flask-login
+
+Version 0.10.2 (2018-07-23)
+---------------------------
+
+- Allow retrieving interfaces by network name (INFRA-424)
+
+Version 0.10.1 (2018-07-17)
+---------------------------
+
+- Wrap long lines in Ansible groups table
+- Add model field to /networks/hosts endpoint (INFRA-414)
+- Add model field to /networks/interfaces endpoint (INFRA-414)
+- Add trigger ZTP configuration task (INFRA-414)
+
+Version 0.10.0 (2018-07-16)
+---------------------------
+
+- Use sqlalchemy events hook to trigger tasks
+- Add Ansible variables on the host table (INFRA-412)
+- Add Ansible groups table and views (INFRA-412)
+- Add new api endpoint /network/groups (INFRA-412)
+
+
+Version 0.9.0 (2018-07-06)
+--------------------------
+
+- Redirect to the view host page when creating a host (INFRA-402)
+- Save tasks (background jobs) to the database (INFRA-403)
+- Add blueprint to view tasks (INFRA-403)
+- Redirect to the task view page when creating a VM
+- Display host creation date and user on view host page
+- Use black for source code formatting
+- Add pre-commit hooks for code formatting and linting
+- Update documentation
+
+Version 0.8.1
+-------------
+
+- Fix VM creation (memory shall be passed in MB)
+
+Version 0.8.0
+-------------
+
+- Hide interface name in create host form (INFRA-287)
+- Allow to link several items to one host (INFRA-267)
+- Add stack_member field to item (INFRA-267)
+- Add extra device types (INFRA-302)
+- Add IOC tag (INFRA-302)
+- Select the IOC tag by default based on device type (INFRA-302)
+- Use bootstrap-select for tags selection
+- Use the last IP as network gateway (INFRA-339)
+- Fix default selected tags in edit interface view (INFRA-344)
+- Use CamelCase for CSEntry device types and tags (INFRA-334)
+- Add RQ to process jobs in the background
+- Add RQ Dashboard blueprint (admin only)
+- Automatically trigger update of TN core services on host or interface change
+- Add Create VM button (admin only for now)
+- Allow to easily identify staging server (INFRA-347)
+- Update documentation
+
 Version 0.7.0
 -------------
 

Dockerfile

-FROM registry.esss.lu.se/ics-docker/miniconda:4.5
+FROM python:3.8-slim as base
 
-USER root
-WORKDIR /app
-RUN chown conda:users /app
+# Add ESS specific debian repository mirror
+RUN echo "deb https://artifactory.esss.lu.se/artifactory/debian-mirror stable main" | tee /etc/apt/sources.list.d/ess-debian-mirror.list
+
+# Install Python dependencies in an intermediate image
+# as some requires a compiler (psycopg2)
+FROM base as builder
+
+# Install dependencies required to compile some Python packages
+# Taken from https://github.com/docker-library/python/blob/master/3.6/stretch/slim/Dockerfile
+# For psycopg2: libpq-dev
+# For pillow: libjpeg-dev libpng-dev libtiff-dev
+RUN apt-get update \
+  && apt-get install -yq --no-install-recommends \
+    gcc \
+    libbz2-dev \
+    libc6-dev \
+    libexpat1-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libgdbm-dev \
+    liblzma-dev \
+    libncursesw5-dev \
+    libpcre3-dev \
+    libpng-dev \
+    libpq-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    libtiff-dev \
+    make \
+    tk-dev \
+    wget \
+    xz-utils \
+    zlib1g-dev \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /requirements.txt
+RUN python -m venv /venv \
+  && . /venv/bin/activate \
+  && pip install --no-cache-dir --upgrade pip \
+  && pip install --no-cache-dir -r /requirements.txt
+
+ARG CSENTRY_BUILD
+COPY requirements-dev.txt /requirements-dev.txt
+RUN if [ "$CSENTRY_BUILD" = "DEV" ] ; then /venv/bin/pip install --no-cache-dir -r /requirements-dev.txt ; fi
 
-USER conda
+FROM base
 
-# Install CSEntry requirements
-COPY environment.yml /app/environment.yml
-RUN conda env create -n csentry -f environment.yml \
-    && rm -rf /opt/conda/pkgs/*
+RUN groupadd -r -g 1000 csi \
+  && useradd --no-log-init -r -g csi -u 1000 csi
+
+COPY --chown=csi:csi --from=builder /venv /venv
+
+# Install libraries for psycopg2 and pillow
+# Shall be the same as the one linked to when compiling in builder image!
+RUN apt-get update \
+  && apt-get install -yq --no-install-recommends \
+    libjpeg62-turbo \
+    libpng16-16 \
+    libtiff5 \
+    libpq5 \
+    libpcre3 \
+    zlib1g \
+    git \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN rm /etc/apt/sources.list.d/ess-debian-mirror.list
+
+COPY --chown=csi:csi . /app/
+WORKDIR /app
+RUN echo "__version__ = \"$(git describe)\"" > app/_version.py
 
-# Install the app
-COPY . /app/
+ENV PATH /venv/bin:$PATH
+EXPOSE 8000
+CMD ["flask", "run", "--host", "0.0.0.0", "--port", "8000"]
 
-# activate the csentry environment
-ENV PATH /opt/conda/envs/csentry/bin:$PATH
+USER csi

Makefile

-.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image
+.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image docs
 
 OWNER := registry.esss.lu.se/ics-infrastructure
 GIT_TAG := $(shell git describe --always)
@@ -38,21 +38,21 @@ release: refresh \
 release: ## build, tag, and push all stacks
 
 db: ## start postgres and redis for development
-	docker-compose up -d postgres redis worker
+	docker-compose up -d postgres redis elasticsearch worker
 
 init_db: ## initialize the dev database
 	docker-compose run --rm web flask db upgrade head
-	docker-compose run --rm web flask create_defaults
+	docker-compose run --rm web flask create-defaults
 
 upgrade_db: ## upgrade the dev database
 	docker-compose run --rm web flask db upgrade head
 
 test:  ## run the tests (on current directory)
-	docker-compose run --rm web pytest --cov=app -v
+	docker-compose -f docker-compose.yml run --rm web pytest --cov=app -v
 
-db_image: ## start postgres and redis to test the latest image
-# Pass docker-compose.yml to skip docker-compose.override.yml
-	docker-compose -f docker-compose.yml up -d postgres redis
+db_test: ## start required containers for test
+# Pass docker-compose.yml to skip docker-compose.override.yml (db not mounted as volume for speed)
+	docker-compose -f docker-compose.yml up -d postgres redis elasticsearch worker
 
 test_image:  ## run the tests (on the latest image)
 # Pass docker-compose.yml to skip docker-compose.override.yml
@@ -60,3 +60,6 @@ test_image:  ## run the tests (on the latest image)
 
 run_uwsgi:  ## run the application with uwsgi (to test prod env)
 	docker-compose run -p 8000:8000 --rm web uwsgi --master --http 0.0.0.0:8000 --http-keepalive --manage-script-name --mount /csentry=wsgi.py --callable app --uid conda --processes 2 -b 16384
+
+docs:  ## run the tests (on current directory)
+	docker run --rm -v $(shell pwd):/app registry.esss.lu.se/ics-infrastructure/csentry:master sphinx-build -M html docs docs/_build

README.rst

 CSEntry
 =======
 
+.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
+    :target: https://github.com/ambv/black
+
+.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=alert_status
+    :target: https://sonarqube.esss.lu.se/dashboard?id=csentry
+
+.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=ncloc
+    :target: https://sonarqube.esss.lu.se/dashboard?id=csentry
+
+.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/pipeline.svg
+
+.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/coverage.svg
+
 Control System Entry web server.
 
 
@@ -77,16 +90,3 @@ To restore the database::
 
     $ gunzip -c csentry_db.sql.gz | docker run --rm --link csentry_postgres:postgres --net csentry_default
       -e PGPASSWORD="<csentry_password>" -i postgres:10 psql -h postgres -U ics csentry_db
-
-
-Dependencies
-------------
-
-The initial dependencies were generated using::
-
-    $ docker run --rm -it -v $(pwd):/app continuumio/miniconda3:latest bash
-    $ conda config --add channels conda-forge
-    $ conda create -n csentry python=3.6 flask alembic flask-debugtoolbar flask-login flask-sqlalchemy flask-wtf pillow psycopg2 pytest pytest-cov qrcode whitenoise factory_boy flask-admin pyjwt ldap3 flask-mail flask-migrate flask-jwt-extended
-    $ source activate csentry
-    $ pip install flask-ldap3-login sqlalchemy-citext sqlalchemy-continuum pytest-factoryboy git+https://github.com/beenje/flask-bootstrap@4.0.0-beta.1.dev1
-    $ conda env export > /app/environment.yml

app/_version.py

+__version__ = "dev"

app/admin/views.py

@@ -28,14 +28,13 @@ sqla.form.Unique.__call__ = lambda x, y, z: None
 # Add custom model converter for CIText type
 # See https://github.com/flask-admin/flask-admin/issues/1196
 class AppAdminModelConverter(sqla.form.AdminModelConverter):
-
-    @converts('CIText')
+    @converts("CIText")
     def conv_CIText(self, field_args, **extra):
         return fields.TextAreaField(**field_args)
 
-    @converts('sqlalchemy.dialects.postgresql.base.CIDR')
+    @converts("sqlalchemy.dialects.postgresql.base.CIDR")
     def conv_PGCidr(self, field_args, **extra):
-        field_args['validators'].append(IPNetwork())
+        field_args["validators"].append(IPNetwork())
         return fields.StringField(**field_args)
 
 
@@ -43,9 +42,7 @@ class AdminModelView(sqla.ModelView):
     model_form_converter = AppAdminModelConverter
 
     # Replace TextAreaField (default for Text) with StringField
-    form_overrides = {
-        'name': fields.StringField,
-    }
+    form_overrides = {"name": fields.StringField}
 
     def is_accessible(self):
         return current_user.is_authenticated and current_user.is_admin
@@ -65,16 +62,17 @@ class TokenAdmin(AdminModelView):
 class ItemAdmin(AdminModelView):
 
     # Replace TextAreaField (default for Text) with StringField
-    form_overrides = {
-        'ics_id': fields.StringField,
-        'serial_number': fields.StringField,
-    }
+    form_overrides = {"ics_id": fields.StringField, "serial_number": fields.StringField}
 
     form_args = {
-        'ics_id': {
-            'label': 'ICS id',
-            'validators': [validators.Regexp(ICS_ID_RE, message='ICS id shall match [A-Z]{3}[0-9]{3}')],
-            'filters': [lambda x: x or None],
+        "ics_id": {
+            "label": "ICS id",
+            "validators": [
+                validators.Regexp(
+                    ICS_ID_RE, message="ICS id shall match [A-Z]{3}[0-9]{3}"
+                )
+            ],
+            "filters": [lambda x: x or None],
         }
     }
 
@@ -82,12 +80,9 @@ class ItemAdmin(AdminModelView):
 class NetworkAdmin(AdminModelView):
 
     # Replace TextAreaField (default for Text) with StringField
-    form_overrides = {
-        'vlan_name': fields.StringField,
-    }
+    form_overrides = {"vlan_name": fields.StringField}
 
-    form_args = {
-        'gateway': {
-            'filters': [lambda x: x or None],
-        },
-    }
+
+class TaskAdmin(AdminModelView):
+    column_display_pk = True
+    can_create = False

app/api/inventory.py

@@ -10,12 +10,12 @@ This module implements the inventory API.
 
 """
 from flask import Blueprint, jsonify, request, current_app
-from flask_jwt_extended import jwt_required
+from flask_login import login_required
 from .. import utils, models
-from ..decorators import jwt_groups_accepted
-from .utils import commit, create_generic_model, get_generic_model
+from ..decorators import login_groups_accepted
+from .utils import commit, create_generic_model, get_generic_model, delete_generic_model
 
-bp = Blueprint('inventory_api', __name__)
+bp = Blueprint("inventory_api", __name__)
 
 
 def get_item_by_id_or_ics_id(id_):
@@ -32,21 +32,20 @@ def get_item_by_id_or_ics_id(id_):
     return item
 
 
-@bp.route('/items')
-@jwt_required
+@bp.route("/items")
+@login_required
 def get_items():
     """Return items
 
     .. :quickref: Inventory; Get items
     """
-    return get_generic_model(models.Item,
-                             order_by=models.Item.created_at)
+    return get_generic_model(models.Item, order_by=models.Item.created_at)
 
 
-@bp.route('/items/<id_>')
-@jwt_required
+@bp.route("/items/<id_>")
+@login_required
 def get_item(id_):
-    """Retrieve item by id or ICS id
+    r"""Retrieve item by id or ICS id
 
     .. :quickref: Inventory; Get item by id or ICS id
 
@@ -56,9 +55,8 @@ def get_item(id_):
     return jsonify(item.to_dict())
 
 
-@bp.route('/items', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/items", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_item():
     """Register a new item
 
@@ -79,14 +77,13 @@ def create_item():
     # But there are existing items (in confluence and JIRA) that we want to
     # import and associate after they have been created.
     # In that case a temporary id is automatically assigned.
-    return create_generic_model(models.Item, mandatory_fields=('serial_number',))
+    return create_generic_model(models.Item, mandatory_fields=("serial_number",))
 
 
-@bp.route('/items/<id_>', methods=['PATCH'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/items/<id_>", methods=["PATCH"])
+@login_groups_accepted("admin", "inventory")
 def patch_item(id_):
-    """Patch an existing item
+    r"""Patch an existing item
 
     .. :quickref: Inventory; Update existing item
 
@@ -100,24 +97,34 @@ def patch_item(id_):
     """
     data = request.get_json()
     if data is None:
-        raise utils.CSEntryError('Body should be a JSON object')
+        raise utils.CSEntryError("Body should be a JSON object")
     if not data:
-        raise utils.CSEntryError('At least one field is required', status_code=422)
+        raise utils.CSEntryError("At least one field is required", status_code=422)
     for key in data.keys():
-        if key not in ('ics_id', 'manufacturer', 'model',
-                       'location', 'status', 'parent'):
+        if key not in (
+            "ics_id",
+            "manufacturer",
+            "model",
+            "location",
+            "status",
+            "parent",
+        ):
             raise utils.CSEntryError(f"Invalid field '{key}'", status_code=422)
     item = get_item_by_id_or_ics_id(id_)
     # Only allow to set ICS id if the current id is a temporary one
-    if item.ics_id.startswith(current_app.config['TEMPORARY_ICS_ID']):
-        item.ics_id = data.get('ics_id', item.ics_id)
-    elif 'ics_id' in data:
+    if item.ics_id.startswith(current_app.config["TEMPORARY_ICS_ID"]):
+        item.ics_id = data.get("ics_id", item.ics_id)
+    elif "ics_id" in data:
         raise utils.CSEntryError("'ics_id' can't be changed", status_code=422)
-    item.manufacturer = utils.convert_to_model(data.get('manufacturer', item.manufacturer), models.Manufacturer)
-    item.model = utils.convert_to_model(data.get('model', item.model), models.Model)
-    item.location = utils.convert_to_model(data.get('location', item.location), models.Location)
-    item.status = utils.convert_to_model(data.get('status', item.status), models.Status)
-    parent_ics_id = data.get('parent')
+    item.manufacturer = utils.convert_to_model(
+        data.get("manufacturer", item.manufacturer), models.Manufacturer
+    )
+    item.model = utils.convert_to_model(data.get("model", item.model), models.Model)
+    item.location = utils.convert_to_model(
+        data.get("location", item.location), models.Location
+    )
+    item.status = utils.convert_to_model(data.get("status", item.status), models.Status)
+    parent_ics_id = data.get("parent")
     if parent_ics_id is not None:
         parent = models.Item.query.filter_by(ics_id=parent_ics_id).first()
         if parent is not None:
@@ -133,10 +140,22 @@ def patch_item(id_):
     return jsonify(item.to_dict())
 
 
-@bp.route('/items/<id_>/comments')
-@jwt_required
+@bp.route("/items/<int:item_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_item(item_id):
+    """Delete an item
+
+    .. :quickref: Inventory; Delete an item
+
+    :param item_id: item primary key
+    """
+    return delete_generic_model(models.Item, item_id)
+
+
+@bp.route("/items/<id_>/comments")
+@login_required
 def get_item_comments(id_):
-    """Get item comments
+    r"""Get item comments
 
     .. :quickref: Inventory; Get item comments
 
@@ -146,11 +165,10 @@ def get_item_comments(id_):
     return jsonify([comment.to_dict() for comment in item.comments])
 
 
-@bp.route('/items/<id_>/comments', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/items/<id_>/comments", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_item_comment(id_):
-    """Create a comment on item
+    r"""Create a comment on item
 
     .. :quickref: Inventory; Create comment on item
 
@@ -158,13 +176,25 @@ def create_item_comment(id_):
     :jsonparam body: comment body
     """
     item = get_item_by_id_or_ics_id(id_)
-    return create_generic_model(models.ItemComment,
-                                mandatory_fields=('body',),
-                                item_id=item.id)
+    return create_generic_model(
+        models.ItemComment, mandatory_fields=("body",), item_id=item.id
+    )
+
+
+@bp.route("/items/comments/<int:comment_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_comment(comment_id):
+    """Delete an item comment
+
+    .. :quickref: Inventory; Delete an item comment
+
+    :param comment_id: comment primary key
+    """
+    return delete_generic_model(models.ItemComment, comment_id)
 
 
-@bp.route('/actions')
-@jwt_required
+@bp.route("/actions")
+@login_required
 def get_actions():
     """Get actions
 
@@ -173,8 +203,8 @@ def get_actions():
     return get_generic_model(models.Action)
 
 
-@bp.route('/manufacturers')
-@jwt_required
+@bp.route("/manufacturers")
+@login_required
 def get_manufacturers():
     """Get manufacturers
 
@@ -183,9 +213,8 @@ def get_manufacturers():
     return get_generic_model(models.Manufacturer)
 
 
-@bp.route('/manufacturers', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/manufacturers", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_manufacturer():
     """Create a new manufacturer
 
@@ -197,8 +226,20 @@ def create_manufacturer():
     return create_generic_model(models.Manufacturer)
 
 
-@bp.route('/models')
-@jwt_required
+@bp.route("/manufacturers/<int:manufacturer_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_manufacturer(manufacturer_id):
+    """Delete a manufacturer
+
+    .. :quickref: Inventory; Delete a manufacturer
+
+    :param manufacturer_id: manufacturer primary key
+    """
+    return delete_generic_model(models.Manufacturer, manufacturer_id)
+
+
+@bp.route("/models")
+@login_required
 def get_models():
     """Get models
 
@@ -207,9 +248,8 @@ def get_models():
     return get_generic_model(models.Model)
 
 
-@bp.route('/models', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/models", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_model():
     """Create a new model
 
@@ -221,8 +261,20 @@ def create_model():
     return create_generic_model(models.Model)
 
 
-@bp.route('/locations')
-@jwt_required
+@bp.route("/models/<int:model_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_model(model_id):
+    """Delete a model
+
+    .. :quickref: Inventory; Delete a model
+
+    :param model_id: model primary key
+    """
+    return delete_generic_model(models.Model, model_id)
+
+
+@bp.route("/locations")
+@login_required
 def get_locations():
     """Get locations
 
@@ -231,9 +283,8 @@ def get_locations():
     return get_generic_model(models.Location)
 
 
-@bp.route('/locations', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/locations", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_locations():
     """Create a new location
 
@@ -245,8 +296,20 @@ def create_locations():
     return create_generic_model(models.Location)
 
 
-@bp.route('/statuses')
-@jwt_required
+@bp.route("/locations/<int:location_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_location(location_id):
+    """Delete a location
+
+    .. :quickref: Inventory; Delete a location
+
+    :param location_id: location primary key
+    """
+    return delete_generic_model(models.Location, location_id)
+
+
+@bp.route("/statuses")
+@login_required
 def get_status():
     """Get statuses
 
@@ -255,9 +318,8 @@ def get_status():
     return get_generic_model(models.Status)
 
 
-@bp.route('/statuses', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/statuses", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_status():
     """Create a new status
 
@@ -267,3 +329,38 @@ def create_status():
     :jsonparam description: (optional) description
     """
     return create_generic_model(models.Status)
+
+
+@bp.route("/statuses/<int:status_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_status(status_id):
+    """Delete a status
+
+    .. :quickref: Inventory; Delete a status
+
+    :param status_id: status primary key
+    """
+    return delete_generic_model(models.Status, status_id)
+
+
+@bp.route("/macs")
+@login_required
+def get_macs():
+    """Return mac addresses
+
+    .. :quickref: Inventory; Get mac addresses
+    """
+    return get_generic_model(models.Mac, order_by=models.Mac.address)
+
+
+@bp.route("/macs", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
+def create_macs():
+    """Create a new mac address
+
+    .. :quickref: Inventory; Create new mac address
+
+    :jsonparam address: MAC address
+    :jsonparam item_id: (optional) linked item primary key
+    """
+    return create_generic_model(models.Mac, mandatory_fields=("address",))

app/api/user.py

@@ -11,54 +11,36 @@ This module implements the user API.
 """
 from flask import current_app, Blueprint, jsonify, request
 from flask_ldap3_login import AuthenticationResponseStatus
-from flask_jwt_extended import jwt_required, get_current_user
+from flask_login import login_required, current_user
 from ..extensions import ldap_manager
-from ..decorators import jwt_groups_accepted
+from ..decorators import login_groups_accepted
 from .. import utils, tokens, models
-from .utils import get_generic_model, create_generic_model
+from .utils import get_generic_model
 
-bp = Blueprint('user_api', __name__)
+bp = Blueprint("user_api", __name__)
 
 
-@bp.route('/users')
-@jwt_required
+@bp.route("/users")
+@login_groups_accepted("admin", "auditor")
 def get_users():
     """Return users information
 
     .. :quickref: User; Get users information
     """
-    return get_generic_model(models.User,
-                             order_by=models.User.username)
+    return get_generic_model(models.User, order_by=models.User.username)
 
 
-@bp.route('/profile')
-@jwt_required
+@bp.route("/profile")
+@login_required
 def get_user_profile():
     """Return the current user profile
 
     .. :quickref: User; Get current user profile
     """
-    user = get_current_user()
-    return jsonify(user.to_dict()), 200
+    return jsonify(current_user.to_dict()), 200
 
 
-@bp.route('/users', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin')
-def create_user():
-    """Create a new user
-
-    .. :quickref: User; Create new user
-
-    :jsonparam username: User's username
-    :jsonparam display_name: User's display name
-    :jsonparam email: User's email
-    """
-    return create_generic_model(models.User, mandatory_fields=(
-        'username', 'display_name', 'email'))
-
-
-@bp.route('/login', methods=['POST'])
+@bp.route("/login", methods=["POST"])
 def login():
     """Return a JSON Web Token
 
@@ -69,20 +51,20 @@ def login():
     """
     data = request.get_json()
     if data is None:
-        raise utils.CSEntryError('Body should be a JSON object')
+        raise utils.CSEntryError("Body should be a JSON object")
     try:
-        username = data['username']
-        password = data['password']
+        username = data["username"]
+        password = data["password"]
     except KeyError:
-        raise utils.CSEntryError('Missing mandatory field (username or password)', status_code=422)
+        raise utils.CSEntryError(
+            "Missing mandatory field (username or password)", status_code=422
+        )
     response = ldap_manager.authenticate(username, password)
     if response.status == AuthenticationResponseStatus.success:
-        current_app.logger.debug(f'{username} successfully logged in')
+        current_app.logger.debug(f"{username} successfully logged in")
         user = ldap_manager._save_user(
-            response.user_dn,
-            response.user_id,
-            response.user_info,
-            response.user_groups)
-        payload = {'access_token': tokens.generate_access_token(identity=user.id)}
+            response.user_dn, response.user_id, response.user_info, response.user_groups
+        )
+        payload = {"access_token": tokens.generate_access_token(identity=user.id)}
         return jsonify(payload), 200
-    raise utils.CSEntryError('Invalid credentials', status_code=401)
+    raise utils.CSEntryError("Invalid credentials", status_code=401)

app/api/utils.py

@@ -12,6 +12,8 @@ This module implements useful functions for the API.
 import urllib.parse
 import sqlalchemy as sa
 from flask import current_app, jsonify, request
+from flask_login import current_user
+from flask_sqlalchemy import Pagination
 from ..extensions import db
 from .. import utils
 
@@ -19,7 +21,7 @@ from .. import utils
 def commit():
     try:
         db.session.commit()
-    except (sa.exc.IntegrityError, sa.exc.DataError) as e:
+    except sa.exc.SQLAlchemyError as e:
         db.session.rollback()
         raise utils.CSEntryError(str(e), status_code=422)
 
@@ -32,71 +34,174 @@ def build_pagination_header(pagination, base_url, **kwargs):
     :param kwargs: extra query string parameters (without page and per_page)
     :returns: dict with X-Total-Count and Link keys
     """
-    header = {'X-Total-Count': pagination.total}
+    header = {"X-Total-Count": pagination.total}
     links = []
     if pagination.page > 1:
-        params = urllib.parse.urlencode({'per_page': pagination.per_page,
-                                         'page': 1,
-                                         **kwargs})
+        params = urllib.parse.urlencode(
+            {"per_page": pagination.per_page, "page": 1, **kwargs}
+        )
         links.append(f'<{base_url}?{params}>; rel="first"')
     if pagination.has_prev:
-        params = urllib.parse.urlencode({'per_page': pagination.per_page,
-                                         'page': pagination.prev_num,
-                                         **kwargs})
+        params = urllib.parse.urlencode(
+            {"per_page": pagination.per_page, "page": pagination.prev_num, **kwargs}
+        )
         links.append(f'<{base_url}?{params}>; rel="prev"')
     if pagination.has_next:
-        params = urllib.parse.urlencode({'per_page': pagination.per_page,
-                                         'page': pagination.next_num,
-                                         **kwargs})
+        params = urllib.parse.urlencode(
+            {"per_page": pagination.per_page, "page": pagination.next_num, **kwargs}
+        )
         links.append(f'<{base_url}?{params}>; rel="next"')
     if pagination.pages > pagination.page:
-        params = urllib.parse.urlencode({'per_page': pagination.per_page,
-                                         'page': pagination.pages,
-                                         **kwargs})
+        params = urllib.parse.urlencode(
+            {"per_page": pagination.per_page, "page": pagination.pages, **kwargs}
+        )
         links.append(f'<{base_url}?{params}>; rel="last"')
     if links:
-        header['Link'] = ', '.join(links)
+        header["Link"] = ", ".join(links)
     return header
 
 
-def get_generic_model(model, order_by=None, query=None):
+def get_generic_model(model, order_by=None, base_query=None, query=None):
     """Return data from model as json
 
     :param model: model class
-    :param order_by: column to order the result by
+    :param order_by: column to order the result by (not used if base_query or query is passed)
+    :param query: optional base_query to use [default: model.query]
     :param query: optional query to use (for more complex queries)
     :returns: data from model as json
     """
     kwargs = request.args.to_dict()
-    page = int(kwargs.pop('page', 1))
-    per_page = int(kwargs.pop('per_page', 20))
+    page = int(kwargs.pop("page", 1))
+    per_page = int(kwargs.pop("per_page", 20))
+    # Remove recursive from kwargs so that it doesn't get passed
+    # to query.filter_by in get_query
+    recursive = kwargs.pop("recursive", "false").lower() == "true"
     if query is None:
-        query = utils.get_query(model.query, **kwargs)
-        if order_by is None:
-            order_by = getattr(model, 'name')
-        query = query.order_by(order_by)
-    pagination = query.paginate(page, per_page)
-    data = [item.to_dict() for item in pagination.items]
+        if base_query is None:
+            base_query = model.query
+            if order_by is None:
+                order_by = getattr(model, "name")
+            base_query = base_query.order_by(order_by)
+        query = utils.get_query(base_query, model, **kwargs)
+    pagination = query.paginate(
+        page, per_page, max_per_page=current_app.config["MAX_PER_PAGE"]
+    )
+    data = [item.to_dict(recursive=recursive) for item in pagination.items]
+    # Re-add recursive to kwargs so that it's part of the pagination url
+    kwargs["recursive"] = recursive
     header = build_pagination_header(pagination, request.base_url, **kwargs)
     return jsonify(data), 200, header
 
 
-def create_generic_model(model, mandatory_fields=('name',), **kwargs):
+def search_generic_model(model, filter_sensitive=False):
+    """Return filtered data from model as json
+
+    :param model: model class
+    :param bool filter_sensitive: filter out sensitive data if set to True
+    :returns: filtered data from model as json
+    """
+    kwargs = request.args.to_dict()
+    page = int(kwargs.pop("page", 1))
+    per_page = int(kwargs.pop("per_page", 20))
+    per_page = min(per_page, current_app.config["MAX_PER_PAGE"])
+    search = kwargs.get("q", "*")
+    instances, nb_filtered = model.search(
+        search, page=page, per_page=per_page, filter_sensitive=filter_sensitive
+    )
+    current_app.logger.debug(
+        f'Found {nb_filtered} {model.__tablename__}(s) when searching "{search}"'
+    )
+    data = [instance.to_dict(recursive=True) for instance in instances]
+    pagination = Pagination(None, page, per_page, nb_filtered, None)
+    header = build_pagination_header(pagination, request.base_url, **kwargs)
+    return jsonify(data), 200, header
+
+
+def get_json_body():
+    """Return the json body from the current request
+
+    Raise a CSEntryError if the body is not a json object
+    """
     data = request.get_json()
     if data is None:
-        raise utils.CSEntryError('Body should be a JSON object')
-    current_app.logger.debug(f'Received: {data}')
+        raise utils.CSEntryError("Body should be a JSON object")
+    if not data:
+        raise utils.CSEntryError("At least one field is required", status_code=422)
+    return data
+
+
+def create_generic_model(model, mandatory_fields=("name",), **kwargs):
+    """Create an instance of the model
+
+    :param model: model class
+    :param mandatory_fields: list of fields that shall be passed in the body
+    :param kwargs: extra fields to use
+    :returns: representation of the created instance as json
+    """
+    data = get_json_body()
     data.update(kwargs)
     for mandatory_field in mandatory_fields:
         if mandatory_field not in data:
-            raise utils.CSEntryError(f"Missing mandatory field '{mandatory_field}'", status_code=422)
+            raise utils.CSEntryError(
+                f"Missing mandatory field '{mandatory_field}'", status_code=422
+            )
     try:
         instance = model(**data)
     except TypeError as e:
-        message = str(e).replace('__init__() got an ', '')
+        message = str(e).replace("__init__() got an ", "")
         raise utils.CSEntryError(message, status_code=422)
     except ValueError as e:
         raise utils.CSEntryError(str(e), status_code=422)
     db.session.add(instance)
     commit()
+    current_app.logger.info(
+        f"New {model.__tablename__} created by {current_user}: {instance.to_dict()}"
+    )
     return jsonify(instance.to_dict()), 201
+
+
+def delete_generic_model(model, primary_key):
+    """Delete the model based on the primary_key
+
+    :param model: model class
+    :param primary_key: primary key of the instance to delete
+    """
+    instance = model.query.get_or_404(primary_key)
+    db.session.delete(instance)
+    commit()
+    current_app.logger.info(
+        f"{model.__tablename__} {instance} deleted by {current_user}"
+    )
+    return jsonify(), 204
+
+
+def update_generic_model(model, primary_key, allowed_fields):
+    """Update the model based on the primary_key
+
+    :param model: model class
+    :param primary_key: primary key of the instance to update
+    :param allowed_fields: list of fields that can be updated
+    :returns: representation of the updated model as json
+    """
+    data = get_json_body()
+    # Use a list and not a set because the order is important
+    allowed_keys = [field[0] for field in allowed_fields]
+    for key in data:
+        if key not in allowed_keys:
+            raise utils.CSEntryError(f"Invalid field '{key}'", status_code=422)
+    instance = model.query.get_or_404(primary_key)
+    data = utils.convert_to_models(data, allowed_fields)
+    try:
+        # Loop on allowed_keys and not data to respect
+        # the order in which the fields are set
+        # setting ip before network is important for Interface
+        for key in allowed_keys:
+            if key in data:
+                setattr(instance, key, data[key])
+    except Exception as e:
+        raise utils.CSEntryError(str(e), status_code=422)
+    commit()
+    current_app.logger.info(
+        f"{model.__tablename__} {primary_key} updated by {current_user}: {instance.to_dict()}"
+    )
+    return jsonify(instance.to_dict()), 200

app/commands.py

@@ -9,60 +9,102 @@ This module defines extra flask commands.
 :license: BSD 2-Clause, see LICENSE for more details.
 
 """
+import click
 import ldap3
 import redis
 import rq
 import sqlalchemy as sa
+import sentry_sdk
 from flask import current_app
+from sentry_sdk.integrations.rq import RqIntegration
 from .extensions import db, ldap_manager
 from .defaults import defaults
-from .models import User
-from . import utils, tokens
+from .tasks import TaskWorker
+from . import models, utils, tokens
+
+
+def disable_user(user):
+    """Clear users'groups, email and tokens"""
+    user.groups = []
+    user.email = ""
+    # Revoke all user's tokens
+    for token in user.tokens:
+        db.session.delete(token)
 
 
 def sync_user(connection, user):
     """Synchronize the user from the database with information from the LDAP server"""
-    search_attr = current_app.config.get('LDAP_USER_LOGIN_ATTR')
-    object_filter = current_app.config.get('LDAP_USER_OBJECT_FILTER')
-    search_filter = f'(&{object_filter}({search_attr}={user.username}))'
+    search_attr = current_app.config.get("LDAP_USER_LOGIN_ATTR")
+    object_filter = current_app.config.get("LDAP_USER_OBJECT_FILTER")
+    search_filter = f"(&{object_filter}({search_attr}={user.username}))"
     connection.search(
         search_base=ldap_manager.full_user_search_dn,
         search_filter=search_filter,
-        search_scope=getattr(
-            ldap3, current_app.config.get('LDAP_USER_SEARCH_SCOPE')),
-        attributes=current_app.config.get('LDAP_GET_USER_ATTRIBUTES')
+        search_scope=getattr(ldap3, current_app.config.get("LDAP_USER_SEARCH_SCOPE")),
+        attributes=current_app.config.get("LDAP_GET_USER_ATTRIBUTES"),
     )
-    if len(connection.response) == 1:
-        ldap_user = connection.response[0]
-        attributes = ldap_user['attributes']
-        user.display_name = utils.attribute_to_string(attributes['cn'])
-        user.email = utils.attribute_to_string(attributes['mail'])
-        groups = ldap_manager.get_user_groups(dn=ldap_user['dn'], _connection=connection)
-        user.groups = sorted([utils.attribute_to_string(group['cn']) for group in groups])
-        current_app.logger.info(f'{user} updated')
+    results = [
+        result for result in connection.response if result["type"] == "searchResEntry"
+    ]
+    if len(results) == 1:
+        ldap_user = results[0]
+        # OU=InActiveUsers is specific to ESS AD
+        if "OU=InActiveUsers" in ldap_user["dn"]:
+            current_app.logger.info(f"{user} is inactive. User disabled.")
+            disable_user(user)
+        else:
+            attributes = ldap_user["attributes"]
+            user.display_name = (
+                utils.attribute_to_string(attributes["cn"]) or user.username
+            )
+            user.email = utils.attribute_to_string(attributes["mail"])
+            groups = ldap_manager.get_user_groups(
+                dn=ldap3.utils.conv.escape_filter_chars(ldap_user["dn"]),
+                _connection=connection,
+            )
+            user.groups = sorted(
+                [utils.attribute_to_string(group["cn"]) for group in groups]
+            )
+            current_app.logger.info(f"{user} updated")
+    elif len(results) == 0:
+        current_app.logger.warning(f"{user} not found! User disabled.")
+        disable_user(user)
     else:
-        # Clear user's groups
-        user.groups = []
-        # Revoke all user's tokens
-        for token in user.tokens:
-            db.session.delete(token)
-        current_app.logger.info(f'{user} disabled')
+        current_app.logger.warning(f"Too many results for {user}!")
+        current_app.logger.warning(f"results: {results}")
     return user
 
 
 def sync_users():
     """Synchronize all users from the database with information the LDAP server"""
-    current_app.logger.info('Synchronize database with information from the LDAP server')
+    current_app.logger.info(
+        "Synchronize database with information from the LDAP server"
+    )
     try:
         connection = ldap_manager.connection
     except ldap3.core.exceptions.LDAPException as e:
-        current_app.logger.warning(f'Failed to connect to the LDAP server: {e}')
+        current_app.logger.warning(f"Failed to connect to the LDAP server: {e}")
         return
-    for user in User.query.all():
+    for user in models.User.query.all():
         sync_user(connection, user)
     db.session.commit()
 
 
+def clean_deferred_tasks():
+    """Set all deferred tasks to failed"""
+    for task in (
+        models.Task.query.filter_by(status=models.JobStatus.DEFERRED)
+        .order_by(models.Task.created_at)
+        .all()
+    ):
+        if task.depends_on is None or task.depends_on.status == models.JobStatus.FAILED:
+            current_app.logger.info(
+                f"Set deferred task {task.id} ({task.name}) to failed"
+            )
+            task.status = models.JobStatus.FAILED
+    db.session.commit()
+
+
 def register_cli(app):
     @app.cli.command()
     def create_defaults():
@@ -71,9 +113,14 @@ def register_cli(app):
             db.session.add(instance)
             try:
                 db.session.commit()
-            except sa.exc.IntegrityError as e:
+            except sa.exc.IntegrityError:
                 db.session.rollback()
-                app.logger.debug(f'{instance} already exists')
+                app.logger.debug(f"{instance} already exists")
+
+    @app.cli.command()
+    def clean_deferred():
+        """Set deferred tasks to failed if the task it depends on failed"""
+        clean_deferred_tasks()
 
     @app.cli.command()
     def syncusers():
@@ -94,8 +141,37 @@ def register_cli(app):
     @app.cli.command()
     def runworker():
         """Run RQ worker"""
-        redis_url = current_app.config['RQ_REDIS_URL']
+        redis_url = current_app.config["RQ_REDIS_URL"]
         redis_connection = redis.from_url(redis_url)
         with rq.Connection(redis_connection):
-            worker = rq.Worker(current_app.config['QUEUES'])
+            worker = TaskWorker(["high", "normal", "low"])
+            if current_app.config["SENTRY_DSN"]:
+                sentry_sdk.init(
+                    current_app.config["SENTRY_DSN"],
+                    environment=current_app.config["CSENTRY_ENVIRONMENT"],
+                    integrations=[RqIntegration()],
+                )
             worker.work()
+
+    @app.cli.command()
+    @click.option(
+        "--delete/--no-delete",
+        default=True,
+        help="Delete and recreate the index [default: True]",
+    )
+    @click.option(
+        "--name", default="all", help="name of the class to reindex [default: all]"
+    )
+    def reindex(delete, name):
+        """Initialize the elasticsearch index"""
+        if delete:
+            current_app.elasticsearch.indices.delete("*", ignore=404)
+        if name == "all":
+            models.Item.reindex(delete)
+            models.Host.reindex(delete)
+            models.AnsibleGroup.reindex(delete)
+            return
+        try:
+            getattr(models, name).reindex(delete)
+        except AttributeError:
+            click.echo(f"The class {name} can't be indexed")

app/decorators.py

@@ -10,67 +10,50 @@ This module defines some useful decorators.
 
 """
 from functools import wraps
-from flask import current_app, abort
+from flask import current_app, abort, g
 from flask_login import current_user
-from flask_jwt_extended import get_current_user
 from .utils import CSEntryError
 
 
-def jwt_groups_accepted(*groups):
-    """Decorator which specifies that a user must have at least one of the specified groups.
-
-    This shall be used for users logged in using a JWT (API).
-
-    Example::
-        @bp.route('/models', methods=['POST'])
-        @jwt_required
-        @jwt_groups_accepted('admin', 'create')
-        def create_model():
-            return create()
-
-    The current user must be in either 'admin' or 'create' group
-    to access this route.
-
-    :param groups: accepted groups
-    """
-    def wrapper(fn):
-        @wraps(fn)
-        def decorated_view(*args, **kwargs):
-            user = get_current_user()
-            if user is None:
-                raise CSEntryError('Invalid indentity', status_code=403)
-            if not user.is_member_of_one_group(groups):
-                raise CSEntryError("User doesn't have the required group", status_code=403)
-            return fn(*args, **kwargs)
-        return decorated_view
-    return wrapper
-
-
 def login_groups_accepted(*groups):
     """Decorator which specifies that a user must have at least one of the specified groups.
 
-    This shall be used for users logged in using a cookie (web UI).
+    This can be used for users logged in using a cookie (web UI) or JWT (API).
 
     Example::
+
         @bp.route('/models', methods=['POST'])
-        @login_groups_accepted('admin', 'create')
+        @login_groups_accepted('admin', 'inventory')
         def create_model():
             return create()
 
-    The current user must be in either 'admin' or 'create' group
-    to access this route.
+        The current user must be in either 'admin' or 'create' group
+        to access the /models route.
+
     This checks that the user is logged in. There is no need to
     use the @login_required decorator.
 
     :param groups: accepted groups
     """
+
     def wrapper(fn):
         @wraps(fn)
         def decorated_view(*args, **kwargs):
+            # LOGIN_DISABLED can be set to True to turn off authentication check when testing.
+            # This variable is also used by flask-login
+            if current_app.config.get("LOGIN_DISABLED"):
+                return fn(*args, **kwargs)
             if not current_user.is_authenticated:
                 return current_app.login_manager.unauthorized()
             if not current_user.is_member_of_one_group(groups):
-                abort(403)
+                if g.get("login_via_request"):
+                    raise CSEntryError(
+                        "User doesn't have the required group", status_code=403
+                    )
+                else:
+                    abort(403)
             return fn(*args, **kwargs)
+
         return decorated_view
+
     return wrapper

app/defaults.py

@@ -13,22 +13,20 @@ from . import models
 
 
 defaults = [
-    models.Action(name='Assign ICS id'),
-    models.Action(name='Clear all'),
-    models.Action(name='Clear attributes'),
-    models.Action(name='Fetch'),
-    models.Action(name='Register'),
-    models.Action(name='Set as parent'),
-    models.Action(name='Update'),
-
-    models.DeviceType(name='Physical'),
-    models.DeviceType(name='Virtual Machine'),
-    models.DeviceType(name='Switch'),
-    models.DeviceType(name='Virtual IOC'),
-    models.DeviceType(name='Industrial PC IOC'),
-    models.DeviceType(name='MicroTCA IOC'),
-    models.DeviceType(name='VME IOC'),
-    models.DeviceType(name='PLC'),
-
-    models.Tag(name='gateway', admin_only=True),
+    models.Action(name="Assign ICS id"),
+    models.Action(name="Clear all"),
+    models.Action(name="Clear attributes"),
+    models.Action(name="Fetch"),
+    models.Action(name="Register"),
+    models.Action(name="Set as parent"),
+    models.Action(name="Update"),
+    models.DeviceType(name="PhysicalMachine"),
+    models.DeviceType(name="VirtualMachine"),
+    models.DeviceType(name="Network"),
+    models.DeviceType(name="MTCA-AMC"),
+    models.DeviceType(name="MTCA-IFC"),
+    models.DeviceType(name="MTCA-MCH"),
+    models.DeviceType(name="MTCA-RTM"),
+    models.DeviceType(name="VME"),
+    models.DeviceType(name="PLC"),
 ]

app/extensions.py

@@ -14,7 +14,6 @@ from flask_login import LoginManager
 from flask_ldap3_login import LDAP3LoginManager
 from flask_sqlalchemy import SQLAlchemy
 from flask_migrate import Migrate
-from flask_bootstrap import Bootstrap
 from flask_admin import Admin
 from flask_mail import Mail
 from flask_jwt_extended import JWTManager
@@ -25,23 +24,22 @@ from flask_caching import Cache
 
 
 convention = {
-    "ix": 'ix_%(column_0_label)s',
+    "ix": "ix_%(column_0_label)s",
     "uq": "uq_%(table_name)s_%(column_0_name)s",
     "ck": "ck_%(table_name)s_%(constraint_name)s",
     "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
-    "pk": "pk_%(table_name)s"
+    "pk": "pk_%(table_name)s",
 }
 
 metadata = MetaData(naming_convention=convention)
-db = SQLAlchemy(metadata=metadata, session_options={'autoflush': False})
+db = SQLAlchemy(metadata=metadata, session_options={"autoflush": False})
 migrate = Migrate(db=db)
 login_manager = LoginManager()
 ldap_manager = LDAP3LoginManager()
-bootstrap = Bootstrap()
-admin = Admin(template_mode='bootstrap3')
+admin = Admin(template_mode="bootstrap3")
 mail = Mail()
 jwt = JWTManager()
 toolbar = DebugToolbarExtension()
-session_redis_store = FlaskRedis(config_prefix='SESSION_REDIS')
+session_redis_store = FlaskRedis(config_prefix="SESSION_REDIS")
 fsession = Session()
 cache = Cache()