.dockerignore

-.git
 **/*.swp
 **/__pycache__
 settings*.cfg
@@ -7,3 +6,8 @@ postgres
 Jenkinsfile
 Makefile
 docker-compose*.yml
+.gitlab-ci.yml
+.cache
+.coverage
+.pytest_cache
+docs/_build

.env

 POSTGRES_USER=ics
 POSTGRES_PASSWORD=icspwd
 POSTGRES_DB=csentry_db
-PGDATA_VOLUME=./data
+PGDATA_VOLUME=./data/postgres
+ELASTIC_DATA_VOLUME=./data/elastic

.flake8

+[flake8]
+# E501: let black handle line length
+# W503 is incompatible with PEP 8
+ignore = E501,W503

.gitignore

@@ -6,3 +6,13 @@ settings*.cfg
 /data
 .cache
 .coverage
+/docs/_build
+.scannerwork
+coverage.xml
+junit.xml
+.pytest_cache
+.tower_cli.cfg
+/app/static/files/*
+!/app/static/files/.empty
+.vscode
+.mypy_cache

.gitlab-ci.yml

+---
+include:
+  - remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/PreCommit.gitlab-ci.yml"
+  - remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/SonarScanner.gitlab-ci.yml"
+
+variables:
+  CONTAINER_TEST_IMAGE: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME"
+  CONTAINER_RELEASE_IMAGE: "$CI_REGISTRY_IMAGE:latest"
+  CONTAINER_CACHE_IMAGE: "$CI_REGISTRY_IMAGE:master"
+  POSTGRES_USER: ics
+  POSTGRES_PASSWORD: icspwd
+  POSTGRES_DB: csentry_db_test
+
+stages:
+  - check
+  - build
+  - test
+  - analyse
+  - release
+  - deploy
+
+default:
+  tags:
+    - docker
+
+before_script:
+  - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+
+build:
+  stage: build
+  image: docker:latest
+  script:
+    - docker pull "$CONTAINER_CACHE_IMAGE" || true
+    - docker build --pull --cache-from "$CONTAINER_CACHE_IMAGE" -t "$CONTAINER_TEST_IMAGE" .
+    - docker push "$CONTAINER_TEST_IMAGE"
+
+test:
+  stage: test
+  image: "$CONTAINER_TEST_IMAGE"
+  services:
+    - postgres:10
+    - redis:4.0
+    - name: docker.elastic.co/elasticsearch/elasticsearch:7.3.2
+      alias: elasticsearch
+      command: ["bin/elasticsearch", "-Ediscovery.type=single-node"]
+  before_script:
+    - pip install -r requirements-dev.txt
+  script:
+    - pytest --junitxml=junit.xml --cov-report=xml:coverage.xml --cov-report=term --cov=app -v
+  artifacts:
+    paths:
+      - junit.xml
+      - coverage.xml
+    reports:
+      junit: junit.xml
+    expire_in: 1 hour
+
+release-image:
+  stage: release
+  image: docker:latest
+  dependencies: []
+  script:
+    - docker pull "$CONTAINER_TEST_IMAGE"
+    - docker tag "$CONTAINER_TEST_IMAGE" "$CONTAINER_RELEASE_IMAGE"
+    - docker push "$CONTAINER_RELEASE_IMAGE"
+  only:
+    - tags
+
+deploy-dev:
+  stage: deploy
+  image: registry.esss.lu.se/ics-docker/tower-cli
+  before_script: []
+  dependencies: []
+  script:
+    - >
+      tower-cli job launch
+      -h torn.tn.esss.lu.se
+      -t ${TOWER_OAUTH_TOKEN}
+      -J deploy-csentry-dev
+      -e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
+  environment:
+    name: dev
+    url: https://csentry-lab-01.cslab.esss.lu.se
+  except:
+    - master
+    - tags
+
+deploy-staging:
+  stage: deploy
+  image: registry.esss.lu.se/ics-docker/tower-cli
+  before_script: []
+  dependencies: []
+  script:
+    - >
+      tower-cli job launch
+      -h torn.tn.esss.lu.se
+      -t ${TOWER_OAUTH_TOKEN}
+      -J deploy-csentry-staging
+      -e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
+  environment:
+    name: staging
+    url: https://csentry-test.esss.lu.se
+  only:
+    - master
+    - tags
+
+pages:
+  stage: deploy
+  image: "$CONTAINER_RELEASE_IMAGE"
+  dependencies: []
+  before_script:
+    - pip install -r requirements-dev.txt
+  script:
+    - sphinx-build -M html docs docs/_build
+    - mv docs/_build/html public
+  artifacts:
+    paths:
+      - public
+  # CI_COMMIT_TAG is used in docs/conf.py
+  # Don't forget to update if not only deploying docs for tags
+  only:
+    - tags
+
+deploy-production:
+  stage: deploy
+  image: registry.esss.lu.se/ics-docker/tower-cli
+  before_script: []
+  dependencies: []
+  script:
+    - >
+      tower-cli job launch
+      -h torn.tn.esss.lu.se
+      -t ${TOWER_OAUTH_TOKEN}
+      -J deploy-csentry
+      -e "csentry_tag=$CI_COMMIT_TAG" --monitor
+  environment:
+    name: production
+    url: https://csentry.esss.lu.se
+  only:
+    - tags
+  when: manual

.pre-commit-config.yaml

+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8

CHANGELOG.rst

+Changelog
+=========
+
+Version 2021.04.06
+------------------
+
+- Validate group names (INFRA-3135)
+- Add warning text for creating vm (INFRA-3292)
+- Paginate history in ansible groups and hosts (INFRA-3136)
+
+Version 2020.11.25
+------------------
+
+- Add documentation about database versioning
+- Set display_name to username when empty (INFRA-2909)
+- Improve logging (INFRA-2899)
+
+Version 2020.11.13
+------------------
+
+- Increased hostname character limit from 20 -> 24 (INFRA-2808)
+- Use SSL by default for LDAP connection (INFRA-2805)
+- Use SonarScanner.gitlab-ci.yml template (INFRA-2832)
+
+Version 2020.10.30
+------------------
+
+- Fix inventory export to excel (INFRA-2780)
+- Add developer documentation
+
+Version 2020.10.27
+------------------
+
+- Update Python and requirements (INFRA-2649)
+- Update RQ and rq-dashboard (INFRA-2649 / INFRA-2754)
+- Fix code smells (INFRA-2729)
+- Add auditor group (INFRA-2728)
+- Update Sphinx and switch to sphinx_rtd_theme (INFRA-2742)
+
+Version 2020.10.06
+------------------
+
+- Make sensitive hosts visible for members of the scope (INFRA-2508)
+- Only hide sensitive networks not part of the user scope (INFRA-2437)
+- Automatically set network_scope group as parent of network groups (INFRA-2639)
+
+Version 2020.07.08
+------------------
+
+- Add view to edit network scope (INFRA-2316)
+- Add API endpoint to patch network scope (INFRA-2316)
+
+Version 2020.06.12
+------------------
+
+- Change MTCA-IOxOS to MTCA-IFC (INFRA-2200)
+- Add description field on Interface (INFRA-2112)
+
+Version 2020.04.16
+------------------
+
+- Allow /network/networks API endpoint for normal users (INFRA-2013)
+- Update Bootstrap to 4.4.1 (INFRA-2021)
+- Update DataTables (INFRA-2021)
+- Add tooltip on search box (INFRA-1893)
+
+Version 2020.03.17
+------------------
+
+- Add extra badges to README and documentation
+- Fix get_hosts for normal users (INFRA-1888)
+
+Version 2020.03.09
+------------------
+
+- Trigger core services update on host change (INFRA-1846)
+- Add DELETE method on all API endpoints (INFRA-1853)
+- Remove create_user API endpoint (INFRA-1854)
+
+Version 2020.03.05
+------------------
+
+- Allow users to view networks (INFRA-1809)
+- Add HOSTNAME Ansible group type (INFRA-1844)
+- Remove IOC git repository creation (INFRA-1839)
+- Make IP optional when creating interface via the API (INFRA-1833)
+- Introduce new device types for MTCA (INFRA-1835)
+
+
+Version 2020.02.25
+------------------
+
+- Fix inventory update (INFRA-1792)
+
+Version 2020.02.24
+------------------
+
+- Add app version in navbar (INFRA-1789)
+- Allow to pass LDAP settings as env variables (INFRA-1788)
+- Implement server side processing for Ansible groups (INFRA-1790)
+
+Version 0.29.0 (2020-02-03)
+---------------------------
+
+- Add MAC address column to hosts table (INFRA-1669)
+- Restrict view of networks and scopes to admin only (INFRA-1671)
+- Add sensitive field to Network class (INFRA-1671)
+- Filter out sensitive hosts for non admin users (INFRA-1671)
+- Add API endpoint and view to update network (INFRA-1724)
+- Make pages title more explicit (INFRA-1726)
+
+Version 0.28.0 (2019-12-19)
+---------------------------
+
+- Fix exception when setting new stack_member (INFRA-1648)
+- Prevent recursive dependency loop in Ansible groups (INFRA-1622)
+- Prevent network and scope overlapping (INFRA-1627)
+- Add netmask to network view (INFRA-1660)
+
+Version 0.27.1 (2019-12-04)
+---------------------------
+
+- Fix regexp for ICS id validation (INFRA-1569)
+- Allow normal users to edit hosts with no interface (INFRA-1604)
+
+Version 0.27.0 (2019-10-30)
+---------------------------
+
+- Increase interface name max length (INFRA-1324)
+- Add endpoints to patch host and interface (INFRA-1406)
+- Limit networks to same scope for extra interfaces (INFRA-1297)
+
+Version 0.26.1 (2019-09-18)
+---------------------------
+
+- Fix inventory update not triggered (INFRA-1290)
+
+Version 0.26.0 (2019-09-18)
+---------------------------
+
+- Decouple inventory and core services update (INFRA-1290)
+- Add broadcast address to network view (INFRA-1291)
+- Add view network scope page (INFRA-1292)
+- Make ansible_vars in host searchable (INFRA-1295)
+- Remove Generate ZTP configuration function (INFRA-1299)
+
+Version 0.25.1 (2019-08-30)
+---------------------------
+
+- Prevent input of invalid Ansible variables (INFRA-1221)
+- Update black and pre-commit config
+- Add pytest.ini to speed test discovery
+
+Version 0.25.0 (2019-08-22)
+---------------------------
+
+- Implement user access on network scope instead of domain (INFRA-1228)
+- Create IOC repositories with same path as name (INFRA-1230)
+
+Version 0.24.1 (2019-07-10)
+---------------------------
+
+- Fix sorting of items with some null stack_member (INFRA-1112)
+
+Version 0.24.0 (2019-07-10)
+---------------------------
+
+- Add serial_number and stack_member to hosts json model (INFRA-1111)
+
+Version 0.23.0 (2019-06-14)
+---------------------------
+
+- Increase RQ default timeout (INFRA-1051)
+
+Version 0.22.0 (2019-06-14)
+---------------------------
+
+- Make vlan optional in NetworkScope and Network (INFRA-1049)
+- Update reverse dependencies of failed tasks (INFRA-1051)
+
+Version 0.21.1 (2019-05-13)
+---------------------------
+
+- Fix users synchronization (INFRA-1026)
+- Remove caching of user retrieval (INFRA-1025)
+
+Version 0.21.0 (2019-05-10)
+---------------------------
+
+- Improve documentation about ansible-vault
+- Fix IOC host indexation (INFRA-1015)
+- Allow users to delete their host (INFRA-1018)
+- Fix job.status DeprecationWarning (INFRA-1021)
+- Allow service users to login (INFRA-1022)
+
+Version 0.20.2 (2019-05-08)
+---------------------------
+
+- Fix IOC repository creation (INFRA-1015)
+- Save ansible var when setting boot profile (INFRA-1016)
+
+Version 0.20.1 (2019-04-25)
+---------------------------
+
+- Save the device_type_id in the session (INFRA-987)
+- Fix exception when no network selected (INFRA-810)
+- Remove graylog support in uwsgi (INFRA-981)
+- Replace raven with sentry-sdk (INFRA-979)
+
+Version 0.20.0 (2019-04-05)
+---------------------------
+
+- Add API endpoint to search hosts (INFRA-931)
+- Initialize IOC repository on GitLab (INFRA-932)
+- Limit the max number of elements returned per page (INFRA-942)
+- Allow to set the boot profile from CSEntry (INFRA-943)
+
+Version 0.19.2 (2019-03-27)
+---------------------------
+
+- Fix model update in admin interface (INFRA-908)
+- Increase timeout for core services update (INFRA-895)
+- Add depends_on field on Task (INFRA-895)
+
+Version 0.19.1 (2019-03-18)
+---------------------------
+
+- Fix excel file download (INFRA-890)
+- Skip post install for windows VM creation (INFRA-877)
+
+Version 0.19.0 (2019-03-18)
+---------------------------
+
+- Add osversion field to CreateVMForm (INFRA-877)
+- Fix link to AWX job id for workflow jobs (INFRA-886)
+- Allow users to create VM/VIOC (INFRA-775)
+- Trigger AWX inventory update on database change (INFRA-887)
+
+Version 0.18.2 (2019-03-07)
+---------------------------
+
+- Fix post install job trigger (INFRA-870)
+
+Version 0.18.1 (2019-03-07)
+---------------------------
+
+- Clean extra vars passed for VM and VIOC creation (INFRA-870)
+
+Version 0.18.0 (2019-03-05)
+---------------------------
+
+- Add gateway field to network table (INFRA-809)
+- Add view network page (INFRA-860)
+- Remove tags and add is_ioc field to host table (INFRA-862)
+- Allow to launch a workflow job (INFRA-867)
+
+Version 0.17.1 (2019-02-07)
+---------------------------
+
+- Catch exception raised by Unique Validator (INFRA-777)
+- Only check the main interface in dynamic groups (INFRA-784)
+- Make sure the first interface is the main one (INFRA-785)
+
+Version 0.17.0 (2019-01-28)
+---------------------------
+
+- Implement new layout (INFRA-763)
+- Fix case insensitive search (INFRA-770)
+- Add post install job after VM creation (INFRA-769)
+
+Version 0.16.0 (2019-01-23)
+---------------------------
+
+- Explicitely define the elasticsearch mapping (INFRA-723)
+- Add favicon (INFRA-725)
+- Pass disk size in inventory when creating VM (INFRA-759)
+
+Version 0.15.1 (2018-12-10)
+---------------------------
+
+- Fix extra interface creation (INFRA-697)
+
+Version 0.15.0 (2018-11-30)
+---------------------------
+
+- Disable DHCP & DNS update on host modification (INFRA-673)
+- Use different groups for inventory and network (INFRA-578)
+- Add network permissions per domain (INFRA-578)
+- Move mac API to inventory endpoint (INFRA-674)
+- Add button to delete Ansible group (INFRA-678)
+- Save VM memory and cores as ansible variables (INFRA-640)
+- Return host fqdn in Ansible groups (INFRA-640)
+- Add extra fields to the API (INFRA-640)
+- Update Python to 3.7.1 and dependencies (INFRA-684)
+- Fix WhiteNoise used static directory (INFRA-683)
+- Fix flask subcommand name (INFRA-687)
+
+Version 0.14.0 (2018-11-08)
+---------------------------
+
+- Fix TypeError in after_commit (INFRA-613)
+- Add zabbix plugin to uwsgi (INFRA-614)
+- Allow to edit and delete comments (INFRA-618)
+- Make mac addresses unique by interface (INFRA-639)
+- Replace bootstrap-select with selectize (INFRA-644)
+- Make host/interface/cname unique (INFRA-245)
+- Switch to new template to deploy VM in proxmox (INFRA-651)
+
+Version 0.13.0 (2018-10-12)
+---------------------------
+
+- Implement full text search for items (INFRA-575)
+- Implement server side processing and full text search for network hosts (INFRA-595)
+- Enable datatables state saving (INFRA-608)
+- Compile uwsgi with graylog2 plugin (INFRA-576)
+- Fix javascript error: Cannot read property '_buttons' of undefined (INFRA-584)
+- Fix LDAPInvalidFilterErrorldap3 (INFRA-550)
+- Fix IndexError when mail is set to an empty list (INFRA-610)
+
+Version 0.12.0 (2018-09-28)
+---------------------------
+
+- Update DataTables with bootstrap 4 theme (INFRA-545)
+- Implement alert flashing from javascript (INFRA-545)
+- Add button to export items to excel file (INFRA-545)
+- Add button to delete hosts (INFRA-557)
+- Allow to delete hosts and interfaces via the API (INFRA-570)
+- Display hosts with no interface on list hosts page (INFRA-569)
+- Ignore old "lost" deferred tasks (INFRA-559)
+
+Version 0.11.4 (2018-09-24)
+---------------------------
+
+- Replace AWX username/password with oauth token for deployment
+- Allow to resize the CodeMirror editor (INFRA-543)
+- Add yaml constructor to support "!vault" tag (INFRA-544)
+- Fix TowerCLIError when ztp_stack is empty (INFRA-547)
+- Fix AttributeError when creating host with group (INFRA-548)
+- Enable sentry user feedback for crash reports
+
+Version 0.11.3 (2018-09-14)
+---------------------------
+
+- Add sentry integration (INFRA-525)
+- Pass sensitive information via environment variables
+
+Version 0.11.2 (2018-09-13)
+---------------------------
+
+- Add versioning on Host and AnsibleGroup (INFRA-500)
+- Disable artifact downloads in release and deploy stages
+
+Version 0.11.1 (2018-08-21)
+---------------------------
+
+- Update tower-cli version for deployment
+- Add host FQDN in host view (INFRA-459)
+- Pass ztp_stack var to trigger_ztp_configuration (INFRA-458)
+- Update Ansible groups documentation
+
+Version 0.11.0 (2018-08-16)
+---------------------------
+
+- Add Ansible parent - child groups
+- Add Ansible dynamic groups
+- Remove flask-bootstrap dependency
+- Update to bootstrap 4.1.3
+- Switch to python-slim image and update requirements
+- Replace jwt decorators with flask-login
+
+Version 0.10.2 (2018-07-23)
+---------------------------
+
+- Allow retrieving interfaces by network name (INFRA-424)
+
+Version 0.10.1 (2018-07-17)
+---------------------------
+
+- Wrap long lines in Ansible groups table
+- Add model field to /networks/hosts endpoint (INFRA-414)
+- Add model field to /networks/interfaces endpoint (INFRA-414)
+- Add trigger ZTP configuration task (INFRA-414)
+
+Version 0.10.0 (2018-07-16)
+---------------------------
+
+- Use sqlalchemy events hook to trigger tasks
+- Add Ansible variables on the host table (INFRA-412)
+- Add Ansible groups table and views (INFRA-412)
+- Add new api endpoint /network/groups (INFRA-412)
+
+
+Version 0.9.0 (2018-07-06)
+--------------------------
+
+- Redirect to the view host page when creating a host (INFRA-402)
+- Save tasks (background jobs) to the database (INFRA-403)
+- Add blueprint to view tasks (INFRA-403)
+- Redirect to the task view page when creating a VM
+- Display host creation date and user on view host page
+- Use black for source code formatting
+- Add pre-commit hooks for code formatting and linting
+- Update documentation
+
+Version 0.8.1
+-------------
+
+- Fix VM creation (memory shall be passed in MB)
+
+Version 0.8.0
+-------------
+
+- Hide interface name in create host form (INFRA-287)
+- Allow to link several items to one host (INFRA-267)
+- Add stack_member field to item (INFRA-267)
+- Add extra device types (INFRA-302)
+- Add IOC tag (INFRA-302)
+- Select the IOC tag by default based on device type (INFRA-302)
+- Use bootstrap-select for tags selection
+- Use the last IP as network gateway (INFRA-339)
+- Fix default selected tags in edit interface view (INFRA-344)
+- Use CamelCase for CSEntry device types and tags (INFRA-334)
+- Add RQ to process jobs in the background
+- Add RQ Dashboard blueprint (admin only)
+- Automatically trigger update of TN core services on host or interface change
+- Add Create VM button (admin only for now)
+- Allow to easily identify staging server (INFRA-347)
+- Update documentation
+
+Version 0.7.0
+-------------
+
+- Add search in model description from inventory page
+- Add model description to view item (INFRA-280)
+- Add device_type table (INFRA-281)
+- Add device_type to interfaces API endpoint (INFRA-277)
+- Add attributes favorites page (INFRA-283)
+- Sort network names in the register new host form (INFRA-284)
+- Clear error messages on form fields on change (INFRA-285)
+
+Version 0.6.8
+-------------
+
+- return username instead of display name in the API (INFRA-241)
+- add /network/cnames API endpoint
+- update default available IPs range
+
+Version 0.6.7
+-------------
+
+- accept a list of LDAP groups
+
+Version 0.6.6
+-------------
+
+- add link to documentation
+- add Action tab next to Attributes
+- switch to centos-miniconda3 base image
+
+Version 0.6.5
+-------------
+
+- add /network/domains API endpoint
+- add interfaces filtering by domain
+- add netmask field in network json representation
+- increase QRCode size
+
+Version 0.6.4
+-------------
+
+- add API endpoint to retrieve the current user profile
+- add Submit action QRCode to create an item
+- update scanner instructions
+- update documentation
+
+Version 0.6.3
+-------------
+
+- add CHANGELOG
+- add sphinx dependency to build documentation
+- add stage to deploy documentation to GitLab pages
+- move QR codes to attributes table
+- improve documentation
+- add manual job to deploy to production

Dockerfile

-FROM continuumio/miniconda3:latest
+FROM python:3.8-slim as base
 
-RUN groupadd -r ics && useradd -r -g ics ics
+# Add ESS specific debian repository mirror
+RUN echo "deb https://artifactory.esss.lu.se/artifactory/debian-mirror stable main" | tee /etc/apt/sources.list.d/ess-debian-mirror.list
 
-WORKDIR /app
+# Install Python dependencies in an intermediate image
+# as some requires a compiler (psycopg2)
+FROM base as builder
+
+# Install dependencies required to compile some Python packages
+# Taken from https://github.com/docker-library/python/blob/master/3.6/stretch/slim/Dockerfile
+# For psycopg2: libpq-dev
+# For pillow: libjpeg-dev libpng-dev libtiff-dev
+RUN apt-get update \
+  && apt-get install -yq --no-install-recommends \
+    gcc \
+    libbz2-dev \
+    libc6-dev \
+    libexpat1-dev \
+    libffi-dev \
+    libjpeg-dev \
+    libgdbm-dev \
+    liblzma-dev \
+    libncursesw5-dev \
+    libpcre3-dev \
+    libpng-dev \
+    libpq-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libssl-dev \
+    libtiff-dev \
+    make \
+    tk-dev \
+    wget \
+    xz-utils \
+    zlib1g-dev \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /requirements.txt
+RUN python -m venv /venv \
+  && . /venv/bin/activate \
+  && pip install --no-cache-dir --upgrade pip \
+  && pip install --no-cache-dir -r /requirements.txt
+
+ARG CSENTRY_BUILD
+COPY requirements-dev.txt /requirements-dev.txt
+RUN if [ "$CSENTRY_BUILD" = "DEV" ] ; then /venv/bin/pip install --no-cache-dir -r /requirements-dev.txt ; fi
+
+FROM base
 
-# Install CSEntry requirements
-COPY environment.yml /app/environment.yml
-RUN conda config --add channels conda-forge \
-    && conda env create -n csentry -f environment.yml \
-    && rm -rf /opt/conda/pkgs/*
+RUN groupadd -r -g 1000 csi \
+  && useradd --no-log-init -r -g csi -u 1000 csi
+
+COPY --chown=csi:csi --from=builder /venv /venv
+
+# Install libraries for psycopg2 and pillow
+# Shall be the same as the one linked to when compiling in builder image!
+RUN apt-get update \
+  && apt-get install -yq --no-install-recommends \
+    libjpeg62-turbo \
+    libpng16-16 \
+    libtiff5 \
+    libpq5 \
+    libpcre3 \
+    zlib1g \
+    git \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN rm /etc/apt/sources.list.d/ess-debian-mirror.list
+
+COPY --chown=csi:csi . /app/
+WORKDIR /app
+RUN echo "__version__ = \"$(git describe)\"" > app/_version.py
 
-# Install the app
-COPY . /app/
-RUN chown -R ics:ics /app/*
+ENV PATH /venv/bin:$PATH
+EXPOSE 8000
+CMD ["flask", "run", "--host", "0.0.0.0", "--port", "8000"]
 
-# activate the csentry environment
-ENV PATH /opt/conda/envs/csentry/bin:$PATH
+USER csi

Jenkinsfile

-pipeline {
-    agent { label 'docker-compose' }
-
-    environment {
-        GIT_TAG = sh(returnStdout: true, script: 'git describe --exact-match || true').trim()
-    }
-
-    stages {
-        stage('Refresh') {
-            steps {
-                slackSend (color: 'good', message: "STARTED: <${env.BUILD_URL}|${env.JOB_NAME} [${env.BUILD_NUMBER}]>")
-                sh 'make clean'
-                sh 'make refresh'
-            }
-        }
-        stage('Build') {
-            steps {
-                ansiColor('xterm') {
-                    sh 'make build'
-                }
-            }
-        }
-        stage('Test') {
-            steps {
-                sh 'make db_image'
-                /* let the time to postgres to start */
-                sh 'sleep 5'
-                sh 'make test_image'
-            }
-        }
-        stage('Push') {
-            when {
-                not { environment name: 'GIT_TAG', value: '' }
-            }
-            steps {
-                sh 'make tag'
-                sh 'make push'
-            }
-        }
-    }
-
-    post {
-        always {
-            sh 'make clean'
-            /* clean up the workspace */
-            deleteDir()
-        }
-        failure {
-            slackSend (color: 'danger', message: "FAILED: <${env.BUILD_URL}|${env.JOB_NAME} [${env.BUILD_NUMBER}]>")
-        }
-        success {
-            slackSend (color: 'good', message: "SUCCESSFUL: <${env.BUILD_URL}|${env.JOB_NAME} [${env.BUILD_NUMBER}]>")
-        }
-    }
-
-}

Makefile

-.PHONY: help build tag push refresh release db initdb test db_image test_image
+.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image docs
 
-OWNER := europeanspallationsource
+OWNER := registry.esss.lu.se/ics-infrastructure
 GIT_TAG := $(shell git describe --always)
 IMAGE := csentry
 
@@ -38,18 +38,28 @@ release: refresh \
 release: ## build, tag, and push all stacks
 
 db: ## start postgres and redis for development
-	docker-compose up -d postgres redis
+	docker-compose up -d postgres redis elasticsearch worker
 
-initdb: ## initialize the dev database
-	docker-compose run --rm web flask initdb
+init_db: ## initialize the dev database
+	docker-compose run --rm web flask db upgrade head
+	docker-compose run --rm web flask create-defaults
+
+upgrade_db: ## upgrade the dev database
+	docker-compose run --rm web flask db upgrade head
 
 test:  ## run the tests (on current directory)
-	docker-compose run --rm web pytest --cov=app -v
+	docker-compose -f docker-compose.yml run --rm web pytest --cov=app -v
 
-db_image: ## start postgres and redis to test the latest image
-# Pass docker-compose.yml to skip docker-compose.override.yml
-	docker-compose -f docker-compose.yml up -d postgres redis
+db_test: ## start required containers for test
+# Pass docker-compose.yml to skip docker-compose.override.yml (db not mounted as volume for speed)
+	docker-compose -f docker-compose.yml up -d postgres redis elasticsearch worker
 
 test_image:  ## run the tests (on the latest image)
 # Pass docker-compose.yml to skip docker-compose.override.yml
 	docker-compose -f docker-compose.yml run --rm web
+
+run_uwsgi:  ## run the application with uwsgi (to test prod env)
+	docker-compose run -p 8000:8000 --rm web uwsgi --master --http 0.0.0.0:8000 --http-keepalive --manage-script-name --mount /csentry=wsgi.py --callable app --uid conda --processes 2 -b 16384
+
+docs:  ## run the tests (on current directory)
+	docker run --rm -v $(shell pwd):/app registry.esss.lu.se/ics-infrastructure/csentry:master sphinx-build -M html docs docs/_build

README.rst

 CSEntry
 =======
 
+.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
+    :target: https://github.com/ambv/black
+
+.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=alert_status
+    :target: https://sonarqube.esss.lu.se/dashboard?id=csentry
+
+.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=ncloc
+    :target: https://sonarqube.esss.lu.se/dashboard?id=csentry
+
+.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/pipeline.svg
+
+.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/coverage.svg
+
 Control System Entry web server.
 
 
@@ -19,9 +32,10 @@ You can use docker for development:
     or
     $ make db
     # Initialize the database
-    $ docker-compose run --rm web flask initdb
+    $ docker-compose run --rm web flask db upgrade head
+    $ docker-compose run --rm web flask create_defaults
     or
-    $ make initdb
+    $ make init_db
 
 3. Start the application::
 
@@ -69,23 +83,10 @@ Backup & restore
 To dump the database::
 
     $ docker run --rm --link csentry_postgres:postgres --net csentry_default -e PGPASSWORD="<csentry_password>"
-      postgres:10 pg_dump -h postgres -U csentry csentry_db | gzip > csentry_db.dump.gz
+      postgres:10 pg_dump -h postgres -U ics csentry_db | gzip > csentry_db.sql.gz
 
 
 To restore the database::
 
-    $ gunzip -c csentry_db.dump.g | docker run --rm --link csentry_postgres:postgres --net csentry_default
-      -e PGPASSWORD="<csentry_password>" -i postgres:10 psql -h postgres -U csentry csentry_db
-
-
-Dependencies
-------------
-
-The initial dependencies were generated using::
-
-    $ docker run --rm -it -v $(pwd):/app continuumio/miniconda3:latest bash
-    $ conda config --add channels conda-forge
-    $ conda create -n csentry python=3.6 flask alembic flask-debugtoolbar flask-login flask-sqlalchemy flask-wtf pillow psycopg2 pytest pytest-cov qrcode whitenoise factory_boy flask-admin pyjwt ldap3 flask-mail flask-migrate flask-jwt-extended
-    $ source activate csentry
-    $ pip install flask-ldap3-login sqlalchemy-citext sqlalchemy-continuum pytest-factoryboy git+https://github.com/beenje/flask-bootstrap@4.0.0-beta.1.dev1
-    $ conda env export > /app/environment.yml
+    $ gunzip -c csentry_db.sql.gz | docker run --rm --link csentry_postgres:postgres --net csentry_default
+      -e PGPASSWORD="<csentry_password>" -i postgres:10 psql -h postgres -U ics csentry_db

app/_version.py

+__version__ = "dev"

app/admin/views.py

@@ -28,14 +28,13 @@ sqla.form.Unique.__call__ = lambda x, y, z: None
 # Add custom model converter for CIText type
 # See https://github.com/flask-admin/flask-admin/issues/1196
 class AppAdminModelConverter(sqla.form.AdminModelConverter):
-
-    @converts('CIText')
+    @converts("CIText")
     def conv_CIText(self, field_args, **extra):
         return fields.TextAreaField(**field_args)
 
-    @converts('sqlalchemy.dialects.postgresql.base.CIDR')
+    @converts("sqlalchemy.dialects.postgresql.base.CIDR")
     def conv_PGCidr(self, field_args, **extra):
-        field_args['validators'].append(IPNetwork())
+        field_args["validators"].append(IPNetwork())
         return fields.StringField(**field_args)
 
 
@@ -43,20 +42,12 @@ class AdminModelView(sqla.ModelView):
     model_form_converter = AppAdminModelConverter
 
     # Replace TextAreaField (default for Text) with StringField
-    form_overrides = {
-        'name': fields.StringField,
-    }
+    form_overrides = {"name": fields.StringField}
 
     def is_accessible(self):
         return current_user.is_authenticated and current_user.is_admin
 
 
-class GroupAdmin(AdminModelView):
-    can_create = False
-    can_edit = False
-    can_delete = False
-
-
 class UserAdmin(AdminModelView):
     can_create = False
     can_edit = False
@@ -71,16 +62,17 @@ class TokenAdmin(AdminModelView):
 class ItemAdmin(AdminModelView):
 
     # Replace TextAreaField (default for Text) with StringField
-    form_overrides = {
-        'ics_id': fields.StringField,
-        'serial_number': fields.StringField,
-    }
+    form_overrides = {"ics_id": fields.StringField, "serial_number": fields.StringField}
 
     form_args = {
-        'ics_id': {
-            'label': 'ICS id',
-            'validators': [validators.Regexp(ICS_ID_RE, message='ICS id shall match [A-Z]{3}[0-9]{3}')],
-            'filters': [lambda x: x or None],
+        "ics_id": {
+            "label": "ICS id",
+            "validators": [
+                validators.Regexp(
+                    ICS_ID_RE, message="ICS id shall match [A-Z]{3}[0-9]{3}"
+                )
+            ],
+            "filters": [lambda x: x or None],
         }
     }
 
@@ -88,12 +80,9 @@ class ItemAdmin(AdminModelView):
 class NetworkAdmin(AdminModelView):
 
     # Replace TextAreaField (default for Text) with StringField
-    form_overrides = {
-        'vlan_name': fields.StringField,
-    }
+    form_overrides = {"vlan_name": fields.StringField}
 
-    form_args = {
-        'gateway': {
-            'filters': [lambda x: x or None],
-        },
-    }
+
+class TaskAdmin(AdminModelView):
+    column_display_pk = True
+    can_create = False

app/api/inventory.py

@@ -10,12 +10,12 @@ This module implements the inventory API.
 
 """
 from flask import Blueprint, jsonify, request, current_app
-from flask_jwt_extended import jwt_required
+from flask_login import login_required
 from .. import utils, models
-from ..decorators import jwt_groups_accepted
-from .utils import commit, create_generic_model, get_generic_model
+from ..decorators import login_groups_accepted
+from .utils import commit, create_generic_model, get_generic_model, delete_generic_model
 
-bp = Blueprint('inventory_api', __name__)
+bp = Blueprint("inventory_api", __name__)
 
 
 def get_item_by_id_or_ics_id(id_):
@@ -32,72 +32,99 @@ def get_item_by_id_or_ics_id(id_):
     return item
 
 
-@bp.route('/items')
-@jwt_required
+@bp.route("/items")
+@login_required
 def get_items():
-    # TODO: add pagination
-    return get_generic_model(models.Item, request.args,
-                             order_by=models.Item.created_at)
+    """Return items
+
+    .. :quickref: Inventory; Get items
+    """
+    return get_generic_model(models.Item, order_by=models.Item.created_at)
 
 
-@bp.route('/items/<id_>')
-@jwt_required
+@bp.route("/items/<id_>")
+@login_required
 def get_item(id_):
-    """Retrieve item by id or ICS id"""
+    r"""Retrieve item by id or ICS id
+
+    .. :quickref: Inventory; Get item by id or ICS id
+
+    :param id\_: ICS id or primary key of the item
+    """
     item = get_item_by_id_or_ics_id(id_)
     return jsonify(item.to_dict())
 
 
-@bp.route('/items', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/items", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_item():
-    """Register a new item"""
+    """Register a new item
+
+    .. :quickref: Inventory; Create new item
+
+    :jsonparam serial_number: serial number
+    :jsonparam ics_id: (optional) ICS id (temporary one generated if not present)
+    :jsonparam quantity: (optional) number of items [default: 1]
+    :jsonparam manufacturer: (optional) name of the manufacturer
+    :jsonparam model: (optional) name of the model
+    :jsonparam location: (optional) name of the location
+    :jsonparam status: (optional) name of the status
+    :jsonparam parent_id: (optional) parent id
+    :jsonparam host_id: (optional) host id
+    """
     # People should assign an ICS id to a serial number when creating
     # an item so ics_id should also be a mandatory field.
     # But there are existing items (in confluence and JIRA) that we want to
     # import and associate after they have been created.
     # In that case a temporary id is automatically assigned.
-    return create_generic_model(models.Item, mandatory_fields=('serial_number',))
+    return create_generic_model(models.Item, mandatory_fields=("serial_number",))
 
 
-@bp.route('/items/<id_>', methods=['PATCH'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/items/<id_>", methods=["PATCH"])
+@login_groups_accepted("admin", "inventory")
 def patch_item(id_):
-    """Patch an existing item
+    r"""Patch an existing item
 
-    id_ can be the primary key or the ics_id field
-    Fields allowed to update are:
-        - ics_id ONLY if current is temporary (422 returned otherwise)
-        - manufacturer
-        - model
-        - location
-        - status
-        - parent
+    .. :quickref: Inventory; Update existing item
 
-    422 is returned if other fields are given.
+    :param id\_: ICS id or primary key of the item
+    :jsonparam ics_id: ICS id - only allowed if the current one is temporary
+    :jsonparam manufacturer: Item's manufacturer
+    :jsonparam model: Item's model
+    :jsonparam location: Item's location
+    :jsonparam status: Item's status
+    :jsonparam parent: Item's parent
     """
     data = request.get_json()
     if data is None:
-        raise utils.CSEntryError('Body should be a JSON object')
+        raise utils.CSEntryError("Body should be a JSON object")
     if not data:
-        raise utils.CSEntryError('At least one field is required', status_code=422)
+        raise utils.CSEntryError("At least one field is required", status_code=422)
     for key in data.keys():
-        if key not in ('ics_id', 'manufacturer', 'model',
-                       'location', 'status', 'parent'):
+        if key not in (
+            "ics_id",
+            "manufacturer",
+            "model",
+            "location",
+            "status",
+            "parent",
+        ):
             raise utils.CSEntryError(f"Invalid field '{key}'", status_code=422)
     item = get_item_by_id_or_ics_id(id_)
     # Only allow to set ICS id if the current id is a temporary one
-    if item.ics_id.startswith(current_app.config['TEMPORARY_ICS_ID']):
-        item.ics_id = data.get('ics_id', item.ics_id)
-    elif 'ics_id' in data:
+    if item.ics_id.startswith(current_app.config["TEMPORARY_ICS_ID"]):
+        item.ics_id = data.get("ics_id", item.ics_id)
+    elif "ics_id" in data:
         raise utils.CSEntryError("'ics_id' can't be changed", status_code=422)
-    item.manufacturer = utils.convert_to_model(data.get('manufacturer', item.manufacturer), models.Manufacturer)
-    item.model = utils.convert_to_model(data.get('model', item.model), models.Model)
-    item.location = utils.convert_to_model(data.get('location', item.location), models.Location)
-    item.status = utils.convert_to_model(data.get('status', item.status), models.Status)
-    parent_ics_id = data.get('parent')
+    item.manufacturer = utils.convert_to_model(
+        data.get("manufacturer", item.manufacturer), models.Manufacturer
+    )
+    item.model = utils.convert_to_model(data.get("model", item.model), models.Model)
+    item.location = utils.convert_to_model(
+        data.get("location", item.location), models.Location
+    )
+    item.status = utils.convert_to_model(data.get("status", item.status), models.Status)
+    parent_ics_id = data.get("parent")
     if parent_ics_id is not None:
         parent = models.Item.query.filter_by(ics_id=parent_ics_id).first()
         if parent is not None:
@@ -113,76 +140,227 @@ def patch_item(id_):
     return jsonify(item.to_dict())
 
 
-@bp.route('/items/<id_>/comments')
-@jwt_required
+@bp.route("/items/<int:item_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_item(item_id):
+    """Delete an item
+
+    .. :quickref: Inventory; Delete an item
+
+    :param item_id: item primary key
+    """
+    return delete_generic_model(models.Item, item_id)
+
+
+@bp.route("/items/<id_>/comments")
+@login_required
 def get_item_comments(id_):
+    r"""Get item comments
+
+    .. :quickref: Inventory; Get item comments
+
+    :param id\_: ICS id or primary key of the item
+    """
     item = get_item_by_id_or_ics_id(id_)
     return jsonify([comment.to_dict() for comment in item.comments])
 
 
-@bp.route('/items/<id_>/comments', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/items/<id_>/comments", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_item_comment(id_):
+    r"""Create a comment on item
+
+    .. :quickref: Inventory; Create comment on item
+
+    :param id\_: ICS id or primary key of the item
+    :jsonparam body: comment body
+    """
     item = get_item_by_id_or_ics_id(id_)
-    return create_generic_model(models.ItemComment,
-                                mandatory_fields=('body',),
-                                item_id=item.id)
+    return create_generic_model(
+        models.ItemComment, mandatory_fields=("body",), item_id=item.id
+    )
+
 
+@bp.route("/items/comments/<int:comment_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_comment(comment_id):
+    """Delete an item comment
 
-@bp.route('/actions')
-@jwt_required
+    .. :quickref: Inventory; Delete an item comment
+
+    :param comment_id: comment primary key
+    """
+    return delete_generic_model(models.ItemComment, comment_id)
+
+
+@bp.route("/actions")
+@login_required
 def get_actions():
-    return get_generic_model(models.Action, request.args)
+    """Get actions
 
+    .. :quickref: Inventory; Get actions
+    """
+    return get_generic_model(models.Action)
 
-@bp.route('/manufacturers')
-@jwt_required
+
+@bp.route("/manufacturers")
+@login_required
 def get_manufacturers():
-    return get_generic_model(models.Manufacturer, request.args)
+    """Get manufacturers
+
+    .. :quickref: Inventory; Get manufacturers
+    """
+    return get_generic_model(models.Manufacturer)
 
 
-@bp.route('/manufacturers', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/manufacturers", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_manufacturer():
+    """Create a new manufacturer
+
+    .. :quickref: Inventory; Create new manufacturer
+
+    :jsonparam name: manufacturer name
+    :jsonparam description: (optional) description
+    """
     return create_generic_model(models.Manufacturer)
 
 
-@bp.route('/models')
-@jwt_required
+@bp.route("/manufacturers/<int:manufacturer_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_manufacturer(manufacturer_id):
+    """Delete a manufacturer
+
+    .. :quickref: Inventory; Delete a manufacturer
+
+    :param manufacturer_id: manufacturer primary key
+    """
+    return delete_generic_model(models.Manufacturer, manufacturer_id)
+
+
+@bp.route("/models")
+@login_required
 def get_models():
-    return get_generic_model(models.Model, request.args)
+    """Get models
+
+    .. :quickref: Inventory; Get models
+    """
+    return get_generic_model(models.Model)
 
 
-@bp.route('/models', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/models", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_model():
+    """Create a new model
+
+    .. :quickref: Inventory; Create new model
+
+    :jsonparam name: model name
+    :jsonparam description: (optional) description
+    """
     return create_generic_model(models.Model)
 
 
-@bp.route('/locations')
-@jwt_required
+@bp.route("/models/<int:model_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_model(model_id):
+    """Delete a model
+
+    .. :quickref: Inventory; Delete a model
+
+    :param model_id: model primary key
+    """
+    return delete_generic_model(models.Model, model_id)
+
+
+@bp.route("/locations")
+@login_required
 def get_locations():
-    return get_generic_model(models.Location, request.args)
+    """Get locations
+
+    .. :quickref: Inventory; Get locations
+    """
+    return get_generic_model(models.Location)
 
 
-@bp.route('/locations', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+@bp.route("/locations", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_locations():
+    """Create a new location
+
+    .. :quickref: Inventory; Create new location
+
+    :jsonparam name: location name
+    :jsonparam description: (optional) description
+    """
     return create_generic_model(models.Location)
 
 
-@bp.route('/status')
-@jwt_required
+@bp.route("/locations/<int:location_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_location(location_id):
+    """Delete a location
+
+    .. :quickref: Inventory; Delete a location
+
+    :param location_id: location primary key
+    """
+    return delete_generic_model(models.Location, location_id)
+
+
+@bp.route("/statuses")
+@login_required
 def get_status():
-    return get_generic_model(models.Status, request.args)
+    """Get statuses
 
+    .. :quickref: Inventory; Get statuses
+    """
+    return get_generic_model(models.Status)
 
-@bp.route('/status', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+
+@bp.route("/statuses", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
 def create_status():
+    """Create a new status
+
+    .. :quickref: Inventory; Create new status
+
+    :jsonparam name: status name
+    :jsonparam description: (optional) description
+    """
     return create_generic_model(models.Status)
+
+
+@bp.route("/statuses/<int:status_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_status(status_id):
+    """Delete a status
+
+    .. :quickref: Inventory; Delete a status
+
+    :param status_id: status primary key
+    """
+    return delete_generic_model(models.Status, status_id)
+
+
+@bp.route("/macs")
+@login_required
+def get_macs():
+    """Return mac addresses
+
+    .. :quickref: Inventory; Get mac addresses
+    """
+    return get_generic_model(models.Mac, order_by=models.Mac.address)
+
+
+@bp.route("/macs", methods=["POST"])
+@login_groups_accepted("admin", "inventory")
+def create_macs():
+    """Create a new mac address
+
+    .. :quickref: Inventory; Create new mac address
+
+    :jsonparam address: MAC address
+    :jsonparam item_id: (optional) linked item primary key
+    """
+    return create_generic_model(models.Mac, mandatory_fields=("address",))

app/api/network.py

@@ -10,59 +10,543 @@ This module implements the network API.
 
 """
 from flask import Blueprint, request
-from flask_jwt_extended import jwt_required
+from flask_login import login_required, current_user
+from wtforms import ValidationError
 from .. import models
-from ..decorators import jwt_groups_accepted
-from .utils import get_generic_model, create_generic_model
+from ..decorators import login_groups_accepted
+from ..utils import CSEntryError, validate_ip
+from . import utils
 
-bp = Blueprint('network_api', __name__)
+bp = Blueprint("network_api", __name__)
 
 
-@bp.route('/scopes')
-@jwt_required
+@bp.route("/scopes")
+@login_groups_accepted("admin", "auditor")
 def get_scopes():
-    # TODO: add pagination
-    return get_generic_model(models.NetworkScope, request.args,
-                             order_by=models.NetworkScope.name)
+    """Return network scopes
 
+    .. :quickref: Network; Get network scopes
+    """
+    return utils.get_generic_model(
+        models.NetworkScope, order_by=models.NetworkScope.name
+    )
 
-@bp.route('/scopes', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin')
+
+@bp.route("/scopes", methods=["POST"])
+@login_groups_accepted("admin")
 def create_scope():
-    """Create a new network scope"""
-    return create_generic_model(models.NetworkScope, mandatory_fields=(
-        'name', 'first_vlan', 'last_vlan', 'supernet'))
+    """Create a new network scope
+
+    .. :quickref: Network; Create new network scope
+
+    :jsonparam name: network scope name
+    :jsonparam first_vlan: network scope first vlan
+    :jsonparam last_vlan: network scope last vlan
+    :jsonparam supernet: network scope supernet
+    :jsonparam domain_id: primary key of the default domain
+    :jsonparam description: (optional) description
+    """
+    return utils.create_generic_model(
+        models.NetworkScope,
+        mandatory_fields=("name", "first_vlan", "last_vlan", "supernet", "domain_id"),
+    )
+
+
+@bp.route("/scopes/<int:scope_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_scope(scope_id):
+    """Delete a network scope
+
+    .. :quickref: Network; Delete a network scope
+
+    :param scope_id: network scope primary key
+    """
+    return utils.delete_generic_model(models.NetworkScope, scope_id)
+
+
+@bp.route("/scopes/<int:scope_id>", methods=["PATCH"])
+@login_groups_accepted("admin")
+def patch_scope(scope_id):
+    r"""Patch an existing network scope
+
+    .. :quickref: Network; Update an existing network scope
+
+    :param scope_id: network scope primary key
+    :jsonparam name: network scope name
+    :jsonparam description: description
+    :jsonparam first_vlan: network scope first vlan
+    :jsonparam last_vlan: network scope last vlan
+    :jsonparam supernet: network scope supernet
+    :jsonparam domain: name of the default domain
+    """
+    allowed_fields = (
+        ("name", str, None),
+        ("description", str, None),
+        ("first_vlan", int, None),
+        ("last_vlan", int, None),
+        ("supernet", str, None),
+        ("domain", models.Domain, "name"),
+    )
+    return utils.update_generic_model(models.NetworkScope, scope_id, allowed_fields)
 
 
-@bp.route('/networks')
-@jwt_required
+@bp.route("/networks")
+@login_groups_accepted("admin", "auditor", "network")
 def get_networks():
-    # TODO: add pagination
-    return get_generic_model(models.Network, request.args,
-                             order_by=models.Network.address)
+    """Return networks
 
+    .. :quickref: Network; Get networks
+    """
+    query = models.Network.query
+    if not (current_user.is_admin or current_user.is_auditor):
+        sensitive_networks = (
+            query.filter(models.Network.sensitive.is_(True))
+            .join(models.Network.scope)
+            .filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
+        )
+        query = (
+            query.filter(models.Network.sensitive.is_(False))
+            .union(sensitive_networks)
+            .distinct(models.Network.id)
+            .from_self()
+        )
+    query = query.order_by(models.Network.address)
+    return utils.get_generic_model(model=models.Network, base_query=query)
 
-@bp.route('/networks', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin')
+
+@bp.route("/networks", methods=["POST"])
+@login_groups_accepted("admin")
 def create_network():
-    """Create a new network"""
-    return create_generic_model(models.Network, mandatory_fields=(
-        'vlan_name', 'vlan_id', 'address', 'first_ip', 'last_ip', 'scope'))
+    """Create a new network
+
+    .. :quickref: Network; Create new network
+
+    :jsonparam vlan_name: vlan name
+    :jsonparam vlan_id: vlan id
+    :jsonparam address: vlan address
+    :jsonparam first_ip: first IP of the allowed range
+    :jsonparam last_ip: last IP of the allowed range
+    :jsonparam gateway: gateway IP
+    :jsonparam scope: network scope name
+    :jsonparam domain_id: (optional) primary key of the domain [default: scope domain]
+    :jsonparam admin_only: (optional) boolean to restrict the network to admin users [default: False]
+    :type admin_only: bool
+    :jsonparam sensitive: hide the network and all hosts if True (for non admin) [default: False]
+    :type sensitive: bool
+    :jsonparam description: (optional) description
+    """
+    return utils.create_generic_model(
+        models.Network,
+        mandatory_fields=(
+            "vlan_name",
+            "vlan_id",
+            "address",
+            "first_ip",
+            "last_ip",
+            "gateway",
+            "scope",
+        ),
+    )
+
+
+@bp.route("/networks/<int:network_id>", methods=["PATCH"])
+@login_groups_accepted("admin")
+def patch_network(network_id):
+    r"""Patch an existing network
+
+    .. :quickref: Network; Update an existing network
+
+    :param network_id: network primary key
+    :jsonparam vlan_name: vlan name
+    :jsonparam address: vlan address
+    :jsonparam first_ip: first IP of the allowed range
+    :jsonparam last_ip: last IP of the allowed range
+    :jsonparam gateway: gateway IP
+    :jsonparam domain: domain name
+    :jsonparam admin_only: boolean to restrict the network to admin users
+    :type admin_only: bool
+    :jsonparam sensitive: hide the network and all hosts if True (for non admin)
+    :type sensitive: bool
+    :jsonparam description: description
+    """
+    # The method currently doesn't allow to update the network_scope
+    # as this will have an impact on the address
+    allowed_fields = (
+        ("vlan_name", str, None),
+        ("address", str, None),
+        ("first_ip", str, None),
+        ("last_ip", str, None),
+        ("gateway", str, None),
+        ("domain", models.Domain, "name"),
+        ("admin_only", bool, None),
+        ("sensitive", bool, None),
+        ("description", str, None),
+    )
+    return utils.update_generic_model(models.Network, network_id, allowed_fields)
+
+
+@bp.route("/networks/<int:network_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_network(network_id):
+    """Delete a network
+
+    .. :quickref: Network; Delete a network
+
+    :param network_id: network primary key
+    """
+    return utils.delete_generic_model(models.Network, network_id)
 
 
-@bp.route('/interfaces')
-@jwt_required
+@bp.route("/interfaces")
+@login_required
 def get_interfaces():
-    # TODO: add pagination
-    return get_generic_model(models.Interface, request.args,
-                             order_by=models.Interface.ip)
+    """Return interfaces
 
+    .. :quickref: Network; Get interfaces
+    """
+    query = models.Interface.query
+    query = query.join(models.Interface.network).order_by(models.Interface.ip)
+    if not (current_user.is_admin or current_user.is_auditor):
+        sensitive_interfaces = (
+            query.filter(models.Network.sensitive.is_(True))
+            .join(models.Network.scope)
+            .filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
+        )
+        query = (
+            query.filter(models.Network.sensitive.is_(False))
+            .union(sensitive_interfaces)
+            .from_self()
+            .join(models.Interface.network)
+            .order_by(models.Interface.ip)
+        )
+    domain = request.args.get("domain", None)
+    if domain is not None:
+        query = query.join(models.Network.domain).filter(models.Domain.name == domain)
+        return utils.get_generic_model(model=None, query=query)
+    network = request.args.get("network", None)
+    if network is not None:
+        query = query.filter(models.Network.vlan_name == network)
+        return utils.get_generic_model(model=None, query=query)
+    return utils.get_generic_model(model=models.Interface, base_query=query)
 
-@bp.route('/interfaces', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin', 'create')
+
+@bp.route("/interfaces", methods=["POST"])
+@login_groups_accepted("admin", "network")
 def create_interface():
-    """Create a new interface"""
-    return create_generic_model(models.Interface, mandatory_fields=('network', 'ip', 'name'))
+    """Create a new interface
+
+    .. :quickref: Network; Create new interface
+
+    :jsonparam network: network name
+    :jsonparam ip: (optional) interface IP - IP will be assigned automatically if not given
+    :jsonparam name: interface name
+    :jsonparam host: host name
+    :jsonparam mac: (optional) MAC address
+    """
+    # The validate_interfaces method from the Network class is called when
+    # setting interface.network. This is why we don't pass network_id here
+    # but network (as vlan_name string)
+    # Same for host
+    data = request.get_json()
+    # Check that the user has the permissions to create an interface on this network
+    try:
+        network = models.Network.query.filter_by(
+            vlan_name=data["network"]
+        ).first_or_404()
+    except Exception:
+        # If we can't get a network, an error will be raised in create_generic_model
+        pass
+    else:
+        if not current_user.has_access_to_network(network):
+            raise CSEntryError("User doesn't have the required group", status_code=403)
+    return utils.create_generic_model(
+        models.Interface, mandatory_fields=("network", "name", "host")
+    )
+
+
+@bp.route("/interfaces/<int:interface_id>", methods=["PATCH"])
+@login_groups_accepted("admin", "network")
+def patch_interface(interface_id):
+    r"""Patch an existing interface
+
+    .. :quickref: Network; Update an existing interface
+
+    :param interface_id: interface primary key
+    :jsonparam ip: interface IP
+    :jsonparam name: interface name
+    :jsonparam network: network name
+    :jsonparam mac: MAC address
+    """
+    interface = models.Interface.query.get_or_404(interface_id)
+    # User shall have access to both the current and new network (if provided)
+    if not current_user.has_access_to_network(interface.network):
+        raise CSEntryError("User doesn't have the required group", status_code=403)
+    data = utils.get_json_body()
+    if "network" in data:
+        new_network = models.Network.query.filter_by(
+            vlan_name=data["network"]
+        ).first_or_404()
+        if not current_user.has_access_to_network(new_network):
+            raise CSEntryError("User doesn't have the required group", status_code=403)
+    # Ensure that the IP is in the network (current one or new one if passed)
+    # This is required because validate_interfaces is only called when a new network
+    # is assigned. So the check is needed even when new_network == interface.network
+    # (same network given in argument)
+    if "ip" in data:
+        try:
+            if "network" in data:
+                validate_ip(data["ip"], new_network)
+            else:
+                validate_ip(data["ip"], interface.network)
+        except ValidationError as e:
+            raise CSEntryError(str(e), status_code=422)
+    # The method doesn't allow to update the interface host
+    # I don't think it makes much sense (and an interface shall start by the host name)
+    # To add a new cname, use create_cname
+    allowed_fields = (
+        ("ip", str, None),
+        ("name", str, None),
+        ("network", models.Network, "vlan_name"),
+        ("mac", str, None),
+    )
+    return utils.update_generic_model(models.Interface, interface_id, allowed_fields)
+
+
+@bp.route("/interfaces/<int:interface_id>", methods=["DELETE"])
+@login_groups_accepted("admin", "network")
+def delete_interface(interface_id):
+    """Delete an interface
+
+    .. :quickref: Network; Delete an interface
+
+    :param interface_id: interface primary key
+    """
+    return utils.delete_generic_model(models.Interface, interface_id)
+
+
+@bp.route("/groups")
+@login_required
+def get_ansible_groups():
+    """Return ansible groups
+
+    .. :quickref: Network; Get Ansible groups
+    """
+    return utils.get_generic_model(
+        models.AnsibleGroup, order_by=models.AnsibleGroup.name
+    )
+
+
+@bp.route("/groups", methods=["POST"])
+@login_groups_accepted("admin")
+def create_ansible_groups():
+    """Create a new Ansible group
+
+    .. :quickref: Network; Create new Ansible group
+
+    :jsonparam name: group name
+    :jsonparam vars: (optional) Ansible variables
+    """
+    return utils.create_generic_model(models.AnsibleGroup, mandatory_fields=("name",))
+
+
+@bp.route("/groups/<int:group_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_ansible_group(group_id):
+    """Delete an Ansible group
+
+    .. :quickref: Network; Delete an Ansible group
+
+    :param group_id: Ansible group primary key
+    """
+    return utils.delete_generic_model(models.AnsibleGroup, group_id)
+
+
+@bp.route("/hosts")
+@login_required
+def get_hosts():
+    """Return hosts
+
+    .. :quickref: Network; Get hosts
+    """
+    query = models.Host.query
+    if not (current_user.is_admin or current_user.is_auditor):
+        # Note that hosts without interface will be filtered out
+        # by this query. This is not an issue as hosts should always
+        # have an interface. They are useless otherwise.
+        non_sensitive_hosts = (
+            query.join(models.Host.interfaces)
+            .join(models.Interface.network)
+            .filter(models.Network.sensitive.is_(False))
+        )
+        sensitive_hosts = (
+            query.join(models.Host.interfaces)
+            .join(models.Interface.network)
+            .filter(models.Network.sensitive.is_(True))
+            .join(models.Network.scope)
+            .filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
+        )
+        query = (
+            non_sensitive_hosts.union(sensitive_hosts)
+            .distinct(models.Host.id)
+            .from_self()
+        )
+    query = query.order_by(models.Host.name)
+    return utils.get_generic_model(model=models.Host, base_query=query)
+
+
+@bp.route("/hosts/search")
+@login_required
+def search_hosts():
+    """Search hosts
+
+    .. :quickref: Network; Search hosts
+
+    :query q: the search query
+    """
+    return utils.search_generic_model(models.Host, filter_sensitive=True)
+
+
+@bp.route("/hosts", methods=["POST"])
+@login_groups_accepted("admin", "network")
+def create_host():
+    """Create a new host
+
+    .. :quickref: Network; Create new host
+
+    :jsonparam name: hostname
+    :jsonparam device_type: Physical|Virtual|...
+    :jsonparam is_ioc: True|False (optional)
+    :jsonparam description: (optional) description
+    :jsonparam items: (optional) list of items ICS id linked to the host
+    :jsonparam ansible_vars: (optional) Ansible variables
+    :jsonparam ansible_groups: (optional) list of Ansible groups names
+    """
+    return utils.create_generic_model(
+        models.Host, mandatory_fields=("name", "device_type")
+    )
+
+
+@bp.route("/hosts/<int:host_id>", methods=["PATCH"])
+@login_groups_accepted("admin", "network")
+def patch_host(host_id):
+    r"""Patch an existing host
+
+    .. :quickref: Network; Update an existing host
+
+    :param host_id: host primary key
+    :jsonparam device_type: Physical|Virtual|...
+    :jsonparam is_ioc: True|False
+    :jsonparam description: description
+    :jsonparam items: list of items ICS id linked to the host
+    :jsonparam ansible_vars: Ansible variables
+    :jsonparam ansible_groups: list of Ansible groups names
+    """
+    host = models.Host.query.get_or_404(host_id)
+    if not current_user.has_access_to_network(host.main_network):
+        raise CSEntryError("User doesn't have the required group", status_code=403)
+    # The method currently doesn't allow to update the host name
+    # If we do, we have to update all the linked interface name as well!
+    # Interfaces shall always start by the host name
+    allowed_fields = (
+        ("device_type", models.DeviceType, "name"),
+        ("is_ioc", bool, None),
+        ("description", str, None),
+        ("items", [models.Item], "ics_id"),
+        ("ansible_vars", dict, None),
+        ("ansible_groups", [models.AnsibleGroup], "name"),
+    )
+    return utils.update_generic_model(models.Host, host_id, allowed_fields)
+
+
+@bp.route("/hosts/<int:host_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_host(host_id):
+    """Delete a host
+
+    .. :quickref: Network; Delete a host
+
+    :param host_id: host primary key
+    """
+    return utils.delete_generic_model(models.Host, host_id)
+
+
+@bp.route("/domains")
+@login_required
+def get_domains():
+    """Return domains
+
+    .. :quickref: Network; Get domains
+    """
+    return utils.get_generic_model(models.Domain, order_by=models.Domain.name)
+
+
+@bp.route("/domains", methods=["POST"])
+@login_groups_accepted("admin")
+def create_domain():
+    """Create a new domain
+
+    .. :quickref: Network; Create new domain
+
+    :jsonparam name: domain name
+    """
+    return utils.create_generic_model(models.Domain, mandatory_fields=("name",))
+
+
+@bp.route("/domains/<int:domain_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_domain(domain_id):
+    """Delete a domain
+
+    .. :quickref: Network; Delete a domain
+
+    :param domain_id: domain primary key
+    """
+    return utils.delete_generic_model(models.Domain, domain_id)
+
+
+@bp.route("/cnames")
+@login_required
+def get_cnames():
+    """Return cnames
+
+    .. :quickref: Network; Get cnames
+    """
+    domain = request.args.get("domain", None)
+    if domain is not None:
+        query = models.Cname.query
+        query = (
+            query.join(models.Cname.interface)
+            .join(models.Interface.network)
+            .join(models.Network.domain)
+            .filter(models.Domain.name == domain)
+        )
+        query = query.order_by(models.Cname.name)
+        return utils.get_generic_model(model=None, query=query)
+    return utils.get_generic_model(models.Cname, order_by=models.Cname.name)
+
+
+@bp.route("/cnames", methods=["POST"])
+@login_groups_accepted("admin")
+def create_cname():
+    """Create a new cname
+
+    .. :quickref: Network; Create new cname
+
+    :jsonparam name: full cname
+    :jsonparam interface_id: primary key of the associated interface
+    """
+    return utils.create_generic_model(
+        models.Cname, mandatory_fields=("name", "interface_id")
+    )
+
+
+@bp.route("/cnames/<int:cname_id>", methods=["DELETE"])
+@login_groups_accepted("admin")
+def delete_cname(cname_id):
+    """Delete a cname
+
+    .. :quickref: Network; Delete a cname
+
+    :param cname_id: cname primary key
+    """
+    return utils.delete_generic_model(models.Cname, cname_id)

app/api/user.py

@@ -11,49 +11,60 @@ This module implements the user API.
 """
 from flask import current_app, Blueprint, jsonify, request
 from flask_ldap3_login import AuthenticationResponseStatus
-from flask_jwt_extended import jwt_required
+from flask_login import login_required, current_user
 from ..extensions import ldap_manager
-from ..decorators import jwt_groups_accepted
+from ..decorators import login_groups_accepted
 from .. import utils, tokens, models
-from .utils import get_generic_model, create_generic_model
+from .utils import get_generic_model
 
-bp = Blueprint('user_api', __name__)
+bp = Blueprint("user_api", __name__)
 
 
-@bp.route('/users')
-@jwt_required
+@bp.route("/users")
+@login_groups_accepted("admin", "auditor")
 def get_users():
-    return get_generic_model(models.User, request.args,
-                             order_by=models.User.username)
+    """Return users information
 
+    .. :quickref: User; Get users information
+    """
+    return get_generic_model(models.User, order_by=models.User.username)
 
-@bp.route('/users', methods=['POST'])
-@jwt_required
-@jwt_groups_accepted('admin')
-def create_user():
-    """Create a new user"""
-    return create_generic_model(models.User, mandatory_fields=(
-        'username', 'display_name', 'email'))
 
+@bp.route("/profile")
+@login_required
+def get_user_profile():
+    """Return the current user profile
 
-@bp.route('/login', methods=['POST'])
+    .. :quickref: User; Get current user profile
+    """
+    return jsonify(current_user.to_dict()), 200
+
+
+@bp.route("/login", methods=["POST"])
 def login():
+    """Return a JSON Web Token
+
+    .. :quickref: User; Get a token
+
+    :jsonparam username: username to login
+    :jsonparam password: password
+    """
     data = request.get_json()
     if data is None:
-        raise utils.CSEntryError('Body should be a JSON object')
+        raise utils.CSEntryError("Body should be a JSON object")
     try:
-        username = data['username']
-        password = data['password']
+        username = data["username"]
+        password = data["password"]
     except KeyError:
-        raise utils.CSEntryError('Missing mandatory field (username or password)', status_code=422)
+        raise utils.CSEntryError(
+            "Missing mandatory field (username or password)", status_code=422
+        )
     response = ldap_manager.authenticate(username, password)
     if response.status == AuthenticationResponseStatus.success:
-        current_app.logger.debug(f'{username} successfully logged in')
+        current_app.logger.debug(f"{username} successfully logged in")
         user = ldap_manager._save_user(
-            response.user_dn,
-            response.user_id,
-            response.user_info,
-            response.user_groups)
-        payload = {'access_token': tokens.generate_access_token(identity=user.id)}
+            response.user_dn, response.user_id, response.user_info, response.user_groups
+        )
+        payload = {"access_token": tokens.generate_access_token(identity=user.id)}
         return jsonify(payload), 200
-    raise utils.CSEntryError('Invalid credentials', status_code=401)
+    raise utils.CSEntryError("Invalid credentials", status_code=401)

app/api/utils.py

@@ -9,8 +9,11 @@ This module implements useful functions for the API.
 :license: BSD 2-Clause, see LICENSE for more details.
 
 """
+import urllib.parse
 import sqlalchemy as sa
 from flask import current_app, jsonify, request
+from flask_login import current_user
+from flask_sqlalchemy import Pagination
 from ..extensions import db
 from .. import utils
 
@@ -18,43 +21,187 @@ from .. import utils
 def commit():
     try:
         db.session.commit()
-    except (sa.exc.IntegrityError, sa.exc.DataError) as e:
+    except sa.exc.SQLAlchemyError as e:
         db.session.rollback()
         raise utils.CSEntryError(str(e), status_code=422)
 
 
-def get_generic_model(model, args, order_by=None):
+def build_pagination_header(pagination, base_url, **kwargs):
+    """Return the X-Total-Count and Link header information
+
+    :param pagination: flask_sqlalchemy Pagination class instance
+    :param base_url: request base_url
+    :param kwargs: extra query string parameters (without page and per_page)
+    :returns: dict with X-Total-Count and Link keys
+    """
+    header = {"X-Total-Count": pagination.total}
+    links = []
+    if pagination.page > 1:
+        params = urllib.parse.urlencode(
+            {"per_page": pagination.per_page, "page": 1, **kwargs}
+        )
+        links.append(f'<{base_url}?{params}>; rel="first"')
+    if pagination.has_prev:
+        params = urllib.parse.urlencode(
+            {"per_page": pagination.per_page, "page": pagination.prev_num, **kwargs}
+        )
+        links.append(f'<{base_url}?{params}>; rel="prev"')
+    if pagination.has_next:
+        params = urllib.parse.urlencode(
+            {"per_page": pagination.per_page, "page": pagination.next_num, **kwargs}
+        )
+        links.append(f'<{base_url}?{params}>; rel="next"')
+    if pagination.pages > pagination.page:
+        params = urllib.parse.urlencode(
+            {"per_page": pagination.per_page, "page": pagination.pages, **kwargs}
+        )
+        links.append(f'<{base_url}?{params}>; rel="last"')
+    if links:
+        header["Link"] = ", ".join(links)
+    return header
+
+
+def get_generic_model(model, order_by=None, base_query=None, query=None):
     """Return data from model as json
 
     :param model: model class
-    :param MultiDict args: args from the request
-    :param order_by: column to order the result by
+    :param order_by: column to order the result by (not used if base_query or query is passed)
+    :param query: optional base_query to use [default: model.query]
+    :param query: optional query to use (for more complex queries)
     :returns: data from model as json
     """
-    query = utils.get_query(model.query, request.args)
-    if order_by is None:
-        order_by = getattr(model, 'name')
-    instances = query.order_by(order_by)
-    data = [instance.to_dict() for instance in instances]
-    return jsonify(data)
+    kwargs = request.args.to_dict()
+    page = int(kwargs.pop("page", 1))
+    per_page = int(kwargs.pop("per_page", 20))
+    # Remove recursive from kwargs so that it doesn't get passed
+    # to query.filter_by in get_query
+    recursive = kwargs.pop("recursive", "false").lower() == "true"
+    if query is None:
+        if base_query is None:
+            base_query = model.query
+            if order_by is None:
+                order_by = getattr(model, "name")
+            base_query = base_query.order_by(order_by)
+        query = utils.get_query(base_query, model, **kwargs)
+    pagination = query.paginate(
+        page, per_page, max_per_page=current_app.config["MAX_PER_PAGE"]
+    )
+    data = [item.to_dict(recursive=recursive) for item in pagination.items]
+    # Re-add recursive to kwargs so that it's part of the pagination url
+    kwargs["recursive"] = recursive
+    header = build_pagination_header(pagination, request.base_url, **kwargs)
+    return jsonify(data), 200, header
 
 
-def create_generic_model(model, mandatory_fields=('name',), **kwargs):
+def search_generic_model(model, filter_sensitive=False):
+    """Return filtered data from model as json
+
+    :param model: model class
+    :param bool filter_sensitive: filter out sensitive data if set to True
+    :returns: filtered data from model as json
+    """
+    kwargs = request.args.to_dict()
+    page = int(kwargs.pop("page", 1))
+    per_page = int(kwargs.pop("per_page", 20))
+    per_page = min(per_page, current_app.config["MAX_PER_PAGE"])
+    search = kwargs.get("q", "*")
+    instances, nb_filtered = model.search(
+        search, page=page, per_page=per_page, filter_sensitive=filter_sensitive
+    )
+    current_app.logger.debug(
+        f'Found {nb_filtered} {model.__tablename__}(s) when searching "{search}"'
+    )
+    data = [instance.to_dict(recursive=True) for instance in instances]
+    pagination = Pagination(None, page, per_page, nb_filtered, None)
+    header = build_pagination_header(pagination, request.base_url, **kwargs)
+    return jsonify(data), 200, header
+
+
+def get_json_body():
+    """Return the json body from the current request
+
+    Raise a CSEntryError if the body is not a json object
+    """
     data = request.get_json()
     if data is None:
-        raise utils.CSEntryError('Body should be a JSON object')
-    current_app.logger.debug(f'Received: {data}')
+        raise utils.CSEntryError("Body should be a JSON object")
+    if not data:
+        raise utils.CSEntryError("At least one field is required", status_code=422)
+    return data
+
+
+def create_generic_model(model, mandatory_fields=("name",), **kwargs):
+    """Create an instance of the model
+
+    :param model: model class
+    :param mandatory_fields: list of fields that shall be passed in the body
+    :param kwargs: extra fields to use
+    :returns: representation of the created instance as json
+    """
+    data = get_json_body()
     data.update(kwargs)
     for mandatory_field in mandatory_fields:
         if mandatory_field not in data:
-            raise utils.CSEntryError(f"Missing mandatory field '{mandatory_field}'", status_code=422)
+            raise utils.CSEntryError(
+                f"Missing mandatory field '{mandatory_field}'", status_code=422
+            )
     try:
         instance = model(**data)
     except TypeError as e:
-        message = str(e).replace('__init__() got an ', '')
+        message = str(e).replace("__init__() got an ", "")
         raise utils.CSEntryError(message, status_code=422)
     except ValueError as e:
         raise utils.CSEntryError(str(e), status_code=422)
     db.session.add(instance)
     commit()
+    current_app.logger.info(
+        f"New {model.__tablename__} created by {current_user}: {instance.to_dict()}"
+    )
     return jsonify(instance.to_dict()), 201
+
+
+def delete_generic_model(model, primary_key):
+    """Delete the model based on the primary_key
+
+    :param model: model class
+    :param primary_key: primary key of the instance to delete
+    """
+    instance = model.query.get_or_404(primary_key)
+    db.session.delete(instance)
+    commit()
+    current_app.logger.info(
+        f"{model.__tablename__} {instance} deleted by {current_user}"
+    )
+    return jsonify(), 204
+
+
+def update_generic_model(model, primary_key, allowed_fields):
+    """Update the model based on the primary_key
+
+    :param model: model class
+    :param primary_key: primary key of the instance to update
+    :param allowed_fields: list of fields that can be updated
+    :returns: representation of the updated model as json
+    """
+    data = get_json_body()
+    # Use a list and not a set because the order is important
+    allowed_keys = [field[0] for field in allowed_fields]
+    for key in data:
+        if key not in allowed_keys:
+            raise utils.CSEntryError(f"Invalid field '{key}'", status_code=422)
+    instance = model.query.get_or_404(primary_key)
+    data = utils.convert_to_models(data, allowed_fields)
+    try:
+        # Loop on allowed_keys and not data to respect
+        # the order in which the fields are set
+        # setting ip before network is important for Interface
+        for key in allowed_keys:
+            if key in data:
+                setattr(instance, key, data[key])
+    except Exception as e:
+        raise utils.CSEntryError(str(e), status_code=422)
+    commit()
+    current_app.logger.info(
+        f"{model.__tablename__} {primary_key} updated by {current_user}: {instance.to_dict()}"
+    )
+    return jsonify(instance.to_dict()), 200

app/commands.py

+# -*- coding: utf-8 -*-
+"""
+app.commands
+~~~~~~~~~~~~
+
+This module defines extra flask commands.
+
+:copyright: (c) 2018 European Spallation Source ERIC
+:license: BSD 2-Clause, see LICENSE for more details.
+
+"""
+import click
+import ldap3
+import redis
+import rq
+import sqlalchemy as sa
+import sentry_sdk
+from flask import current_app
+from sentry_sdk.integrations.rq import RqIntegration
+from .extensions import db, ldap_manager
+from .defaults import defaults
+from .tasks import TaskWorker
+from . import models, utils, tokens
+
+
+def disable_user(user):
+    """Clear users'groups, email and tokens"""
+    user.groups = []
+    user.email = ""
+    # Revoke all user's tokens
+    for token in user.tokens:
+        db.session.delete(token)
+
+
+def sync_user(connection, user):
+    """Synchronize the user from the database with information from the LDAP server"""
+    search_attr = current_app.config.get("LDAP_USER_LOGIN_ATTR")
+    object_filter = current_app.config.get("LDAP_USER_OBJECT_FILTER")
+    search_filter = f"(&{object_filter}({search_attr}={user.username}))"
+    connection.search(
+        search_base=ldap_manager.full_user_search_dn,
+        search_filter=search_filter,
+        search_scope=getattr(ldap3, current_app.config.get("LDAP_USER_SEARCH_SCOPE")),
+        attributes=current_app.config.get("LDAP_GET_USER_ATTRIBUTES"),
+    )
+    results = [
+        result for result in connection.response if result["type"] == "searchResEntry"
+    ]
+    if len(results) == 1:
+        ldap_user = results[0]
+        # OU=InActiveUsers is specific to ESS AD
+        if "OU=InActiveUsers" in ldap_user["dn"]:
+            current_app.logger.info(f"{user} is inactive. User disabled.")
+            disable_user(user)
+        else:
+            attributes = ldap_user["attributes"]
+            user.display_name = (
+                utils.attribute_to_string(attributes["cn"]) or user.username
+            )
+            user.email = utils.attribute_to_string(attributes["mail"])
+            groups = ldap_manager.get_user_groups(
+                dn=ldap3.utils.conv.escape_filter_chars(ldap_user["dn"]),
+                _connection=connection,
+            )
+            user.groups = sorted(
+                [utils.attribute_to_string(group["cn"]) for group in groups]
+            )
+            current_app.logger.info(f"{user} updated")
+    elif len(results) == 0:
+        current_app.logger.warning(f"{user} not found! User disabled.")
+        disable_user(user)
+    else:
+        current_app.logger.warning(f"Too many results for {user}!")
+        current_app.logger.warning(f"results: {results}")
+    return user
+
+
+def sync_users():
+    """Synchronize all users from the database with information the LDAP server"""
+    current_app.logger.info(
+        "Synchronize database with information from the LDAP server"
+    )
+    try:
+        connection = ldap_manager.connection
+    except ldap3.core.exceptions.LDAPException as e:
+        current_app.logger.warning(f"Failed to connect to the LDAP server: {e}")
+        return
+    for user in models.User.query.all():
+        sync_user(connection, user)
+    db.session.commit()
+
+
+def clean_deferred_tasks():
+    """Set all deferred tasks to failed"""
+    for task in (
+        models.Task.query.filter_by(status=models.JobStatus.DEFERRED)
+        .order_by(models.Task.created_at)
+        .all()
+    ):
+        if task.depends_on is None or task.depends_on.status == models.JobStatus.FAILED:
+            current_app.logger.info(
+                f"Set deferred task {task.id} ({task.name}) to failed"
+            )
+            task.status = models.JobStatus.FAILED
+    db.session.commit()
+
+
+def register_cli(app):
+    @app.cli.command()
+    def create_defaults():
+        """Create the database default values"""
+        for instance in defaults:
+            db.session.add(instance)
+            try:
+                db.session.commit()
+            except sa.exc.IntegrityError:
+                db.session.rollback()
+                app.logger.debug(f"{instance} already exists")
+
+    @app.cli.command()
+    def clean_deferred():
+        """Set deferred tasks to failed if the task it depends on failed"""
+        clean_deferred_tasks()
+
+    @app.cli.command()
+    def syncusers():
+        """Synchronize all users from the database with information the LDAP server"""
+        sync_users()
+
+    @app.cli.command()
+    def delete_expired_tokens():
+        """Prune database from expired tokens"""
+        tokens.prune_database()
+
+    @app.cli.command()
+    def maintenance():
+        """Run maintenance commands"""
+        sync_users()
+        tokens.prune_database()
+
+    @app.cli.command()
+    def runworker():
+        """Run RQ worker"""
+        redis_url = current_app.config["RQ_REDIS_URL"]
+        redis_connection = redis.from_url(redis_url)
+        with rq.Connection(redis_connection):
+            worker = TaskWorker(["high", "normal", "low"])
+            if current_app.config["SENTRY_DSN"]:
+                sentry_sdk.init(
+                    current_app.config["SENTRY_DSN"],
+                    environment=current_app.config["CSENTRY_ENVIRONMENT"],
+                    integrations=[RqIntegration()],
+                )
+            worker.work()
+
+    @app.cli.command()
+    @click.option(
+        "--delete/--no-delete",
+        default=True,
+        help="Delete and recreate the index [default: True]",
+    )
+    @click.option(
+        "--name", default="all", help="name of the class to reindex [default: all]"
+    )
+    def reindex(delete, name):
+        """Initialize the elasticsearch index"""
+        if delete:
+            current_app.elasticsearch.indices.delete("*", ignore=404)
+        if name == "all":
+            models.Item.reindex(delete)
+            models.Host.reindex(delete)
+            models.AnsibleGroup.reindex(delete)
+            return
+        try:
+            getattr(models, name).reindex(delete)
+        except AttributeError:
+            click.echo(f"The class {name} can't be indexed")

app/decorators.py

@@ -10,67 +10,50 @@ This module defines some useful decorators.
 
 """
 from functools import wraps
-from flask import current_app, abort
+from flask import current_app, abort, g
 from flask_login import current_user
-from flask_jwt_extended import get_current_user
 from .utils import CSEntryError
 
 
-def jwt_groups_accepted(*groups):
-    """Decorator which specifies that a user must have at least one of the specified groups.
-
-    This shall be used for users logged in using a JWT (API).
-
-    Example::
-        @bp.route('/models', methods=['POST'])
-        @jwt_required
-        @jwt_groups_accepted('admin', 'create')
-        def create_model():
-            return create()
-
-    The current user must be in either 'admin' or 'create' group
-    to access this route.
-
-    :param groups: accepted groups
-    """
-    def wrapper(fn):
-        @wraps(fn)
-        def decorated_view(*args, **kwargs):
-            user = get_current_user()
-            if user is None:
-                raise CSEntryError('Invalid indentity', status_code=403)
-            if not user.is_member_of_one_group(groups):
-                raise CSEntryError("User doesn't have the required group", status_code=403)
-            return fn(*args, **kwargs)
-        return decorated_view
-    return wrapper
-
-
 def login_groups_accepted(*groups):
     """Decorator which specifies that a user must have at least one of the specified groups.
 
-    This shall be used for users logged in using a cookie (web UI).
+    This can be used for users logged in using a cookie (web UI) or JWT (API).
 
     Example::
+
         @bp.route('/models', methods=['POST'])
-        @login_groups_accepted('admin', 'create')
+        @login_groups_accepted('admin', 'inventory')
         def create_model():
             return create()
 
-    The current user must be in either 'admin' or 'create' group
-    to access this route.
+        The current user must be in either 'admin' or 'create' group
+        to access the /models route.
+
     This checks that the user is logged in. There is no need to
     use the @login_required decorator.
 
     :param groups: accepted groups
     """
+
     def wrapper(fn):
         @wraps(fn)
         def decorated_view(*args, **kwargs):
+            # LOGIN_DISABLED can be set to True to turn off authentication check when testing.
+            # This variable is also used by flask-login
+            if current_app.config.get("LOGIN_DISABLED"):
+                return fn(*args, **kwargs)
             if not current_user.is_authenticated:
                 return current_app.login_manager.unauthorized()
             if not current_user.is_member_of_one_group(groups):
-                abort(403)
+                if g.get("login_via_request"):
+                    raise CSEntryError(
+                        "User doesn't have the required group", status_code=403
+                    )
+                else:
+                    abort(403)
             return fn(*args, **kwargs)
+
         return decorated_view
+
     return wrapper

app/defaults.py

@@ -13,47 +13,20 @@ from . import models
 
 
 defaults = [
-    models.Location(name='ICS lab'),
-    models.Location(name='Utgård'),
-    models.Location(name='Site'),
-    models.Location(name='ESS'),
-
-    models.Status(name='Stock'),
-    models.Status(name='In service'),
-    models.Status(name='Disposed'),
-
-    models.Manufacturer(name='ESS'),
-    models.Manufacturer(name='MRF'),
-    models.Manufacturer(name='IOxOS'),
-    models.Manufacturer(name='Wiener'),
-    models.Manufacturer(name='Moxa'),
-    models.Manufacturer(name='NAT'),
-    models.Manufacturer(name='Concurrent'),
-    models.Manufacturer(name='Schroff'),
-    models.Manufacturer(name='Struck'),
-    models.Manufacturer(name='Dell'),
-    models.Manufacturer(name='Samsung'),
-    models.Manufacturer(name='HP'),
-    models.Manufacturer(name='IBM'),
-    models.Manufacturer(name='CAEN'),
-    models.Manufacturer(name='Raritan'),
-    models.Manufacturer(name='DYMO'),
-
-    models.Model(name='vme-evm-300'),
-    models.Model(name='mtca-evr-300u'),
-    models.Model(name='pcie-evr-300dc'),
-    models.Model(name='Moxa-Nport-6650'),
-    models.Model(name='Dyno-LabelWriter-450-Duo'),
-    models.Model(name='NAT-MCH-PHYS'),
-    models.Model(name='microSD-EVO-32G'),
-
-    models.Action(name='Assign ICS id'),
-    models.Action(name='Clear all'),
-    models.Action(name='Clear attributes'),
-    models.Action(name='Fetch'),
-    models.Action(name='Register'),
-    models.Action(name='Set as parent'),
-    models.Action(name='Update'),
-
-    models.Tag(name='gateway'),
+    models.Action(name="Assign ICS id"),
+    models.Action(name="Clear all"),
+    models.Action(name="Clear attributes"),
+    models.Action(name="Fetch"),
+    models.Action(name="Register"),
+    models.Action(name="Set as parent"),
+    models.Action(name="Update"),
+    models.DeviceType(name="PhysicalMachine"),
+    models.DeviceType(name="VirtualMachine"),
+    models.DeviceType(name="Network"),
+    models.DeviceType(name="MTCA-AMC"),
+    models.DeviceType(name="MTCA-IFC"),
+    models.DeviceType(name="MTCA-MCH"),
+    models.DeviceType(name="MTCA-RTM"),
+    models.DeviceType(name="VME"),
+    models.DeviceType(name="PLC"),
 ]