Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • andersharrisson/csentry
  • ics-infrastructure/csentry
2 results
Show changes
Commits on Source (138)
38 additional commits have been omitted to prevent performance issues.
Hide whitespace changes
Inline Side-by-side
Showing
with 781 additions and 152 deletions
.env
View file @ 534cf643
...@@ -3,4 +3,3 @@ POSTGRES_PASSWORD=icspwd ...@@ -3,4 +3,3 @@ POSTGRES_PASSWORD=icspwd
POSTGRES_DB=csentry_db POSTGRES_DB=csentry_db
PGDATA_VOLUME=./data/postgres PGDATA_VOLUME=./data/postgres
ELASTIC_DATA_VOLUME=./data/elastic ELASTIC_DATA_VOLUME=./data/elastic
GL_ACCESS_TOKEN=xxxx
.gitignore
View file @ 534cf643
...@@ -15,3 +15,4 @@ junit.xml ...@@ -15,3 +15,4 @@ junit.xml
/app/static/files/* /app/static/files/*
!/app/static/files/.empty !/app/static/files/.empty
.vscode .vscode
.mypy_cache
.gitlab-ci.yml
View file @ 534cf643
image: docker:latest ---
include:
.runner_tags: &runner_tags - remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/PreCommit.gitlab-ci.yml"
tags: - remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/SonarScanner.gitlab-ci.yml"
- docker
variables: variables:
CONTAINER_TEST_IMAGE: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME" CONTAINER_TEST_IMAGE: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME"
...@@ -20,38 +19,34 @@ stages: ...@@ -20,38 +19,34 @@ stages:
- release - release
- deploy - deploy
default:
tags:
- docker
before_script: before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
check:
<<: *runner_tags
stage: check
image: registry.esss.lu.se/ics-docker/pre-commit:latest
before_script: []
script:
- pre-commit run --all-files
build: build:
<<: *runner_tags
stage: build stage: build
image: docker:latest
script: script:
- docker pull "$CONTAINER_CACHE_IMAGE" || true - docker pull "$CONTAINER_CACHE_IMAGE" || true
- docker build --pull --cache-from "$CONTAINER_CACHE_IMAGE" -t "$CONTAINER_TEST_IMAGE" . - docker build --pull --cache-from "$CONTAINER_CACHE_IMAGE" -t "$CONTAINER_TEST_IMAGE" .
- docker push "$CONTAINER_TEST_IMAGE" - docker push "$CONTAINER_TEST_IMAGE"
test: test:
<<: *runner_tags
stage: test stage: test
image: "$CONTAINER_TEST_IMAGE" image: "$CONTAINER_TEST_IMAGE"
services: services:
- postgres:10 - postgres:10
- redis:4.0 - redis:4.0
- name: docker.elastic.co/elasticsearch/elasticsearch:6.4.2 - name: docker.elastic.co/elasticsearch/elasticsearch:7.3.2
alias: elasticsearch alias: elasticsearch
command: ["bin/elasticsearch", "-Ediscovery.type=single-node"]
before_script: before_script:
- pip install -r requirements-dev.txt - pip install -r requirements-dev.txt
script: script:
- pytest --junitxml=junit.xml --cov-report xml:coverage.xml --cov=app -v - pytest --junitxml=junit.xml --cov-report=xml:coverage.xml --cov-report=term --cov=app -v
artifacts: artifacts:
paths: paths:
- junit.xml - junit.xml
...@@ -60,17 +55,9 @@ test: ...@@ -60,17 +55,9 @@ test:
junit: junit.xml junit: junit.xml
expire_in: 1 hour expire_in: 1 hour
analyse:
<<: *runner_tags
stage: analyse
image: registry.esss.lu.se/ics-docker/sonar-scanner:3
before_script: []
script:
- sonar-scanner -Dsonar.login=$SONARQUBE_TOKEN -Dsonar.projectVersion=$CI_COMMIT_REF_NAME
release-image: release-image:
<<: *runner_tags
stage: release stage: release
image: docker:latest
dependencies: [] dependencies: []
script: script:
- docker pull "$CONTAINER_TEST_IMAGE" - docker pull "$CONTAINER_TEST_IMAGE"
...@@ -79,10 +66,28 @@ release-image: ...@@ -79,10 +66,28 @@ release-image:
only: only:
- tags - tags
deploy-dev:
stage: deploy
image: registry.esss.lu.se/ics-docker/tower-cli
before_script: []
dependencies: []
script:
- >
tower-cli job launch
-h torn.tn.esss.lu.se
-t ${TOWER_OAUTH_TOKEN}
-J deploy-csentry-dev
-e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
environment:
name: dev
url: https://csentry-lab-01.cslab.esss.lu.se
except:
- master
- tags
deploy-staging: deploy-staging:
<<: *runner_tags
stage: deploy stage: deploy
image: registry.esss.lu.se/ics-docker/tower-cli:3.3 image: registry.esss.lu.se/ics-docker/tower-cli
before_script: [] before_script: []
dependencies: [] dependencies: []
script: script:
...@@ -100,7 +105,6 @@ deploy-staging: ...@@ -100,7 +105,6 @@ deploy-staging:
- tags - tags
pages: pages:
<<: *runner_tags
stage: deploy stage: deploy
image: "$CONTAINER_RELEASE_IMAGE" image: "$CONTAINER_RELEASE_IMAGE"
dependencies: [] dependencies: []
...@@ -118,9 +122,8 @@ pages: ...@@ -118,9 +122,8 @@ pages:
- tags - tags
deploy-production: deploy-production:
<<: *runner_tags
stage: deploy stage: deploy
image: registry.esss.lu.se/ics-docker/tower-cli:3.3 image: registry.esss.lu.se/ics-docker/tower-cli
before_script: [] before_script: []
dependencies: [] dependencies: []
script: script:
......
.pre-commit-config.yaml
View file @ 534cf643
repos: repos:
- repo: https://github.com/ambv/black - repo: https://github.com/psf/black
rev: stable rev: 22.6.0
hooks: hooks:
- id: black - id: black
language_version: python3.6 - repo: https://github.com/pycqa/flake8
- repo: https://github.com/pre-commit/pre-commit-hooks rev: 4.0.1
rev: v1.3.0
hooks: hooks:
- id: flake8 - id: flake8
CHANGELOG.rst
View file @ 534cf643
Changelog Changelog
========= =========
Version 2021.04.06
------------------
- Validate group names (INFRA-3135)
- Add warning text for creating vm (INFRA-3292)
- Paginate history in ansible groups and hosts (INFRA-3136)
Version 2020.11.25
------------------
- Add documentation about database versioning
- Set display_name to username when empty (INFRA-2909)
- Improve logging (INFRA-2899)
Version 2020.11.13
------------------
- Increased hostname character limit from 20 -> 24 (INFRA-2808)
- Use SSL by default for LDAP connection (INFRA-2805)
- Use SonarScanner.gitlab-ci.yml template (INFRA-2832)
Version 2020.10.30
------------------
- Fix inventory export to excel (INFRA-2780)
- Add developer documentation
Version 2020.10.27
------------------
- Update Python and requirements (INFRA-2649)
- Update RQ and rq-dashboard (INFRA-2649 / INFRA-2754)
- Fix code smells (INFRA-2729)
- Add auditor group (INFRA-2728)
- Update Sphinx and switch to sphinx_rtd_theme (INFRA-2742)
Version 2020.10.06
------------------
- Make sensitive hosts visible for members of the scope (INFRA-2508)
- Only hide sensitive networks not part of the user scope (INFRA-2437)
- Automatically set network_scope group as parent of network groups (INFRA-2639)
Version 2020.07.08
------------------
- Add view to edit network scope (INFRA-2316)
- Add API endpoint to patch network scope (INFRA-2316)
Version 2020.06.12
------------------
- Change MTCA-IOxOS to MTCA-IFC (INFRA-2200)
- Add description field on Interface (INFRA-2112)
Version 2020.04.16
------------------
- Allow /network/networks API endpoint for normal users (INFRA-2013)
- Update Bootstrap to 4.4.1 (INFRA-2021)
- Update DataTables (INFRA-2021)
- Add tooltip on search box (INFRA-1893)
Version 2020.03.17
------------------
- Add extra badges to README and documentation
- Fix get_hosts for normal users (INFRA-1888)
Version 2020.03.09
------------------
- Trigger core services update on host change (INFRA-1846)
- Add DELETE method on all API endpoints (INFRA-1853)
- Remove create_user API endpoint (INFRA-1854)
Version 2020.03.05
------------------
- Allow users to view networks (INFRA-1809)
- Add HOSTNAME Ansible group type (INFRA-1844)
- Remove IOC git repository creation (INFRA-1839)
- Make IP optional when creating interface via the API (INFRA-1833)
- Introduce new device types for MTCA (INFRA-1835)
Version 2020.02.25
------------------
- Fix inventory update (INFRA-1792)
Version 2020.02.24
------------------
- Add app version in navbar (INFRA-1789)
- Allow to pass LDAP settings as env variables (INFRA-1788)
- Implement server side processing for Ansible groups (INFRA-1790)
Version 0.29.0 (2020-02-03)
---------------------------
- Add MAC address column to hosts table (INFRA-1669)
- Restrict view of networks and scopes to admin only (INFRA-1671)
- Add sensitive field to Network class (INFRA-1671)
- Filter out sensitive hosts for non admin users (INFRA-1671)
- Add API endpoint and view to update network (INFRA-1724)
- Make pages title more explicit (INFRA-1726)
Version 0.28.0 (2019-12-19)
---------------------------
- Fix exception when setting new stack_member (INFRA-1648)
- Prevent recursive dependency loop in Ansible groups (INFRA-1622)
- Prevent network and scope overlapping (INFRA-1627)
- Add netmask to network view (INFRA-1660)
Version 0.27.1 (2019-12-04)
---------------------------
- Fix regexp for ICS id validation (INFRA-1569)
- Allow normal users to edit hosts with no interface (INFRA-1604)
Version 0.27.0 (2019-10-30)
---------------------------
- Increase interface name max length (INFRA-1324)
- Add endpoints to patch host and interface (INFRA-1406)
- Limit networks to same scope for extra interfaces (INFRA-1297)
Version 0.26.1 (2019-09-18)
---------------------------
- Fix inventory update not triggered (INFRA-1290)
Version 0.26.0 (2019-09-18)
---------------------------
- Decouple inventory and core services update (INFRA-1290)
- Add broadcast address to network view (INFRA-1291)
- Add view network scope page (INFRA-1292)
- Make ansible_vars in host searchable (INFRA-1295)
- Remove Generate ZTP configuration function (INFRA-1299)
Version 0.25.1 (2019-08-30)
---------------------------
- Prevent input of invalid Ansible variables (INFRA-1221)
- Update black and pre-commit config
- Add pytest.ini to speed test discovery
Version 0.25.0 (2019-08-22)
---------------------------
- Implement user access on network scope instead of domain (INFRA-1228)
- Create IOC repositories with same path as name (INFRA-1230)
Version 0.24.1 (2019-07-10)
---------------------------
- Fix sorting of items with some null stack_member (INFRA-1112)
Version 0.24.0 (2019-07-10)
---------------------------
- Add serial_number and stack_member to hosts json model (INFRA-1111)
Version 0.23.0 (2019-06-14)
---------------------------
- Increase RQ default timeout (INFRA-1051)
Version 0.22.0 (2019-06-14)
---------------------------
- Make vlan optional in NetworkScope and Network (INFRA-1049)
- Update reverse dependencies of failed tasks (INFRA-1051)
Version 0.21.1 (2019-05-13)
---------------------------
- Fix users synchronization (INFRA-1026)
- Remove caching of user retrieval (INFRA-1025)
Version 0.21.0 (2019-05-10)
---------------------------
- Improve documentation about ansible-vault
- Fix IOC host indexation (INFRA-1015)
- Allow users to delete their host (INFRA-1018)
- Fix job.status DeprecationWarning (INFRA-1021)
- Allow service users to login (INFRA-1022)
Version 0.20.2 (2019-05-08) Version 0.20.2 (2019-05-08)
--------------------------- ---------------------------
......
Dockerfile
View file @ 534cf643
FROM python:3.7-slim as base FROM python:3.8-slim as base
# Add ESS specific debian repository mirror
RUN echo "deb https://artifactory.esss.lu.se/artifactory/debian-mirror stable main" | tee /etc/apt/sources.list.d/ess-debian-mirror.list
# Install Python dependencies in an intermediate image # Install Python dependencies in an intermediate image
# as some requires a compiler (psycopg2) # as some requires a compiler (psycopg2)
...@@ -37,6 +40,7 @@ RUN apt-get update \ ...@@ -37,6 +40,7 @@ RUN apt-get update \
COPY requirements.txt /requirements.txt COPY requirements.txt /requirements.txt
RUN python -m venv /venv \ RUN python -m venv /venv \
&& . /venv/bin/activate \ && . /venv/bin/activate \
&& pip install --no-cache-dir --upgrade pip \
&& pip install --no-cache-dir -r /requirements.txt && pip install --no-cache-dir -r /requirements.txt
ARG CSENTRY_BUILD ARG CSENTRY_BUILD
...@@ -60,12 +64,18 @@ RUN apt-get update \ ...@@ -60,12 +64,18 @@ RUN apt-get update \
libpq5 \ libpq5 \
libpcre3 \ libpcre3 \
zlib1g \ zlib1g \
git \
&& apt-get clean \ && apt-get clean \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
RUN rm /etc/apt/sources.list.d/ess-debian-mirror.list
COPY --chown=csi:csi . /app/ COPY --chown=csi:csi . /app/
WORKDIR /app WORKDIR /app
RUN echo "__version__ = \"$(git describe)\"" > app/_version.py
ENV PATH /venv/bin:$PATH ENV PATH /venv/bin:$PATH
EXPOSE 8000
CMD ["flask", "run", "--host", "0.0.0.0", "--port", "8000"]
USER csi USER csi
Makefile
View file @ 534cf643
.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image .PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image docs
OWNER := registry.esss.lu.se/ics-infrastructure OWNER := registry.esss.lu.se/ics-infrastructure
GIT_TAG := $(shell git describe --always) GIT_TAG := $(shell git describe --always)
...@@ -48,11 +48,11 @@ upgrade_db: ## upgrade the dev database ...@@ -48,11 +48,11 @@ upgrade_db: ## upgrade the dev database
docker-compose run --rm web flask db upgrade head docker-compose run --rm web flask db upgrade head
test: ## run the tests (on current directory) test: ## run the tests (on current directory)
docker-compose run --rm web pytest --cov=app -v docker-compose -f docker-compose.yml run --rm web pytest --cov=app -v
db_image: ## start postgres and redis to test the latest image db_test: ## start required containers for test
# Pass docker-compose.yml to skip docker-compose.override.yml # Pass docker-compose.yml to skip docker-compose.override.yml (db not mounted as volume for speed)
docker-compose -f docker-compose.yml up -d postgres redis docker-compose -f docker-compose.yml up -d postgres redis elasticsearch worker
test_image: ## run the tests (on the latest image) test_image: ## run the tests (on the latest image)
# Pass docker-compose.yml to skip docker-compose.override.yml # Pass docker-compose.yml to skip docker-compose.override.yml
...@@ -60,3 +60,6 @@ test_image: ## run the tests (on the latest image) ...@@ -60,3 +60,6 @@ test_image: ## run the tests (on the latest image)
run_uwsgi: ## run the application with uwsgi (to test prod env) run_uwsgi: ## run the application with uwsgi (to test prod env)
docker-compose run -p 8000:8000 --rm web uwsgi --master --http 0.0.0.0:8000 --http-keepalive --manage-script-name --mount /csentry=wsgi.py --callable app --uid conda --processes 2 -b 16384 docker-compose run -p 8000:8000 --rm web uwsgi --master --http 0.0.0.0:8000 --http-keepalive --manage-script-name --mount /csentry=wsgi.py --callable app --uid conda --processes 2 -b 16384
docs: ## run the tests (on current directory)
docker run --rm -v $(shell pwd):/app registry.esss.lu.se/ics-infrastructure/csentry:master sphinx-build -M html docs docs/_build
README.rst
View file @ 534cf643
...@@ -4,6 +4,16 @@ CSEntry ...@@ -4,6 +4,16 @@ CSEntry
.. image:: https://img.shields.io/badge/code%20style-black-000000.svg .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
:target: https://github.com/ambv/black :target: https://github.com/ambv/black
.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=alert_status
:target: https://sonarqube.esss.lu.se/dashboard?id=csentry
.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=ncloc
:target: https://sonarqube.esss.lu.se/dashboard?id=csentry
.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/pipeline.svg
.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/coverage.svg
Control System Entry web server. Control System Entry web server.
......
app/_version.py 0 → 100644
View file @ 534cf643
__version__ = "dev"
app/admin/views.py
View file @ 534cf643
...@@ -14,7 +14,6 @@ from flask_admin.contrib import sqla ...@@ -14,7 +14,6 @@ from flask_admin.contrib import sqla
from flask_admin.model.form import converts from flask_admin.model.form import converts
from flask_login import current_user from flask_login import current_user
from ..validators import IPNetwork, ICS_ID_RE from ..validators import IPNetwork, ICS_ID_RE
from .. import models
# Monkey patch flask-admin Unique validator to disable it # Monkey patch flask-admin Unique validator to disable it
...@@ -48,16 +47,6 @@ class AdminModelView(sqla.ModelView): ...@@ -48,16 +47,6 @@ class AdminModelView(sqla.ModelView):
def is_accessible(self): def is_accessible(self):
return current_user.is_authenticated and current_user.is_admin return current_user.is_authenticated and current_user.is_admin
def update_model(self, form, model):
# Remove the current user object added by flask-admin from the session
# Adding the current_user object later raises an exception otherwise:
# sqlalchemy.exc.InvalidRequestError: Can't attach instance <User at 0x7f6f0e39ce10>;
# another instance with key (<class 'app.models.User'>, (1,), None) is already present in this session.
for obj in self.session:
if isinstance(obj, models.User) and obj.id == current_user.id:
self.session.expunge(obj)
return super().update_model(form, model)
class UserAdmin(AdminModelView): class UserAdmin(AdminModelView):
can_create = False can_create = False
......
app/api/inventory.py
View file @ 534cf643
...@@ -13,7 +13,7 @@ from flask import Blueprint, jsonify, request, current_app ...@@ -13,7 +13,7 @@ from flask import Blueprint, jsonify, request, current_app
from flask_login import login_required from flask_login import login_required
from .. import utils, models from .. import utils, models
from ..decorators import login_groups_accepted from ..decorators import login_groups_accepted
from .utils import commit, create_generic_model, get_generic_model from .utils import commit, create_generic_model, get_generic_model, delete_generic_model
bp = Blueprint("inventory_api", __name__) bp = Blueprint("inventory_api", __name__)
...@@ -45,7 +45,7 @@ def get_items(): ...@@ -45,7 +45,7 @@ def get_items():
@bp.route("/items/<id_>") @bp.route("/items/<id_>")
@login_required @login_required
def get_item(id_): def get_item(id_):
"""Retrieve item by id or ICS id r"""Retrieve item by id or ICS id
.. :quickref: Inventory; Get item by id or ICS id .. :quickref: Inventory; Get item by id or ICS id
...@@ -83,7 +83,7 @@ def create_item(): ...@@ -83,7 +83,7 @@ def create_item():
@bp.route("/items/<id_>", methods=["PATCH"]) @bp.route("/items/<id_>", methods=["PATCH"])
@login_groups_accepted("admin", "inventory") @login_groups_accepted("admin", "inventory")
def patch_item(id_): def patch_item(id_):
"""Patch an existing item r"""Patch an existing item
.. :quickref: Inventory; Update existing item .. :quickref: Inventory; Update existing item
...@@ -140,10 +140,22 @@ def patch_item(id_): ...@@ -140,10 +140,22 @@ def patch_item(id_):
return jsonify(item.to_dict()) return jsonify(item.to_dict())
@bp.route("/items/<int:item_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_item(item_id):
"""Delete an item
.. :quickref: Inventory; Delete an item
:param item_id: item primary key
"""
return delete_generic_model(models.Item, item_id)
@bp.route("/items/<id_>/comments") @bp.route("/items/<id_>/comments")
@login_required @login_required
def get_item_comments(id_): def get_item_comments(id_):
"""Get item comments r"""Get item comments
.. :quickref: Inventory; Get item comments .. :quickref: Inventory; Get item comments
...@@ -156,7 +168,7 @@ def get_item_comments(id_): ...@@ -156,7 +168,7 @@ def get_item_comments(id_):
@bp.route("/items/<id_>/comments", methods=["POST"]) @bp.route("/items/<id_>/comments", methods=["POST"])
@login_groups_accepted("admin", "inventory") @login_groups_accepted("admin", "inventory")
def create_item_comment(id_): def create_item_comment(id_):
"""Create a comment on item r"""Create a comment on item
.. :quickref: Inventory; Create comment on item .. :quickref: Inventory; Create comment on item
...@@ -169,6 +181,18 @@ def create_item_comment(id_): ...@@ -169,6 +181,18 @@ def create_item_comment(id_):
) )
@bp.route("/items/comments/<int:comment_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_comment(comment_id):
"""Delete an item comment
.. :quickref: Inventory; Delete an item comment
:param comment_id: comment primary key
"""
return delete_generic_model(models.ItemComment, comment_id)
@bp.route("/actions") @bp.route("/actions")
@login_required @login_required
def get_actions(): def get_actions():
...@@ -202,6 +226,18 @@ def create_manufacturer(): ...@@ -202,6 +226,18 @@ def create_manufacturer():
return create_generic_model(models.Manufacturer) return create_generic_model(models.Manufacturer)
@bp.route("/manufacturers/<int:manufacturer_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_manufacturer(manufacturer_id):
"""Delete a manufacturer
.. :quickref: Inventory; Delete a manufacturer
:param manufacturer_id: manufacturer primary key
"""
return delete_generic_model(models.Manufacturer, manufacturer_id)
@bp.route("/models") @bp.route("/models")
@login_required @login_required
def get_models(): def get_models():
...@@ -225,6 +261,18 @@ def create_model(): ...@@ -225,6 +261,18 @@ def create_model():
return create_generic_model(models.Model) return create_generic_model(models.Model)
@bp.route("/models/<int:model_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_model(model_id):
"""Delete a model
.. :quickref: Inventory; Delete a model
:param model_id: model primary key
"""
return delete_generic_model(models.Model, model_id)
@bp.route("/locations") @bp.route("/locations")
@login_required @login_required
def get_locations(): def get_locations():
...@@ -248,6 +296,18 @@ def create_locations(): ...@@ -248,6 +296,18 @@ def create_locations():
return create_generic_model(models.Location) return create_generic_model(models.Location)
@bp.route("/locations/<int:location_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_location(location_id):
"""Delete a location
.. :quickref: Inventory; Delete a location
:param location_id: location primary key
"""
return delete_generic_model(models.Location, location_id)
@bp.route("/statuses") @bp.route("/statuses")
@login_required @login_required
def get_status(): def get_status():
...@@ -271,6 +331,18 @@ def create_status(): ...@@ -271,6 +331,18 @@ def create_status():
return create_generic_model(models.Status) return create_generic_model(models.Status)
@bp.route("/statuses/<int:status_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_status(status_id):
"""Delete a status
.. :quickref: Inventory; Delete a status
:param status_id: status primary key
"""
return delete_generic_model(models.Status, status_id)
@bp.route("/macs") @bp.route("/macs")
@login_required @login_required
def get_macs(): def get_macs():
......
app/api/network.py
View file @ 534cf643
...@@ -11,16 +11,17 @@ This module implements the network API. ...@@ -11,16 +11,17 @@ This module implements the network API.
""" """
from flask import Blueprint, request from flask import Blueprint, request
from flask_login import login_required, current_user from flask_login import login_required, current_user
from wtforms import ValidationError
from .. import models from .. import models
from ..decorators import login_groups_accepted from ..decorators import login_groups_accepted
from ..utils import CSEntryError from ..utils import CSEntryError, validate_ip
from . import utils from . import utils
bp = Blueprint("network_api", __name__) bp = Blueprint("network_api", __name__)
@bp.route("/scopes") @bp.route("/scopes")
@login_required @login_groups_accepted("admin", "auditor")
def get_scopes(): def get_scopes():
"""Return network scopes """Return network scopes
...@@ -51,14 +52,66 @@ def create_scope(): ...@@ -51,14 +52,66 @@ def create_scope():
) )
@bp.route("/scopes/<int:scope_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_scope(scope_id):
"""Delete a network scope
.. :quickref: Network; Delete a network scope
:param scope_id: network scope primary key
"""
return utils.delete_generic_model(models.NetworkScope, scope_id)
@bp.route("/scopes/<int:scope_id>", methods=["PATCH"])
@login_groups_accepted("admin")
def patch_scope(scope_id):
r"""Patch an existing network scope
.. :quickref: Network; Update an existing network scope
:param scope_id: network scope primary key
:jsonparam name: network scope name
:jsonparam description: description
:jsonparam first_vlan: network scope first vlan
:jsonparam last_vlan: network scope last vlan
:jsonparam supernet: network scope supernet
:jsonparam domain: name of the default domain
"""
allowed_fields = (
("name", str, None),
("description", str, None),
("first_vlan", int, None),
("last_vlan", int, None),
("supernet", str, None),
("domain", models.Domain, "name"),
)
return utils.update_generic_model(models.NetworkScope, scope_id, allowed_fields)
@bp.route("/networks") @bp.route("/networks")
@login_required @login_groups_accepted("admin", "auditor", "network")
def get_networks(): def get_networks():
"""Return networks """Return networks
.. :quickref: Network; Get networks .. :quickref: Network; Get networks
""" """
return utils.get_generic_model(models.Network, order_by=models.Network.address) query = models.Network.query
if not (current_user.is_admin or current_user.is_auditor):
sensitive_networks = (
query.filter(models.Network.sensitive.is_(True))
.join(models.Network.scope)
.filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
)
query = (
query.filter(models.Network.sensitive.is_(False))
.union(sensitive_networks)
.distinct(models.Network.id)
.from_self()
)
query = query.order_by(models.Network.address)
return utils.get_generic_model(model=models.Network, base_query=query)
@bp.route("/networks", methods=["POST"]) @bp.route("/networks", methods=["POST"])
...@@ -78,6 +131,8 @@ def create_network(): ...@@ -78,6 +131,8 @@ def create_network():
:jsonparam domain_id: (optional) primary key of the domain [default: scope domain] :jsonparam domain_id: (optional) primary key of the domain [default: scope domain]
:jsonparam admin_only: (optional) boolean to restrict the network to admin users [default: False] :jsonparam admin_only: (optional) boolean to restrict the network to admin users [default: False]
:type admin_only: bool :type admin_only: bool
:jsonparam sensitive: hide the network and all hosts if True (for non admin) [default: False]
:type sensitive: bool
:jsonparam description: (optional) description :jsonparam description: (optional) description
""" """
return utils.create_generic_model( return utils.create_generic_model(
...@@ -94,6 +149,54 @@ def create_network(): ...@@ -94,6 +149,54 @@ def create_network():
) )
@bp.route("/networks/<int:network_id>", methods=["PATCH"])
@login_groups_accepted("admin")
def patch_network(network_id):
r"""Patch an existing network
.. :quickref: Network; Update an existing network
:param network_id: network primary key
:jsonparam vlan_name: vlan name
:jsonparam address: vlan address
:jsonparam first_ip: first IP of the allowed range
:jsonparam last_ip: last IP of the allowed range
:jsonparam gateway: gateway IP
:jsonparam domain: domain name
:jsonparam admin_only: boolean to restrict the network to admin users
:type admin_only: bool
:jsonparam sensitive: hide the network and all hosts if True (for non admin)
:type sensitive: bool
:jsonparam description: description
"""
# The method currently doesn't allow to update the network_scope
# as this will have an impact on the address
allowed_fields = (
("vlan_name", str, None),
("address", str, None),
("first_ip", str, None),
("last_ip", str, None),
("gateway", str, None),
("domain", models.Domain, "name"),
("admin_only", bool, None),
("sensitive", bool, None),
("description", str, None),
)
return utils.update_generic_model(models.Network, network_id, allowed_fields)
@bp.route("/networks/<int:network_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_network(network_id):
"""Delete a network
.. :quickref: Network; Delete a network
:param network_id: network primary key
"""
return utils.delete_generic_model(models.Network, network_id)
@bp.route("/interfaces") @bp.route("/interfaces")
@login_required @login_required
def get_interfaces(): def get_interfaces():
...@@ -101,25 +204,30 @@ def get_interfaces(): ...@@ -101,25 +204,30 @@ def get_interfaces():
.. :quickref: Network; Get interfaces .. :quickref: Network; Get interfaces
""" """
domain = request.args.get("domain", None) query = models.Interface.query
if domain is not None: query = query.join(models.Interface.network).order_by(models.Interface.ip)
query = models.Interface.query if not (current_user.is_admin or current_user.is_auditor):
sensitive_interfaces = (
query.filter(models.Network.sensitive.is_(True))
.join(models.Network.scope)
.filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
)
query = ( query = (
query.join(models.Interface.network) query.filter(models.Network.sensitive.is_(False))
.join(models.Network.domain) .union(sensitive_interfaces)
.filter(models.Domain.name == domain) .from_self()
.join(models.Interface.network)
.order_by(models.Interface.ip)
) )
query = query.order_by(models.Interface.ip) domain = request.args.get("domain", None)
if domain is not None:
query = query.join(models.Network.domain).filter(models.Domain.name == domain)
return utils.get_generic_model(model=None, query=query) return utils.get_generic_model(model=None, query=query)
network = request.args.get("network", None) network = request.args.get("network", None)
if network is not None: if network is not None:
query = models.Interface.query query = query.filter(models.Network.vlan_name == network)
query = query.join(models.Interface.network).filter(
models.Network.vlan_name == network
)
query = query.order_by(models.Interface.ip)
return utils.get_generic_model(model=None, query=query) return utils.get_generic_model(model=None, query=query)
return utils.get_generic_model(models.Interface, order_by=models.Interface.ip) return utils.get_generic_model(model=models.Interface, base_query=query)
@bp.route("/interfaces", methods=["POST"]) @bp.route("/interfaces", methods=["POST"])
...@@ -130,7 +238,7 @@ def create_interface(): ...@@ -130,7 +238,7 @@ def create_interface():
.. :quickref: Network; Create new interface .. :quickref: Network; Create new interface
:jsonparam network: network name :jsonparam network: network name
:jsonparam ip: interface IP :jsonparam ip: (optional) interface IP - IP will be assigned automatically if not given
:jsonparam name: interface name :jsonparam name: interface name
:jsonparam host: host name :jsonparam host: host name
:jsonparam mac: (optional) MAC address :jsonparam mac: (optional) MAC address
...@@ -152,12 +260,60 @@ def create_interface(): ...@@ -152,12 +260,60 @@ def create_interface():
if not current_user.has_access_to_network(network): if not current_user.has_access_to_network(network):
raise CSEntryError("User doesn't have the required group", status_code=403) raise CSEntryError("User doesn't have the required group", status_code=403)
return utils.create_generic_model( return utils.create_generic_model(
models.Interface, mandatory_fields=("network", "ip", "name", "host") models.Interface, mandatory_fields=("network", "name", "host")
) )
@bp.route("/interfaces/<int:interface_id>", methods=["PATCH"])
@login_groups_accepted("admin", "network")
def patch_interface(interface_id):
r"""Patch an existing interface
.. :quickref: Network; Update an existing interface
:param interface_id: interface primary key
:jsonparam ip: interface IP
:jsonparam name: interface name
:jsonparam network: network name
:jsonparam mac: MAC address
"""
interface = models.Interface.query.get_or_404(interface_id)
# User shall have access to both the current and new network (if provided)
if not current_user.has_access_to_network(interface.network):
raise CSEntryError("User doesn't have the required group", status_code=403)
data = utils.get_json_body()
if "network" in data:
new_network = models.Network.query.filter_by(
vlan_name=data["network"]
).first_or_404()
if not current_user.has_access_to_network(new_network):
raise CSEntryError("User doesn't have the required group", status_code=403)
# Ensure that the IP is in the network (current one or new one if passed)
# This is required because validate_interfaces is only called when a new network
# is assigned. So the check is needed even when new_network == interface.network
# (same network given in argument)
if "ip" in data:
try:
if "network" in data:
validate_ip(data["ip"], new_network)
else:
validate_ip(data["ip"], interface.network)
except ValidationError as e:
raise CSEntryError(str(e), status_code=422)
# The method doesn't allow to update the interface host
# I don't think it makes much sense (and an interface shall start by the host name)
# To add a new cname, use create_cname
allowed_fields = (
("ip", str, None),
("name", str, None),
("network", models.Network, "vlan_name"),
("mac", str, None),
)
return utils.update_generic_model(models.Interface, interface_id, allowed_fields)
@bp.route("/interfaces/<int:interface_id>", methods=["DELETE"]) @bp.route("/interfaces/<int:interface_id>", methods=["DELETE"])
@login_groups_accepted("admin") @login_groups_accepted("admin", "network")
def delete_interface(interface_id): def delete_interface(interface_id):
"""Delete an interface """Delete an interface
...@@ -193,6 +349,18 @@ def create_ansible_groups(): ...@@ -193,6 +349,18 @@ def create_ansible_groups():
return utils.create_generic_model(models.AnsibleGroup, mandatory_fields=("name",)) return utils.create_generic_model(models.AnsibleGroup, mandatory_fields=("name",))
@bp.route("/groups/<int:group_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_ansible_group(group_id):
"""Delete an Ansible group
.. :quickref: Network; Delete an Ansible group
:param group_id: Ansible group primary key
"""
return utils.delete_generic_model(models.AnsibleGroup, group_id)
@bp.route("/hosts") @bp.route("/hosts")
@login_required @login_required
def get_hosts(): def get_hosts():
...@@ -200,7 +368,30 @@ def get_hosts(): ...@@ -200,7 +368,30 @@ def get_hosts():
.. :quickref: Network; Get hosts .. :quickref: Network; Get hosts
""" """
return utils.get_generic_model(models.Host, order_by=models.Host.name) query = models.Host.query
if not (current_user.is_admin or current_user.is_auditor):
# Note that hosts without interface will be filtered out
# by this query. This is not an issue as hosts should always
# have an interface. They are useless otherwise.
non_sensitive_hosts = (
query.join(models.Host.interfaces)
.join(models.Interface.network)
.filter(models.Network.sensitive.is_(False))
)
sensitive_hosts = (
query.join(models.Host.interfaces)
.join(models.Interface.network)
.filter(models.Network.sensitive.is_(True))
.join(models.Network.scope)
.filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
)
query = (
non_sensitive_hosts.union(sensitive_hosts)
.distinct(models.Host.id)
.from_self()
)
query = query.order_by(models.Host.name)
return utils.get_generic_model(model=models.Host, base_query=query)
@bp.route("/hosts/search") @bp.route("/hosts/search")
...@@ -212,7 +403,7 @@ def search_hosts(): ...@@ -212,7 +403,7 @@ def search_hosts():
:query q: the search query :query q: the search query
""" """
return utils.search_generic_model(models.Host) return utils.search_generic_model(models.Host, filter_sensitive=True)
@bp.route("/hosts", methods=["POST"]) @bp.route("/hosts", methods=["POST"])
...@@ -235,6 +426,38 @@ def create_host(): ...@@ -235,6 +426,38 @@ def create_host():
) )
@bp.route("/hosts/<int:host_id>", methods=["PATCH"])
@login_groups_accepted("admin", "network")
def patch_host(host_id):
r"""Patch an existing host
.. :quickref: Network; Update an existing host
:param host_id: host primary key
:jsonparam device_type: Physical|Virtual|...
:jsonparam is_ioc: True|False
:jsonparam description: description
:jsonparam items: list of items ICS id linked to the host
:jsonparam ansible_vars: Ansible variables
:jsonparam ansible_groups: list of Ansible groups names
"""
host = models.Host.query.get_or_404(host_id)
if not current_user.has_access_to_network(host.main_network):
raise CSEntryError("User doesn't have the required group", status_code=403)
# The method currently doesn't allow to update the host name
# If we do, we have to update all the linked interface name as well!
# Interfaces shall always start by the host name
allowed_fields = (
("device_type", models.DeviceType, "name"),
("is_ioc", bool, None),
("description", str, None),
("items", [models.Item], "ics_id"),
("ansible_vars", dict, None),
("ansible_groups", [models.AnsibleGroup], "name"),
)
return utils.update_generic_model(models.Host, host_id, allowed_fields)
@bp.route("/hosts/<int:host_id>", methods=["DELETE"]) @bp.route("/hosts/<int:host_id>", methods=["DELETE"])
@login_groups_accepted("admin") @login_groups_accepted("admin")
def delete_host(host_id): def delete_host(host_id):
...@@ -269,6 +492,18 @@ def create_domain(): ...@@ -269,6 +492,18 @@ def create_domain():
return utils.create_generic_model(models.Domain, mandatory_fields=("name",)) return utils.create_generic_model(models.Domain, mandatory_fields=("name",))
@bp.route("/domains/<int:domain_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_domain(domain_id):
"""Delete a domain
.. :quickref: Network; Delete a domain
:param domain_id: domain primary key
"""
return utils.delete_generic_model(models.Domain, domain_id)
@bp.route("/cnames") @bp.route("/cnames")
@login_required @login_required
def get_cnames(): def get_cnames():
...@@ -303,3 +538,15 @@ def create_cname(): ...@@ -303,3 +538,15 @@ def create_cname():
return utils.create_generic_model( return utils.create_generic_model(
models.Cname, mandatory_fields=("name", "interface_id") models.Cname, mandatory_fields=("name", "interface_id")
) )
@bp.route("/cnames/<int:cname_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_cname(cname_id):
"""Delete a cname
.. :quickref: Network; Delete a cname
:param cname_id: cname primary key
"""
return utils.delete_generic_model(models.Cname, cname_id)
app/api/user.py
View file @ 534cf643
...@@ -15,13 +15,13 @@ from flask_login import login_required, current_user ...@@ -15,13 +15,13 @@ from flask_login import login_required, current_user
from ..extensions import ldap_manager from ..extensions import ldap_manager
from ..decorators import login_groups_accepted from ..decorators import login_groups_accepted
from .. import utils, tokens, models from .. import utils, tokens, models
from .utils import get_generic_model, create_generic_model from .utils import get_generic_model
bp = Blueprint("user_api", __name__) bp = Blueprint("user_api", __name__)
@bp.route("/users") @bp.route("/users")
@login_required @login_groups_accepted("admin", "auditor")
def get_users(): def get_users():
"""Return users information """Return users information
...@@ -40,22 +40,6 @@ def get_user_profile(): ...@@ -40,22 +40,6 @@ def get_user_profile():
return jsonify(current_user.to_dict()), 200 return jsonify(current_user.to_dict()), 200
@bp.route("/users", methods=["POST"])
@login_groups_accepted("admin")
def create_user():
"""Create a new user
.. :quickref: User; Create new user
:jsonparam username: User's username
:jsonparam display_name: User's display name
:jsonparam email: User's email
"""
return create_generic_model(
models.User, mandatory_fields=("username", "display_name", "email")
)
@bp.route("/login", methods=["POST"]) @bp.route("/login", methods=["POST"])
def login(): def login():
"""Return a JSON Web Token """Return a JSON Web Token
......
app/api/utils.py
View file @ 534cf643
...@@ -12,6 +12,7 @@ This module implements useful functions for the API. ...@@ -12,6 +12,7 @@ This module implements useful functions for the API.
import urllib.parse import urllib.parse
import sqlalchemy as sa import sqlalchemy as sa
from flask import current_app, jsonify, request from flask import current_app, jsonify, request
from flask_login import current_user
from flask_sqlalchemy import Pagination from flask_sqlalchemy import Pagination
from ..extensions import db from ..extensions import db
from .. import utils from .. import utils
...@@ -20,7 +21,7 @@ from .. import utils ...@@ -20,7 +21,7 @@ from .. import utils
def commit(): def commit():
try: try:
db.session.commit() db.session.commit()
except (sa.exc.IntegrityError, sa.exc.DataError) as e: except sa.exc.SQLAlchemyError as e:
db.session.rollback() db.session.rollback()
raise utils.CSEntryError(str(e), status_code=422) raise utils.CSEntryError(str(e), status_code=422)
...@@ -60,11 +61,12 @@ def build_pagination_header(pagination, base_url, **kwargs): ...@@ -60,11 +61,12 @@ def build_pagination_header(pagination, base_url, **kwargs):
return header return header
def get_generic_model(model, order_by=None, query=None): def get_generic_model(model, order_by=None, base_query=None, query=None):
"""Return data from model as json """Return data from model as json
:param model: model class :param model: model class
:param order_by: column to order the result by :param order_by: column to order the result by (not used if base_query or query is passed)
:param query: optional base_query to use [default: model.query]
:param query: optional query to use (for more complex queries) :param query: optional query to use (for more complex queries)
:returns: data from model as json :returns: data from model as json
""" """
...@@ -75,10 +77,12 @@ def get_generic_model(model, order_by=None, query=None): ...@@ -75,10 +77,12 @@ def get_generic_model(model, order_by=None, query=None):
# to query.filter_by in get_query # to query.filter_by in get_query
recursive = kwargs.pop("recursive", "false").lower() == "true" recursive = kwargs.pop("recursive", "false").lower() == "true"
if query is None: if query is None:
query = utils.get_query(model.query, **kwargs) if base_query is None:
if order_by is None: base_query = model.query
order_by = getattr(model, "name") if order_by is None:
query = query.order_by(order_by) order_by = getattr(model, "name")
base_query = base_query.order_by(order_by)
query = utils.get_query(base_query, model, **kwargs)
pagination = query.paginate( pagination = query.paginate(
page, per_page, max_per_page=current_app.config["MAX_PER_PAGE"] page, per_page, max_per_page=current_app.config["MAX_PER_PAGE"]
) )
...@@ -89,10 +93,11 @@ def get_generic_model(model, order_by=None, query=None): ...@@ -89,10 +93,11 @@ def get_generic_model(model, order_by=None, query=None):
return jsonify(data), 200, header return jsonify(data), 200, header
def search_generic_model(model): def search_generic_model(model, filter_sensitive=False):
"""Return filtered data from model as json """Return filtered data from model as json
:param model: model class :param model: model class
:param bool filter_sensitive: filter out sensitive data if set to True
:returns: filtered data from model as json :returns: filtered data from model as json
""" """
kwargs = request.args.to_dict() kwargs = request.args.to_dict()
...@@ -100,7 +105,9 @@ def search_generic_model(model): ...@@ -100,7 +105,9 @@ def search_generic_model(model):
per_page = int(kwargs.pop("per_page", 20)) per_page = int(kwargs.pop("per_page", 20))
per_page = min(per_page, current_app.config["MAX_PER_PAGE"]) per_page = min(per_page, current_app.config["MAX_PER_PAGE"])
search = kwargs.get("q", "*") search = kwargs.get("q", "*")
instances, nb_filtered = model.search(search, page=page, per_page=per_page) instances, nb_filtered = model.search(
search, page=page, per_page=per_page, filter_sensitive=filter_sensitive
)
current_app.logger.debug( current_app.logger.debug(
f'Found {nb_filtered} {model.__tablename__}(s) when searching "{search}"' f'Found {nb_filtered} {model.__tablename__}(s) when searching "{search}"'
) )
...@@ -110,11 +117,28 @@ def search_generic_model(model): ...@@ -110,11 +117,28 @@ def search_generic_model(model):
return jsonify(data), 200, header return jsonify(data), 200, header
def create_generic_model(model, mandatory_fields=("name",), **kwargs): def get_json_body():
"""Return the json body from the current request
Raise a CSEntryError if the body is not a json object
"""
data = request.get_json() data = request.get_json()
if data is None: if data is None:
raise utils.CSEntryError("Body should be a JSON object") raise utils.CSEntryError("Body should be a JSON object")
current_app.logger.debug(f"Received: {data}") if not data:
raise utils.CSEntryError("At least one field is required", status_code=422)
return data
def create_generic_model(model, mandatory_fields=("name",), **kwargs):
"""Create an instance of the model
:param model: model class
:param mandatory_fields: list of fields that shall be passed in the body
:param kwargs: extra fields to use
:returns: representation of the created instance as json
"""
data = get_json_body()
data.update(kwargs) data.update(kwargs)
for mandatory_field in mandatory_fields: for mandatory_field in mandatory_fields:
if mandatory_field not in data: if mandatory_field not in data:
...@@ -130,6 +154,9 @@ def create_generic_model(model, mandatory_fields=("name",), **kwargs): ...@@ -130,6 +154,9 @@ def create_generic_model(model, mandatory_fields=("name",), **kwargs):
raise utils.CSEntryError(str(e), status_code=422) raise utils.CSEntryError(str(e), status_code=422)
db.session.add(instance) db.session.add(instance)
commit() commit()
current_app.logger.info(
f"New {model.__tablename__} created by {current_user}: {instance.to_dict()}"
)
return jsonify(instance.to_dict()), 201 return jsonify(instance.to_dict()), 201
...@@ -141,5 +168,40 @@ def delete_generic_model(model, primary_key): ...@@ -141,5 +168,40 @@ def delete_generic_model(model, primary_key):
""" """
instance = model.query.get_or_404(primary_key) instance = model.query.get_or_404(primary_key)
db.session.delete(instance) db.session.delete(instance)
db.session.commit() commit()
current_app.logger.info(
f"{model.__tablename__} {instance} deleted by {current_user}"
)
return jsonify(), 204 return jsonify(), 204
def update_generic_model(model, primary_key, allowed_fields):
"""Update the model based on the primary_key
:param model: model class
:param primary_key: primary key of the instance to update
:param allowed_fields: list of fields that can be updated
:returns: representation of the updated model as json
"""
data = get_json_body()
# Use a list and not a set because the order is important
allowed_keys = [field[0] for field in allowed_fields]
for key in data:
if key not in allowed_keys:
raise utils.CSEntryError(f"Invalid field '{key}'", status_code=422)
instance = model.query.get_or_404(primary_key)
data = utils.convert_to_models(data, allowed_fields)
try:
# Loop on allowed_keys and not data to respect
# the order in which the fields are set
# setting ip before network is important for Interface
for key in allowed_keys:
if key in data:
setattr(instance, key, data[key])
except Exception as e:
raise utils.CSEntryError(str(e), status_code=422)
commit()
current_app.logger.info(
f"{model.__tablename__} {primary_key} updated by {current_user}: {instance.to_dict()}"
)
return jsonify(instance.to_dict()), 200
app/commands.py
View file @ 534cf643
...@@ -9,6 +9,7 @@ This module defines extra flask commands. ...@@ -9,6 +9,7 @@ This module defines extra flask commands.
:license: BSD 2-Clause, see LICENSE for more details. :license: BSD 2-Clause, see LICENSE for more details.
""" """
import click
import ldap3 import ldap3
import redis import redis
import rq import rq
...@@ -22,6 +23,15 @@ from .tasks import TaskWorker ...@@ -22,6 +23,15 @@ from .tasks import TaskWorker
from . import models, utils, tokens from . import models, utils, tokens
def disable_user(user):
"""Clear users'groups, email and tokens"""
user.groups = []
user.email = ""
# Revoke all user's tokens
for token in user.tokens:
db.session.delete(token)
def sync_user(connection, user): def sync_user(connection, user):
"""Synchronize the user from the database with information from the LDAP server""" """Synchronize the user from the database with information from the LDAP server"""
search_attr = current_app.config.get("LDAP_USER_LOGIN_ATTR") search_attr = current_app.config.get("LDAP_USER_LOGIN_ATTR")
...@@ -33,26 +43,35 @@ def sync_user(connection, user): ...@@ -33,26 +43,35 @@ def sync_user(connection, user):
search_scope=getattr(ldap3, current_app.config.get("LDAP_USER_SEARCH_SCOPE")), search_scope=getattr(ldap3, current_app.config.get("LDAP_USER_SEARCH_SCOPE")),
attributes=current_app.config.get("LDAP_GET_USER_ATTRIBUTES"), attributes=current_app.config.get("LDAP_GET_USER_ATTRIBUTES"),
) )
if len(connection.response) == 1: results = [
ldap_user = connection.response[0] result for result in connection.response if result["type"] == "searchResEntry"
attributes = ldap_user["attributes"] ]
user.display_name = utils.attribute_to_string(attributes["cn"]) if len(results) == 1:
user.email = utils.attribute_to_string(attributes["mail"]) ldap_user = results[0]
groups = ldap_manager.get_user_groups( # OU=InActiveUsers is specific to ESS AD
dn=ldap3.utils.conv.escape_filter_chars(ldap_user["dn"]), if "OU=InActiveUsers" in ldap_user["dn"]:
_connection=connection, current_app.logger.info(f"{user} is inactive. User disabled.")
) disable_user(user)
user.groups = sorted( else:
[utils.attribute_to_string(group["cn"]) for group in groups] attributes = ldap_user["attributes"]
) user.display_name = (
current_app.logger.info(f"{user} updated") utils.attribute_to_string(attributes["cn"]) or user.username
)
user.email = utils.attribute_to_string(attributes["mail"])
groups = ldap_manager.get_user_groups(
dn=ldap3.utils.conv.escape_filter_chars(ldap_user["dn"]),
_connection=connection,
)
user.groups = sorted(
[utils.attribute_to_string(group["cn"]) for group in groups]
)
current_app.logger.info(f"{user} updated")
elif len(results) == 0:
current_app.logger.warning(f"{user} not found! User disabled.")
disable_user(user)
else: else:
# Clear user's groups current_app.logger.warning(f"Too many results for {user}!")
user.groups = [] current_app.logger.warning(f"results: {results}")
# Revoke all user's tokens
for token in user.tokens:
db.session.delete(token)
current_app.logger.info(f"{user} disabled")
return user return user
...@@ -71,6 +90,21 @@ def sync_users(): ...@@ -71,6 +90,21 @@ def sync_users():
db.session.commit() db.session.commit()
def clean_deferred_tasks():
"""Set all deferred tasks to failed"""
for task in (
models.Task.query.filter_by(status=models.JobStatus.DEFERRED)
.order_by(models.Task.created_at)
.all()
):
if task.depends_on is None or task.depends_on.status == models.JobStatus.FAILED:
current_app.logger.info(
f"Set deferred task {task.id} ({task.name}) to failed"
)
task.status = models.JobStatus.FAILED
db.session.commit()
def register_cli(app): def register_cli(app):
@app.cli.command() @app.cli.command()
def create_defaults(): def create_defaults():
...@@ -83,6 +117,11 @@ def register_cli(app): ...@@ -83,6 +117,11 @@ def register_cli(app):
db.session.rollback() db.session.rollback()
app.logger.debug(f"{instance} already exists") app.logger.debug(f"{instance} already exists")
@app.cli.command()
def clean_deferred():
"""Set deferred tasks to failed if the task it depends on failed"""
clean_deferred_tasks()
@app.cli.command() @app.cli.command()
def syncusers(): def syncusers():
"""Synchronize all users from the database with information the LDAP server""" """Synchronize all users from the database with information the LDAP server"""
...@@ -102,7 +141,7 @@ def register_cli(app): ...@@ -102,7 +141,7 @@ def register_cli(app):
@app.cli.command() @app.cli.command()
def runworker(): def runworker():
"""Run RQ worker""" """Run RQ worker"""
redis_url = current_app.config["REDIS_URL"] redis_url = current_app.config["RQ_REDIS_URL"]
redis_connection = redis.from_url(redis_url) redis_connection = redis.from_url(redis_url)
with rq.Connection(redis_connection): with rq.Connection(redis_connection):
worker = TaskWorker(["high", "normal", "low"]) worker = TaskWorker(["high", "normal", "low"])
...@@ -115,8 +154,24 @@ def register_cli(app): ...@@ -115,8 +154,24 @@ def register_cli(app):
worker.work() worker.work()
@app.cli.command() @app.cli.command()
def reindex(): @click.option(
"--delete/--no-delete",
default=True,
help="Delete and recreate the index [default: True]",
)
@click.option(
"--name", default="all", help="name of the class to reindex [default: all]"
)
def reindex(delete, name):
"""Initialize the elasticsearch index""" """Initialize the elasticsearch index"""
current_app.elasticsearch.indices.delete("*", ignore=404) if delete:
models.Item.reindex() current_app.elasticsearch.indices.delete("*", ignore=404)
models.Host.reindex() if name == "all":
models.Item.reindex(delete)
models.Host.reindex(delete)
models.AnsibleGroup.reindex(delete)
return
try:
getattr(models, name).reindex(delete)
except AttributeError:
click.echo(f"The class {name} can't be indexed")
app/decorators.py
View file @ 534cf643
...@@ -21,6 +21,7 @@ def login_groups_accepted(*groups): ...@@ -21,6 +21,7 @@ def login_groups_accepted(*groups):
This can be used for users logged in using a cookie (web UI) or JWT (API). This can be used for users logged in using a cookie (web UI) or JWT (API).
Example:: Example::
@bp.route('/models', methods=['POST']) @bp.route('/models', methods=['POST'])
@login_groups_accepted('admin', 'inventory') @login_groups_accepted('admin', 'inventory')
def create_model(): def create_model():
......
app/defaults.py
View file @ 534cf643
...@@ -23,7 +23,10 @@ defaults = [ ...@@ -23,7 +23,10 @@ defaults = [
models.DeviceType(name="PhysicalMachine"), models.DeviceType(name="PhysicalMachine"),
models.DeviceType(name="VirtualMachine"), models.DeviceType(name="VirtualMachine"),
models.DeviceType(name="Network"), models.DeviceType(name="Network"),
models.DeviceType(name="MicroTCA"), models.DeviceType(name="MTCA-AMC"),
models.DeviceType(name="MTCA-IFC"),
models.DeviceType(name="MTCA-MCH"),
models.DeviceType(name="MTCA-RTM"),
models.DeviceType(name="VME"), models.DeviceType(name="VME"),
models.DeviceType(name="PLC"), models.DeviceType(name="PLC"),
] ]
app/factory.py
View file @ 534cf643
...@@ -48,6 +48,7 @@ from .api.user import bp as user_api ...@@ -48,6 +48,7 @@ from .api.user import bp as user_api
from .api.inventory import bp as inventory_api from .api.inventory import bp as inventory_api
from .api.network import bp as network_api from .api.network import bp as network_api
from .commands import register_cli from .commands import register_cli
from ._version import __version__
from . import utils from . import utils
...@@ -61,6 +62,7 @@ def create_app(config=None): ...@@ -61,6 +62,7 @@ def create_app(config=None):
app.jinja_env.filters["datetimeformat"] = utils.format_datetime app.jinja_env.filters["datetimeformat"] = utils.format_datetime
app.jinja_env.filters["toyaml"] = utils.pretty_yaml app.jinja_env.filters["toyaml"] = utils.pretty_yaml
app.jinja_env.globals["__version__"] = __version__
if app.config["SENTRY_DSN"]: if app.config["SENTRY_DSN"]:
sentry_sdk.init( sentry_sdk.init(
...@@ -70,19 +72,7 @@ def create_app(config=None): ...@@ -70,19 +72,7 @@ def create_app(config=None):
integrations=[FlaskIntegration()], integrations=[FlaskIntegration()],
) )
if not app.debug: app.logger.setLevel(logging.DEBUG)
# Log to stderr
handler = logging.StreamHandler()
handler.setFormatter(
logging.Formatter(
"%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
)
)
# Set app logger level to DEBUG
# otherwise only WARNING and above are propagated
app.logger.setLevel(logging.DEBUG)
handler.setLevel(logging.DEBUG)
app.logger.addHandler(handler)
app.logger.info("CSEntry created!") app.logger.info("CSEntry created!")
# Remove variables that contain a password # Remove variables that contain a password
settings_to_display = { settings_to_display = {
...@@ -105,6 +95,9 @@ def create_app(config=None): ...@@ -105,6 +95,9 @@ def create_app(config=None):
) )
app.logger.info(f"Settings:\n{settings_string}") app.logger.info(f"Settings:\n{settings_string}")
# Force RQ_DASHBOARD_REDIS_URL to RQ_REDIS_URL
app.config["RQ_DASHBOARD_REDIS_URL"] = app.config["RQ_REDIS_URL"]
db.init_app(app) db.init_app(app)
migrate.init_app(app) migrate.init_app(app)
login_manager.init_app(app) login_manager.init_app(app)
......
app/fields.py
View file @ 534cf643
...@@ -32,11 +32,13 @@ class YAMLField(TextAreaField): ...@@ -32,11 +32,13 @@ class YAMLField(TextAreaField):
return yaml.safe_dump(self.data, default_flow_style=False) if self.data else "" return yaml.safe_dump(self.data, default_flow_style=False) if self.data else ""
def process_formdata(self, valuelist): def process_formdata(self, valuelist):
if valuelist: if valuelist and valuelist[0].strip() != "":
try: try:
self.data = yaml.safe_load(valuelist[0]) self.data = yaml.safe_load(valuelist[0])
except yaml.YAMLError: except yaml.YAMLError:
raise ValueError("This field contains invalid YAML") raise ValueError("This field contains invalid YAML")
if not isinstance(self.data, dict):
raise ValueError("This field shall only contain key-value-pairs")
else: else:
self.data = None self.data = None
......
app/inventory/forms.py
View file @ 534cf643
...@@ -31,7 +31,10 @@ class ItemForm(CSEntryForm): ...@@ -31,7 +31,10 @@ class ItemForm(CSEntryForm):
"ICS id", "ICS id",
validators=[ validators=[
validators.InputRequired(), validators.InputRequired(),
validators.Regexp(ICS_ID_RE), validators.Regexp(
ICS_ID_RE,
message="The ICS id shall be composed of 3 letters and 3 digits.",
),
Unique(models.Item, "ics_id"), Unique(models.Item, "ics_id"),
], ],
) )
...@@ -48,7 +51,7 @@ class ItemForm(CSEntryForm): ...@@ -48,7 +51,7 @@ class ItemForm(CSEntryForm):
parent_id = SelectField("Parent", coerce=utils.coerce_to_str_or_none) parent_id = SelectField("Parent", coerce=utils.coerce_to_str_or_none)
host_id = SelectField("Host", coerce=utils.coerce_to_str_or_none) host_id = SelectField("Host", coerce=utils.coerce_to_str_or_none)
stack_member = NoValidateSelectField( stack_member = NoValidateSelectField(
"Stack member", coerce=utils.coerce_to_str_or_none, choices=[] "Stack member", coerce=utils.coerce_to_int_or_none, choices=[]
) )
mac_addresses = StringField( mac_addresses = StringField(
"MAC addresses", "MAC addresses",
......