Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • andersharrisson/csentry
  • ics-infrastructure/csentry
2 results
Show changes
Commits on Source (223)
123 additional commits have been omitted to prevent performance issues.
Hide whitespace changes
Inline Side-by-side
Showing
with 1015 additions and 236 deletions
.gitignore
View file @ 534cf643
......@@ -14,3 +14,5 @@ junit.xml
.tower_cli.cfg
/app/static/files/*
!/app/static/files/.empty
.vscode
.mypy_cache
.gitlab-ci.yml
View file @ 534cf643
image: docker:latest
.runner_tags: &runner_tags
tags:
- docker
---
include:
- remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/PreCommit.gitlab-ci.yml"
- remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/SonarScanner.gitlab-ci.yml"
variables:
CONTAINER_TEST_IMAGE: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME"
......@@ -20,38 +19,34 @@ stages:
- release
- deploy
default:
tags:
- docker
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
check:
<<: *runner_tags
stage: check
image: registry.esss.lu.se/ics-docker/pre-commit:latest
before_script: []
script:
- pre-commit run --all-files
build:
<<: *runner_tags
stage: build
image: docker:latest
script:
- docker pull "$CONTAINER_CACHE_IMAGE" || true
- docker build --pull --cache-from "$CONTAINER_CACHE_IMAGE" -t "$CONTAINER_TEST_IMAGE" .
- docker push "$CONTAINER_TEST_IMAGE"
test:
<<: *runner_tags
stage: test
image: "$CONTAINER_TEST_IMAGE"
services:
- postgres:10
- redis:4.0
- name: docker.elastic.co/elasticsearch/elasticsearch:6.4.2
- name: docker.elastic.co/elasticsearch/elasticsearch:7.3.2
alias: elasticsearch
command: ["bin/elasticsearch", "-Ediscovery.type=single-node"]
before_script:
- pip install -r requirements-dev.txt
script:
- pytest --junitxml=junit.xml --cov-report xml:coverage.xml --cov=app -v
- pytest --junitxml=junit.xml --cov-report=xml:coverage.xml --cov-report=term --cov=app -v
artifacts:
paths:
- junit.xml
......@@ -60,17 +55,9 @@ test:
junit: junit.xml
expire_in: 1 hour
analyse:
<<: *runner_tags
stage: analyse
image: registry.esss.lu.se/ics-docker/sonar-scanner:3
before_script: []
script:
- sonar-scanner -Dsonar.login=$SONARQUBE_TOKEN -Dsonar.projectVersion=$CI_COMMIT_REF_NAME
release-image:
<<: *runner_tags
stage: release
image: docker:latest
dependencies: []
script:
- docker pull "$CONTAINER_TEST_IMAGE"
......@@ -79,10 +66,28 @@ release-image:
only:
- tags
deploy-dev:
stage: deploy
image: registry.esss.lu.se/ics-docker/tower-cli
before_script: []
dependencies: []
script:
- >
tower-cli job launch
-h torn.tn.esss.lu.se
-t ${TOWER_OAUTH_TOKEN}
-J deploy-csentry-dev
-e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
environment:
name: dev
url: https://csentry-lab-01.cslab.esss.lu.se
except:
- master
- tags
deploy-staging:
<<: *runner_tags
stage: deploy
image: registry.esss.lu.se/ics-docker/tower-cli:3.3
image: registry.esss.lu.se/ics-docker/tower-cli
before_script: []
dependencies: []
script:
......@@ -100,7 +105,6 @@ deploy-staging:
- tags
pages:
<<: *runner_tags
stage: deploy
image: "$CONTAINER_RELEASE_IMAGE"
dependencies: []
......@@ -118,9 +122,8 @@ pages:
- tags
deploy-production:
<<: *runner_tags
stage: deploy
image: registry.esss.lu.se/ics-docker/tower-cli:3.3
image: registry.esss.lu.se/ics-docker/tower-cli
before_script: []
dependencies: []
script:
......
.pre-commit-config.yaml
View file @ 534cf643
repos:
- repo: https://github.com/ambv/black
rev: stable
- repo: https://github.com/psf/black
rev: 22.6.0
hooks:
- id: black
language_version: python3.6
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v1.3.0
- repo: https://github.com/pycqa/flake8
rev: 4.0.1
hooks:
- id: flake8
CHANGELOG.rst
View file @ 534cf643
Changelog
=========
Version 2021.04.06
------------------
- Validate group names (INFRA-3135)
- Add warning text for creating vm (INFRA-3292)
- Paginate history in ansible groups and hosts (INFRA-3136)
Version 2020.11.25
------------------
- Add documentation about database versioning
- Set display_name to username when empty (INFRA-2909)
- Improve logging (INFRA-2899)
Version 2020.11.13
------------------
- Increased hostname character limit from 20 -> 24 (INFRA-2808)
- Use SSL by default for LDAP connection (INFRA-2805)
- Use SonarScanner.gitlab-ci.yml template (INFRA-2832)
Version 2020.10.30
------------------
- Fix inventory export to excel (INFRA-2780)
- Add developer documentation
Version 2020.10.27
------------------
- Update Python and requirements (INFRA-2649)
- Update RQ and rq-dashboard (INFRA-2649 / INFRA-2754)
- Fix code smells (INFRA-2729)
- Add auditor group (INFRA-2728)
- Update Sphinx and switch to sphinx_rtd_theme (INFRA-2742)
Version 2020.10.06
------------------
- Make sensitive hosts visible for members of the scope (INFRA-2508)
- Only hide sensitive networks not part of the user scope (INFRA-2437)
- Automatically set network_scope group as parent of network groups (INFRA-2639)
Version 2020.07.08
------------------
- Add view to edit network scope (INFRA-2316)
- Add API endpoint to patch network scope (INFRA-2316)
Version 2020.06.12
------------------
- Change MTCA-IOxOS to MTCA-IFC (INFRA-2200)
- Add description field on Interface (INFRA-2112)
Version 2020.04.16
------------------
- Allow /network/networks API endpoint for normal users (INFRA-2013)
- Update Bootstrap to 4.4.1 (INFRA-2021)
- Update DataTables (INFRA-2021)
- Add tooltip on search box (INFRA-1893)
Version 2020.03.17
------------------
- Add extra badges to README and documentation
- Fix get_hosts for normal users (INFRA-1888)
Version 2020.03.09
------------------
- Trigger core services update on host change (INFRA-1846)
- Add DELETE method on all API endpoints (INFRA-1853)
- Remove create_user API endpoint (INFRA-1854)
Version 2020.03.05
------------------
- Allow users to view networks (INFRA-1809)
- Add HOSTNAME Ansible group type (INFRA-1844)
- Remove IOC git repository creation (INFRA-1839)
- Make IP optional when creating interface via the API (INFRA-1833)
- Introduce new device types for MTCA (INFRA-1835)
Version 2020.02.25
------------------
- Fix inventory update (INFRA-1792)
Version 2020.02.24
------------------
- Add app version in navbar (INFRA-1789)
- Allow to pass LDAP settings as env variables (INFRA-1788)
- Implement server side processing for Ansible groups (INFRA-1790)
Version 0.29.0 (2020-02-03)
---------------------------
- Add MAC address column to hosts table (INFRA-1669)
- Restrict view of networks and scopes to admin only (INFRA-1671)
- Add sensitive field to Network class (INFRA-1671)
- Filter out sensitive hosts for non admin users (INFRA-1671)
- Add API endpoint and view to update network (INFRA-1724)
- Make pages title more explicit (INFRA-1726)
Version 0.28.0 (2019-12-19)
---------------------------
- Fix exception when setting new stack_member (INFRA-1648)
- Prevent recursive dependency loop in Ansible groups (INFRA-1622)
- Prevent network and scope overlapping (INFRA-1627)
- Add netmask to network view (INFRA-1660)
Version 0.27.1 (2019-12-04)
---------------------------
- Fix regexp for ICS id validation (INFRA-1569)
- Allow normal users to edit hosts with no interface (INFRA-1604)
Version 0.27.0 (2019-10-30)
---------------------------
- Increase interface name max length (INFRA-1324)
- Add endpoints to patch host and interface (INFRA-1406)
- Limit networks to same scope for extra interfaces (INFRA-1297)
Version 0.26.1 (2019-09-18)
---------------------------
- Fix inventory update not triggered (INFRA-1290)
Version 0.26.0 (2019-09-18)
---------------------------
- Decouple inventory and core services update (INFRA-1290)
- Add broadcast address to network view (INFRA-1291)
- Add view network scope page (INFRA-1292)
- Make ansible_vars in host searchable (INFRA-1295)
- Remove Generate ZTP configuration function (INFRA-1299)
Version 0.25.1 (2019-08-30)
---------------------------
- Prevent input of invalid Ansible variables (INFRA-1221)
- Update black and pre-commit config
- Add pytest.ini to speed test discovery
Version 0.25.0 (2019-08-22)
---------------------------
- Implement user access on network scope instead of domain (INFRA-1228)
- Create IOC repositories with same path as name (INFRA-1230)
Version 0.24.1 (2019-07-10)
---------------------------
- Fix sorting of items with some null stack_member (INFRA-1112)
Version 0.24.0 (2019-07-10)
---------------------------
- Add serial_number and stack_member to hosts json model (INFRA-1111)
Version 0.23.0 (2019-06-14)
---------------------------
- Increase RQ default timeout (INFRA-1051)
Version 0.22.0 (2019-06-14)
---------------------------
- Make vlan optional in NetworkScope and Network (INFRA-1049)
- Update reverse dependencies of failed tasks (INFRA-1051)
Version 0.21.1 (2019-05-13)
---------------------------
- Fix users synchronization (INFRA-1026)
- Remove caching of user retrieval (INFRA-1025)
Version 0.21.0 (2019-05-10)
---------------------------
- Improve documentation about ansible-vault
- Fix IOC host indexation (INFRA-1015)
- Allow users to delete their host (INFRA-1018)
- Fix job.status DeprecationWarning (INFRA-1021)
- Allow service users to login (INFRA-1022)
Version 0.20.2 (2019-05-08)
---------------------------
- Fix IOC repository creation (INFRA-1015)
- Save ansible var when setting boot profile (INFRA-1016)
Version 0.20.1 (2019-04-25)
---------------------------
- Save the device_type_id in the session (INFRA-987)
- Fix exception when no network selected (INFRA-810)
- Remove graylog support in uwsgi (INFRA-981)
- Replace raven with sentry-sdk (INFRA-979)
Version 0.20.0 (2019-04-05)
---------------------------
- Add API endpoint to search hosts (INFRA-931)
- Initialize IOC repository on GitLab (INFRA-932)
- Limit the max number of elements returned per page (INFRA-942)
- Allow to set the boot profile from CSEntry (INFRA-943)
Version 0.19.2 (2019-03-27)
---------------------------
- Fix model update in admin interface (INFRA-908)
- Increase timeout for core services update (INFRA-895)
- Add depends_on field on Task (INFRA-895)
Version 0.19.1 (2019-03-18)
---------------------------
- Fix excel file download (INFRA-890)
- Skip post install for windows VM creation (INFRA-877)
Version 0.19.0 (2019-03-18)
---------------------------
- Add osversion field to CreateVMForm (INFRA-877)
- Fix link to AWX job id for workflow jobs (INFRA-886)
- Allow users to create VM/VIOC (INFRA-775)
- Trigger AWX inventory update on database change (INFRA-887)
Version 0.18.2 (2019-03-07)
---------------------------
- Fix post install job trigger (INFRA-870)
Version 0.18.1 (2019-03-07)
---------------------------
- Clean extra vars passed for VM and VIOC creation (INFRA-870)
Version 0.18.0 (2019-03-05)
---------------------------
- Add gateway field to network table (INFRA-809)
- Add view network page (INFRA-860)
- Remove tags and add is_ioc field to host table (INFRA-862)
- Allow to launch a workflow job (INFRA-867)
Version 0.17.1 (2019-02-07)
---------------------------
- Catch exception raised by Unique Validator (INFRA-777)
- Only check the main interface in dynamic groups (INFRA-784)
- Make sure the first interface is the main one (INFRA-785)
Version 0.17.0 (2019-01-28)
---------------------------
- Implement new layout (INFRA-763)
- Fix case insensitive search (INFRA-770)
- Add post install job after VM creation (INFRA-769)
Version 0.16.0 (2019-01-23)
---------------------------
- Explicitely define the elasticsearch mapping (INFRA-723)
- Add favicon (INFRA-725)
- Pass disk size in inventory when creating VM (INFRA-759)
Version 0.15.1 (2018-12-10)
---------------------------
- Fix extra interface creation (INFRA-697)
Version 0.15.0 (2018-11-30)
---------------------------
- Disable DHCP & DNS update on host modification (INFRA-673)
- Use different groups for inventory and network (INFRA-578)
- Add network permissions per domain (INFRA-578)
- Move mac API to inventory endpoint (INFRA-674)
- Add button to delete Ansible group (INFRA-678)
- Save VM memory and cores as ansible variables (INFRA-640)
- Return host fqdn in Ansible groups (INFRA-640)
- Add extra fields to the API (INFRA-640)
- Update Python to 3.7.1 and dependencies (INFRA-684)
- Fix WhiteNoise used static directory (INFRA-683)
- Fix flask subcommand name (INFRA-687)
Version 0.14.0 (2018-11-08)
---------------------------
- Fix TypeError in after_commit (INFRA-613)
- Add zabbix plugin to uwsgi (INFRA-614)
- Allow to edit and delete comments (INFRA-618)
- Make mac addresses unique by interface (INFRA-639)
- Replace bootstrap-select with selectize (INFRA-644)
- Make host/interface/cname unique (INFRA-245)
- Switch to new template to deploy VM in proxmox (INFRA-651)
Version 0.13.0 (2018-10-12)
---------------------------
......
Dockerfile
View file @ 534cf643
FROM python:3.6-slim as base
FROM python:3.8-slim as base
# Add ESS specific debian repository mirror
RUN echo "deb https://artifactory.esss.lu.se/artifactory/debian-mirror stable main" | tee /etc/apt/sources.list.d/ess-debian-mirror.list
# Install Python dependencies in an intermediate image
# as some requires a compiler (uwsgi)
# as some requires a compiler (psycopg2)
FROM base as builder
# Install dependencies required to compile some Python packages
......@@ -34,15 +37,10 @@ RUN apt-get update \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Compile uwsgi with graylog2 plugin
RUN wget https://projects.unbit.it/downloads/uwsgi-2.0.17.1.tar.gz \
&& tar xfz uwsgi-2.0.17.1.tar.gz \
&& cd uwsgi-2.0.17.1 \
&& UWSGI_EMBED_PLUGINS=graylog2,zabbix python uwsgiconfig.py --build
COPY requirements.txt /requirements.txt
RUN python -m venv /venv \
&& . /venv/bin/activate \
&& pip install --no-cache-dir --upgrade pip \
&& pip install --no-cache-dir -r /requirements.txt
ARG CSENTRY_BUILD
......@@ -55,9 +53,8 @@ RUN groupadd -r -g 1000 csi \
&& useradd --no-log-init -r -g csi -u 1000 csi
COPY --chown=csi:csi --from=builder /venv /venv
COPY --chown=csi:csi --from=builder /uwsgi-2.0.17.1/uwsgi /venv/bin/uwsgi
# Install libraries for psycopg2, pillow and uwsgi
# Install libraries for psycopg2 and pillow
# Shall be the same as the one linked to when compiling in builder image!
RUN apt-get update \
&& apt-get install -yq --no-install-recommends \
......@@ -67,12 +64,18 @@ RUN apt-get update \
libpq5 \
libpcre3 \
zlib1g \
git \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
RUN rm /etc/apt/sources.list.d/ess-debian-mirror.list
COPY --chown=csi:csi . /app/
WORKDIR /app
RUN echo "__version__ = \"$(git describe)\"" > app/_version.py
ENV PATH /venv/bin:$PATH
EXPOSE 8000
CMD ["flask", "run", "--host", "0.0.0.0", "--port", "8000"]
USER csi
Makefile
View file @ 534cf643
.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image
.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image docs
OWNER := registry.esss.lu.se/ics-infrastructure
GIT_TAG := $(shell git describe --always)
......@@ -42,17 +42,17 @@ db: ## start postgres and redis for development
init_db: ## initialize the dev database
docker-compose run --rm web flask db upgrade head
docker-compose run --rm web flask create_defaults
docker-compose run --rm web flask create-defaults
upgrade_db: ## upgrade the dev database
docker-compose run --rm web flask db upgrade head
test: ## run the tests (on current directory)
docker-compose run --rm web pytest --cov=app -v
docker-compose -f docker-compose.yml run --rm web pytest --cov=app -v
db_image: ## start postgres and redis to test the latest image
# Pass docker-compose.yml to skip docker-compose.override.yml
docker-compose -f docker-compose.yml up -d postgres redis
db_test: ## start required containers for test
# Pass docker-compose.yml to skip docker-compose.override.yml (db not mounted as volume for speed)
docker-compose -f docker-compose.yml up -d postgres redis elasticsearch worker
test_image: ## run the tests (on the latest image)
# Pass docker-compose.yml to skip docker-compose.override.yml
......@@ -60,3 +60,6 @@ test_image: ## run the tests (on the latest image)
run_uwsgi: ## run the application with uwsgi (to test prod env)
docker-compose run -p 8000:8000 --rm web uwsgi --master --http 0.0.0.0:8000 --http-keepalive --manage-script-name --mount /csentry=wsgi.py --callable app --uid conda --processes 2 -b 16384
docs: ## run the tests (on current directory)
docker run --rm -v $(shell pwd):/app registry.esss.lu.se/ics-infrastructure/csentry:master sphinx-build -M html docs docs/_build
README.rst
View file @ 534cf643
......@@ -4,6 +4,16 @@ CSEntry
.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
:target: https://github.com/ambv/black
.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=alert_status
:target: https://sonarqube.esss.lu.se/dashboard?id=csentry
.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=ncloc
:target: https://sonarqube.esss.lu.se/dashboard?id=csentry
.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/pipeline.svg
.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/coverage.svg
Control System Entry web server.
......@@ -80,16 +90,3 @@ To restore the database::
$ gunzip -c csentry_db.sql.gz | docker run --rm --link csentry_postgres:postgres --net csentry_default
-e PGPASSWORD="<csentry_password>" -i postgres:10 psql -h postgres -U ics csentry_db
Dependencies
------------
The initial dependencies were generated using::
$ docker run --rm -it -v $(pwd):/app continuumio/miniconda3:latest bash
$ conda config --add channels conda-forge
$ conda create -n csentry python=3.6 flask alembic flask-debugtoolbar flask-login flask-sqlalchemy flask-wtf pillow psycopg2 pytest pytest-cov qrcode whitenoise factory_boy flask-admin pyjwt ldap3 flask-mail flask-migrate flask-jwt-extended
$ source activate csentry
$ pip install flask-ldap3-login sqlalchemy-citext sqlalchemy-continuum pytest-factoryboy git+https://github.com/beenje/flask-bootstrap@4.0.0-beta.1.dev1
$ conda env export > /app/environment.yml
app/_version.py 0 → 100644
View file @ 534cf643
__version__ = "dev"
app/api/inventory.py
View file @ 534cf643
......@@ -13,7 +13,7 @@ from flask import Blueprint, jsonify, request, current_app
from flask_login import login_required
from .. import utils, models
from ..decorators import login_groups_accepted
from .utils import commit, create_generic_model, get_generic_model
from .utils import commit, create_generic_model, get_generic_model, delete_generic_model
bp = Blueprint("inventory_api", __name__)
......@@ -45,7 +45,7 @@ def get_items():
@bp.route("/items/<id_>")
@login_required
def get_item(id_):
"""Retrieve item by id or ICS id
r"""Retrieve item by id or ICS id
.. :quickref: Inventory; Get item by id or ICS id
......@@ -56,7 +56,7 @@ def get_item(id_):
@bp.route("/items", methods=["POST"])
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def create_item():
"""Register a new item
......@@ -81,9 +81,9 @@ def create_item():
@bp.route("/items/<id_>", methods=["PATCH"])
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def patch_item(id_):
"""Patch an existing item
r"""Patch an existing item
.. :quickref: Inventory; Update existing item
......@@ -140,10 +140,22 @@ def patch_item(id_):
return jsonify(item.to_dict())
@bp.route("/items/<int:item_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_item(item_id):
"""Delete an item
.. :quickref: Inventory; Delete an item
:param item_id: item primary key
"""
return delete_generic_model(models.Item, item_id)
@bp.route("/items/<id_>/comments")
@login_required
def get_item_comments(id_):
"""Get item comments
r"""Get item comments
.. :quickref: Inventory; Get item comments
......@@ -154,9 +166,9 @@ def get_item_comments(id_):
@bp.route("/items/<id_>/comments", methods=["POST"])
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def create_item_comment(id_):
"""Create a comment on item
r"""Create a comment on item
.. :quickref: Inventory; Create comment on item
......@@ -169,6 +181,18 @@ def create_item_comment(id_):
)
@bp.route("/items/comments/<int:comment_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_comment(comment_id):
"""Delete an item comment
.. :quickref: Inventory; Delete an item comment
:param comment_id: comment primary key
"""
return delete_generic_model(models.ItemComment, comment_id)
@bp.route("/actions")
@login_required
def get_actions():
......@@ -190,7 +214,7 @@ def get_manufacturers():
@bp.route("/manufacturers", methods=["POST"])
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def create_manufacturer():
"""Create a new manufacturer
......@@ -202,6 +226,18 @@ def create_manufacturer():
return create_generic_model(models.Manufacturer)
@bp.route("/manufacturers/<int:manufacturer_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_manufacturer(manufacturer_id):
"""Delete a manufacturer
.. :quickref: Inventory; Delete a manufacturer
:param manufacturer_id: manufacturer primary key
"""
return delete_generic_model(models.Manufacturer, manufacturer_id)
@bp.route("/models")
@login_required
def get_models():
......@@ -213,7 +249,7 @@ def get_models():
@bp.route("/models", methods=["POST"])
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def create_model():
"""Create a new model
......@@ -225,6 +261,18 @@ def create_model():
return create_generic_model(models.Model)
@bp.route("/models/<int:model_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_model(model_id):
"""Delete a model
.. :quickref: Inventory; Delete a model
:param model_id: model primary key
"""
return delete_generic_model(models.Model, model_id)
@bp.route("/locations")
@login_required
def get_locations():
......@@ -236,7 +284,7 @@ def get_locations():
@bp.route("/locations", methods=["POST"])
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def create_locations():
"""Create a new location
......@@ -248,6 +296,18 @@ def create_locations():
return create_generic_model(models.Location)
@bp.route("/locations/<int:location_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_location(location_id):
"""Delete a location
.. :quickref: Inventory; Delete a location
:param location_id: location primary key
"""
return delete_generic_model(models.Location, location_id)
@bp.route("/statuses")
@login_required
def get_status():
......@@ -259,7 +319,7 @@ def get_status():
@bp.route("/statuses", methods=["POST"])
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def create_status():
"""Create a new status
......@@ -269,3 +329,38 @@ def create_status():
:jsonparam description: (optional) description
"""
return create_generic_model(models.Status)
@bp.route("/statuses/<int:status_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_status(status_id):
"""Delete a status
.. :quickref: Inventory; Delete a status
:param status_id: status primary key
"""
return delete_generic_model(models.Status, status_id)
@bp.route("/macs")
@login_required
def get_macs():
"""Return mac addresses
.. :quickref: Inventory; Get mac addresses
"""
return get_generic_model(models.Mac, order_by=models.Mac.address)
@bp.route("/macs", methods=["POST"])
@login_groups_accepted("admin", "inventory")
def create_macs():
"""Create a new mac address
.. :quickref: Inventory; Create new mac address
:jsonparam address: MAC address
:jsonparam item_id: (optional) linked item primary key
"""
return create_generic_model(models.Mac, mandatory_fields=("address",))
app/api/network.py
View file @ 534cf643
......@@ -10,16 +10,18 @@ This module implements the network API.
"""
from flask import Blueprint, request
from flask_login import login_required
from flask_login import login_required, current_user
from wtforms import ValidationError
from .. import models
from ..decorators import login_groups_accepted
from ..utils import CSEntryError, validate_ip
from . import utils
bp = Blueprint("network_api", __name__)
@bp.route("/scopes")
@login_required
@login_groups_accepted("admin", "auditor")
def get_scopes():
"""Return network scopes
......@@ -50,14 +52,66 @@ def create_scope():
)
@bp.route("/scopes/<int:scope_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_scope(scope_id):
"""Delete a network scope
.. :quickref: Network; Delete a network scope
:param scope_id: network scope primary key
"""
return utils.delete_generic_model(models.NetworkScope, scope_id)
@bp.route("/scopes/<int:scope_id>", methods=["PATCH"])
@login_groups_accepted("admin")
def patch_scope(scope_id):
r"""Patch an existing network scope
.. :quickref: Network; Update an existing network scope
:param scope_id: network scope primary key
:jsonparam name: network scope name
:jsonparam description: description
:jsonparam first_vlan: network scope first vlan
:jsonparam last_vlan: network scope last vlan
:jsonparam supernet: network scope supernet
:jsonparam domain: name of the default domain
"""
allowed_fields = (
("name", str, None),
("description", str, None),
("first_vlan", int, None),
("last_vlan", int, None),
("supernet", str, None),
("domain", models.Domain, "name"),
)
return utils.update_generic_model(models.NetworkScope, scope_id, allowed_fields)
@bp.route("/networks")
@login_required
@login_groups_accepted("admin", "auditor", "network")
def get_networks():
"""Return networks
.. :quickref: Network; Get networks
"""
return utils.get_generic_model(models.Network, order_by=models.Network.address)
query = models.Network.query
if not (current_user.is_admin or current_user.is_auditor):
sensitive_networks = (
query.filter(models.Network.sensitive.is_(True))
.join(models.Network.scope)
.filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
)
query = (
query.filter(models.Network.sensitive.is_(False))
.union(sensitive_networks)
.distinct(models.Network.id)
.from_self()
)
query = query.order_by(models.Network.address)
return utils.get_generic_model(model=models.Network, base_query=query)
@bp.route("/networks", methods=["POST"])
......@@ -72,10 +126,13 @@ def create_network():
:jsonparam address: vlan address
:jsonparam first_ip: first IP of the allowed range
:jsonparam last_ip: last IP of the allowed range
:jsonparam gateway: gateway IP
:jsonparam scope: network scope name
:jsonparam domain_id: (optional) primary key of the domain [default: scope domain]
:jsonparam admin_only: (optional) boolean to restrict the network to admin users [default: False]
:type admin_only: bool
:jsonparam sensitive: hide the network and all hosts if True (for non admin) [default: False]
:type sensitive: bool
:jsonparam description: (optional) description
"""
return utils.create_generic_model(
......@@ -86,11 +143,60 @@ def create_network():
"address",
"first_ip",
"last_ip",
"gateway",
"scope",
),
)
@bp.route("/networks/<int:network_id>", methods=["PATCH"])
@login_groups_accepted("admin")
def patch_network(network_id):
r"""Patch an existing network
.. :quickref: Network; Update an existing network
:param network_id: network primary key
:jsonparam vlan_name: vlan name
:jsonparam address: vlan address
:jsonparam first_ip: first IP of the allowed range
:jsonparam last_ip: last IP of the allowed range
:jsonparam gateway: gateway IP
:jsonparam domain: domain name
:jsonparam admin_only: boolean to restrict the network to admin users
:type admin_only: bool
:jsonparam sensitive: hide the network and all hosts if True (for non admin)
:type sensitive: bool
:jsonparam description: description
"""
# The method currently doesn't allow to update the network_scope
# as this will have an impact on the address
allowed_fields = (
("vlan_name", str, None),
("address", str, None),
("first_ip", str, None),
("last_ip", str, None),
("gateway", str, None),
("domain", models.Domain, "name"),
("admin_only", bool, None),
("sensitive", bool, None),
("description", str, None),
)
return utils.update_generic_model(models.Network, network_id, allowed_fields)
@bp.route("/networks/<int:network_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_network(network_id):
"""Delete a network
.. :quickref: Network; Delete a network
:param network_id: network primary key
"""
return utils.delete_generic_model(models.Network, network_id)
@bp.route("/interfaces")
@login_required
def get_interfaces():
......@@ -98,50 +204,116 @@ def get_interfaces():
.. :quickref: Network; Get interfaces
"""
domain = request.args.get("domain", None)
if domain is not None:
query = models.Interface.query
query = models.Interface.query
query = query.join(models.Interface.network).order_by(models.Interface.ip)
if not (current_user.is_admin or current_user.is_auditor):
sensitive_interfaces = (
query.filter(models.Network.sensitive.is_(True))
.join(models.Network.scope)
.filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
)
query = (
query.join(models.Interface.network)
.join(models.Network.domain)
.filter(models.Domain.name == domain)
query.filter(models.Network.sensitive.is_(False))
.union(sensitive_interfaces)
.from_self()
.join(models.Interface.network)
.order_by(models.Interface.ip)
)
query = query.order_by(models.Interface.ip)
domain = request.args.get("domain", None)
if domain is not None:
query = query.join(models.Network.domain).filter(models.Domain.name == domain)
return utils.get_generic_model(model=None, query=query)
network = request.args.get("network", None)
if network is not None:
query = models.Interface.query
query = query.join(models.Interface.network).filter(
models.Network.vlan_name == network
)
query = query.order_by(models.Interface.ip)
query = query.filter(models.Network.vlan_name == network)
return utils.get_generic_model(model=None, query=query)
return utils.get_generic_model(models.Interface, order_by=models.Interface.ip)
return utils.get_generic_model(model=models.Interface, base_query=query)
@bp.route("/interfaces", methods=["POST"])
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "network")
def create_interface():
"""Create a new interface
.. :quickref: Network; Create new interface
:jsonparam network: network name
:jsonparam ip: interface IP
:jsonparam ip: (optional) interface IP - IP will be assigned automatically if not given
:jsonparam name: interface name
:jsonparam host: host name
:jsonparam mac: (optional) MAC address
:jsonparam host_id: (optional) linked host primary key
"""
# The validate_interfaces method from the Network class is called when
# setting interface.network. This is why we don't pass network_id here
# but network (as vlan_name string)
# Same for host
data = request.get_json()
# Check that the user has the permissions to create an interface on this network
try:
network = models.Network.query.filter_by(
vlan_name=data["network"]
).first_or_404()
except Exception:
# If we can't get a network, an error will be raised in create_generic_model
pass
else:
if not current_user.has_access_to_network(network):
raise CSEntryError("User doesn't have the required group", status_code=403)
return utils.create_generic_model(
models.Interface, mandatory_fields=("network", "ip", "name")
models.Interface, mandatory_fields=("network", "name", "host")
)
@bp.route("/interfaces/<int:interface_id>", methods=["PATCH"])
@login_groups_accepted("admin", "network")
def patch_interface(interface_id):
r"""Patch an existing interface
.. :quickref: Network; Update an existing interface
:param interface_id: interface primary key
:jsonparam ip: interface IP
:jsonparam name: interface name
:jsonparam network: network name
:jsonparam mac: MAC address
"""
interface = models.Interface.query.get_or_404(interface_id)
# User shall have access to both the current and new network (if provided)
if not current_user.has_access_to_network(interface.network):
raise CSEntryError("User doesn't have the required group", status_code=403)
data = utils.get_json_body()
if "network" in data:
new_network = models.Network.query.filter_by(
vlan_name=data["network"]
).first_or_404()
if not current_user.has_access_to_network(new_network):
raise CSEntryError("User doesn't have the required group", status_code=403)
# Ensure that the IP is in the network (current one or new one if passed)
# This is required because validate_interfaces is only called when a new network
# is assigned. So the check is needed even when new_network == interface.network
# (same network given in argument)
if "ip" in data:
try:
if "network" in data:
validate_ip(data["ip"], new_network)
else:
validate_ip(data["ip"], interface.network)
except ValidationError as e:
raise CSEntryError(str(e), status_code=422)
# The method doesn't allow to update the interface host
# I don't think it makes much sense (and an interface shall start by the host name)
# To add a new cname, use create_cname
allowed_fields = (
("ip", str, None),
("name", str, None),
("network", models.Network, "vlan_name"),
("mac", str, None),
)
return utils.update_generic_model(models.Interface, interface_id, allowed_fields)
@bp.route("/interfaces/<int:interface_id>", methods=["DELETE"])
@login_groups_accepted("admin")
@login_groups_accepted("admin", "network")
def delete_interface(interface_id):
"""Delete an interface
......@@ -177,6 +349,18 @@ def create_ansible_groups():
return utils.create_generic_model(models.AnsibleGroup, mandatory_fields=("name",))
@bp.route("/groups/<int:group_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_ansible_group(group_id):
"""Delete an Ansible group
.. :quickref: Network; Delete an Ansible group
:param group_id: Ansible group primary key
"""
return utils.delete_generic_model(models.AnsibleGroup, group_id)
@bp.route("/hosts")
@login_required
def get_hosts():
......@@ -184,11 +368,46 @@ def get_hosts():
.. :quickref: Network; Get hosts
"""
return utils.get_generic_model(models.Host, order_by=models.Host.name)
query = models.Host.query
if not (current_user.is_admin or current_user.is_auditor):
# Note that hosts without interface will be filtered out
# by this query. This is not an issue as hosts should always
# have an interface. They are useless otherwise.
non_sensitive_hosts = (
query.join(models.Host.interfaces)
.join(models.Interface.network)
.filter(models.Network.sensitive.is_(False))
)
sensitive_hosts = (
query.join(models.Host.interfaces)
.join(models.Interface.network)
.filter(models.Network.sensitive.is_(True))
.join(models.Network.scope)
.filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
)
query = (
non_sensitive_hosts.union(sensitive_hosts)
.distinct(models.Host.id)
.from_self()
)
query = query.order_by(models.Host.name)
return utils.get_generic_model(model=models.Host, base_query=query)
@bp.route("/hosts/search")
@login_required
def search_hosts():
"""Search hosts
.. :quickref: Network; Search hosts
:query q: the search query
"""
return utils.search_generic_model(models.Host, filter_sensitive=True)
@bp.route("/hosts", methods=["POST"])
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "network")
def create_host():
"""Create a new host
......@@ -196,6 +415,7 @@ def create_host():
:jsonparam name: hostname
:jsonparam device_type: Physical|Virtual|...
:jsonparam is_ioc: True|False (optional)
:jsonparam description: (optional) description
:jsonparam items: (optional) list of items ICS id linked to the host
:jsonparam ansible_vars: (optional) Ansible variables
......@@ -206,6 +426,38 @@ def create_host():
)
@bp.route("/hosts/<int:host_id>", methods=["PATCH"])
@login_groups_accepted("admin", "network")
def patch_host(host_id):
r"""Patch an existing host
.. :quickref: Network; Update an existing host
:param host_id: host primary key
:jsonparam device_type: Physical|Virtual|...
:jsonparam is_ioc: True|False
:jsonparam description: description
:jsonparam items: list of items ICS id linked to the host
:jsonparam ansible_vars: Ansible variables
:jsonparam ansible_groups: list of Ansible groups names
"""
host = models.Host.query.get_or_404(host_id)
if not current_user.has_access_to_network(host.main_network):
raise CSEntryError("User doesn't have the required group", status_code=403)
# The method currently doesn't allow to update the host name
# If we do, we have to update all the linked interface name as well!
# Interfaces shall always start by the host name
allowed_fields = (
("device_type", models.DeviceType, "name"),
("is_ioc", bool, None),
("description", str, None),
("items", [models.Item], "ics_id"),
("ansible_vars", dict, None),
("ansible_groups", [models.AnsibleGroup], "name"),
)
return utils.update_generic_model(models.Host, host_id, allowed_fields)
@bp.route("/hosts/<int:host_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_host(host_id):
......@@ -218,29 +470,6 @@ def delete_host(host_id):
return utils.delete_generic_model(models.Host, host_id)
@bp.route("/macs")
@login_required
def get_macs():
"""Return mac addresses
.. :quickref: Network; Get mac addresses
"""
return utils.get_generic_model(models.Mac, order_by=models.Mac.address)
@bp.route("/macs", methods=["POST"])
@login_groups_accepted("admin", "create")
def create_macs():
"""Create a new mac address
.. :quickref: Network; Create new mac address
:jsonparam address: MAC address
:jsonparam item_id: (optional) linked item primary key
"""
return utils.create_generic_model(models.Mac, mandatory_fields=("address",))
@bp.route("/domains")
@login_required
def get_domains():
......@@ -263,6 +492,18 @@ def create_domain():
return utils.create_generic_model(models.Domain, mandatory_fields=("name",))
@bp.route("/domains/<int:domain_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_domain(domain_id):
"""Delete a domain
.. :quickref: Network; Delete a domain
:param domain_id: domain primary key
"""
return utils.delete_generic_model(models.Domain, domain_id)
@bp.route("/cnames")
@login_required
def get_cnames():
......@@ -297,3 +538,15 @@ def create_cname():
return utils.create_generic_model(
models.Cname, mandatory_fields=("name", "interface_id")
)
@bp.route("/cnames/<int:cname_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_cname(cname_id):
"""Delete a cname
.. :quickref: Network; Delete a cname
:param cname_id: cname primary key
"""
return utils.delete_generic_model(models.Cname, cname_id)
app/api/user.py
View file @ 534cf643
......@@ -15,13 +15,13 @@ from flask_login import login_required, current_user
from ..extensions import ldap_manager
from ..decorators import login_groups_accepted
from .. import utils, tokens, models
from .utils import get_generic_model, create_generic_model
from .utils import get_generic_model
bp = Blueprint("user_api", __name__)
@bp.route("/users")
@login_required
@login_groups_accepted("admin", "auditor")
def get_users():
"""Return users information
......@@ -40,22 +40,6 @@ def get_user_profile():
return jsonify(current_user.to_dict()), 200
@bp.route("/users", methods=["POST"])
@login_groups_accepted("admin")
def create_user():
"""Create a new user
.. :quickref: User; Create new user
:jsonparam username: User's username
:jsonparam display_name: User's display name
:jsonparam email: User's email
"""
return create_generic_model(
models.User, mandatory_fields=("username", "display_name", "email")
)
@bp.route("/login", methods=["POST"])
def login():
"""Return a JSON Web Token
......
app/api/utils.py
View file @ 534cf643
......@@ -12,6 +12,8 @@ This module implements useful functions for the API.
import urllib.parse
import sqlalchemy as sa
from flask import current_app, jsonify, request
from flask_login import current_user
from flask_sqlalchemy import Pagination
from ..extensions import db
from .. import utils
......@@ -19,7 +21,7 @@ from .. import utils
def commit():
try:
db.session.commit()
except (sa.exc.IntegrityError, sa.exc.DataError) as e:
except sa.exc.SQLAlchemyError as e:
db.session.rollback()
raise utils.CSEntryError(str(e), status_code=422)
......@@ -59,33 +61,84 @@ def build_pagination_header(pagination, base_url, **kwargs):
return header
def get_generic_model(model, order_by=None, query=None):
def get_generic_model(model, order_by=None, base_query=None, query=None):
"""Return data from model as json
:param model: model class
:param order_by: column to order the result by
:param order_by: column to order the result by (not used if base_query or query is passed)
:param query: optional base_query to use [default: model.query]
:param query: optional query to use (for more complex queries)
:returns: data from model as json
"""
kwargs = request.args.to_dict()
page = int(kwargs.pop("page", 1))
per_page = int(kwargs.pop("per_page", 20))
# Remove recursive from kwargs so that it doesn't get passed
# to query.filter_by in get_query
recursive = kwargs.pop("recursive", "false").lower() == "true"
if query is None:
query = utils.get_query(model.query, **kwargs)
if order_by is None:
order_by = getattr(model, "name")
query = query.order_by(order_by)
pagination = query.paginate(page, per_page)
data = [item.to_dict() for item in pagination.items]
if base_query is None:
base_query = model.query
if order_by is None:
order_by = getattr(model, "name")
base_query = base_query.order_by(order_by)
query = utils.get_query(base_query, model, **kwargs)
pagination = query.paginate(
page, per_page, max_per_page=current_app.config["MAX_PER_PAGE"]
)
data = [item.to_dict(recursive=recursive) for item in pagination.items]
# Re-add recursive to kwargs so that it's part of the pagination url
kwargs["recursive"] = recursive
header = build_pagination_header(pagination, request.base_url, **kwargs)
return jsonify(data), 200, header
def create_generic_model(model, mandatory_fields=("name",), **kwargs):
def search_generic_model(model, filter_sensitive=False):
"""Return filtered data from model as json
:param model: model class
:param bool filter_sensitive: filter out sensitive data if set to True
:returns: filtered data from model as json
"""
kwargs = request.args.to_dict()
page = int(kwargs.pop("page", 1))
per_page = int(kwargs.pop("per_page", 20))
per_page = min(per_page, current_app.config["MAX_PER_PAGE"])
search = kwargs.get("q", "*")
instances, nb_filtered = model.search(
search, page=page, per_page=per_page, filter_sensitive=filter_sensitive
)
current_app.logger.debug(
f'Found {nb_filtered} {model.__tablename__}(s) when searching "{search}"'
)
data = [instance.to_dict(recursive=True) for instance in instances]
pagination = Pagination(None, page, per_page, nb_filtered, None)
header = build_pagination_header(pagination, request.base_url, **kwargs)
return jsonify(data), 200, header
def get_json_body():
"""Return the json body from the current request
Raise a CSEntryError if the body is not a json object
"""
data = request.get_json()
if data is None:
raise utils.CSEntryError("Body should be a JSON object")
current_app.logger.debug(f"Received: {data}")
if not data:
raise utils.CSEntryError("At least one field is required", status_code=422)
return data
def create_generic_model(model, mandatory_fields=("name",), **kwargs):
"""Create an instance of the model
:param model: model class
:param mandatory_fields: list of fields that shall be passed in the body
:param kwargs: extra fields to use
:returns: representation of the created instance as json
"""
data = get_json_body()
data.update(kwargs)
for mandatory_field in mandatory_fields:
if mandatory_field not in data:
......@@ -101,6 +154,9 @@ def create_generic_model(model, mandatory_fields=("name",), **kwargs):
raise utils.CSEntryError(str(e), status_code=422)
db.session.add(instance)
commit()
current_app.logger.info(
f"New {model.__tablename__} created by {current_user}: {instance.to_dict()}"
)
return jsonify(instance.to_dict()), 201
......@@ -112,5 +168,40 @@ def delete_generic_model(model, primary_key):
"""
instance = model.query.get_or_404(primary_key)
db.session.delete(instance)
db.session.commit()
commit()
current_app.logger.info(
f"{model.__tablename__} {instance} deleted by {current_user}"
)
return jsonify(), 204
def update_generic_model(model, primary_key, allowed_fields):
"""Update the model based on the primary_key
:param model: model class
:param primary_key: primary key of the instance to update
:param allowed_fields: list of fields that can be updated
:returns: representation of the updated model as json
"""
data = get_json_body()
# Use a list and not a set because the order is important
allowed_keys = [field[0] for field in allowed_fields]
for key in data:
if key not in allowed_keys:
raise utils.CSEntryError(f"Invalid field '{key}'", status_code=422)
instance = model.query.get_or_404(primary_key)
data = utils.convert_to_models(data, allowed_fields)
try:
# Loop on allowed_keys and not data to respect
# the order in which the fields are set
# setting ip before network is important for Interface
for key in allowed_keys:
if key in data:
setattr(instance, key, data[key])
except Exception as e:
raise utils.CSEntryError(str(e), status_code=422)
commit()
current_app.logger.info(
f"{model.__tablename__} {primary_key} updated by {current_user}: {instance.to_dict()}"
)
return jsonify(instance.to_dict()), 200
app/commands.py
View file @ 534cf643
......@@ -9,20 +9,29 @@ This module defines extra flask commands.
:license: BSD 2-Clause, see LICENSE for more details.
"""
import click
import ldap3
import redis
import rq
import sqlalchemy as sa
import sentry_sdk
from flask import current_app
from rq.contrib.sentry import register_sentry
from raven import Client
from raven.transport.http import HTTPTransport
from sentry_sdk.integrations.rq import RqIntegration
from .extensions import db, ldap_manager
from .defaults import defaults
from .tasks import TaskWorker
from . import models, utils, tokens
def disable_user(user):
"""Clear users'groups, email and tokens"""
user.groups = []
user.email = ""
# Revoke all user's tokens
for token in user.tokens:
db.session.delete(token)
def sync_user(connection, user):
"""Synchronize the user from the database with information from the LDAP server"""
search_attr = current_app.config.get("LDAP_USER_LOGIN_ATTR")
......@@ -34,26 +43,35 @@ def sync_user(connection, user):
search_scope=getattr(ldap3, current_app.config.get("LDAP_USER_SEARCH_SCOPE")),
attributes=current_app.config.get("LDAP_GET_USER_ATTRIBUTES"),
)
if len(connection.response) == 1:
ldap_user = connection.response[0]
attributes = ldap_user["attributes"]
user.display_name = utils.attribute_to_string(attributes["cn"])
user.email = utils.attribute_to_string(attributes["mail"])
groups = ldap_manager.get_user_groups(
dn=ldap3.utils.conv.escape_filter_chars(ldap_user["dn"]),
_connection=connection,
)
user.groups = sorted(
[utils.attribute_to_string(group["cn"]) for group in groups]
)
current_app.logger.info(f"{user} updated")
results = [
result for result in connection.response if result["type"] == "searchResEntry"
]
if len(results) == 1:
ldap_user = results[0]
# OU=InActiveUsers is specific to ESS AD
if "OU=InActiveUsers" in ldap_user["dn"]:
current_app.logger.info(f"{user} is inactive. User disabled.")
disable_user(user)
else:
attributes = ldap_user["attributes"]
user.display_name = (
utils.attribute_to_string(attributes["cn"]) or user.username
)
user.email = utils.attribute_to_string(attributes["mail"])
groups = ldap_manager.get_user_groups(
dn=ldap3.utils.conv.escape_filter_chars(ldap_user["dn"]),
_connection=connection,
)
user.groups = sorted(
[utils.attribute_to_string(group["cn"]) for group in groups]
)
current_app.logger.info(f"{user} updated")
elif len(results) == 0:
current_app.logger.warning(f"{user} not found! User disabled.")
disable_user(user)
else:
# Clear user's groups
user.groups = []
# Revoke all user's tokens
for token in user.tokens:
db.session.delete(token)
current_app.logger.info(f"{user} disabled")
current_app.logger.warning(f"Too many results for {user}!")
current_app.logger.warning(f"results: {results}")
return user
......@@ -72,6 +90,21 @@ def sync_users():
db.session.commit()
def clean_deferred_tasks():
"""Set all deferred tasks to failed"""
for task in (
models.Task.query.filter_by(status=models.JobStatus.DEFERRED)
.order_by(models.Task.created_at)
.all()
):
if task.depends_on is None or task.depends_on.status == models.JobStatus.FAILED:
current_app.logger.info(
f"Set deferred task {task.id} ({task.name}) to failed"
)
task.status = models.JobStatus.FAILED
db.session.commit()
def register_cli(app):
@app.cli.command()
def create_defaults():
......@@ -80,10 +113,15 @@ def register_cli(app):
db.session.add(instance)
try:
db.session.commit()
except sa.exc.IntegrityError as e:
except sa.exc.IntegrityError:
db.session.rollback()
app.logger.debug(f"{instance} already exists")
@app.cli.command()
def clean_deferred():
"""Set deferred tasks to failed if the task it depends on failed"""
clean_deferred_tasks()
@app.cli.command()
def syncusers():
"""Synchronize all users from the database with information the LDAP server"""
......@@ -103,23 +141,37 @@ def register_cli(app):
@app.cli.command()
def runworker():
"""Run RQ worker"""
redis_url = current_app.config["REDIS_URL"]
redis_url = current_app.config["RQ_REDIS_URL"]
redis_connection = redis.from_url(redis_url)
with rq.Connection(redis_connection):
worker = TaskWorker(current_app.config["QUEUES"])
worker = TaskWorker(["high", "normal", "low"])
if current_app.config["SENTRY_DSN"]:
client = Client(
sentry_sdk.init(
current_app.config["SENTRY_DSN"],
transport=HTTPTransport,
environment=current_app.config["CSENTRY_ENVIRONMENT"],
release=current_app.config["CSENTRY_RELEASE"],
integrations=[RqIntegration()],
)
register_sentry(client, worker)
worker.work()
@app.cli.command()
def reindex():
@click.option(
"--delete/--no-delete",
default=True,
help="Delete and recreate the index [default: True]",
)
@click.option(
"--name", default="all", help="name of the class to reindex [default: all]"
)
def reindex(delete, name):
"""Initialize the elasticsearch index"""
current_app.elasticsearch.indices.delete("*", ignore=404)
models.Item.reindex()
models.Host.reindex()
if delete:
current_app.elasticsearch.indices.delete("*", ignore=404)
if name == "all":
models.Item.reindex(delete)
models.Host.reindex(delete)
models.AnsibleGroup.reindex(delete)
return
try:
getattr(models, name).reindex(delete)
except AttributeError:
click.echo(f"The class {name} can't be indexed")
app/decorators.py
View file @ 534cf643
......@@ -21,8 +21,9 @@ def login_groups_accepted(*groups):
This can be used for users logged in using a cookie (web UI) or JWT (API).
Example::
@bp.route('/models', methods=['POST'])
@login_groups_accepted('admin', 'create')
@login_groups_accepted('admin', 'inventory')
def create_model():
return create()
......@@ -38,6 +39,10 @@ def login_groups_accepted(*groups):
def wrapper(fn):
@wraps(fn)
def decorated_view(*args, **kwargs):
# LOGIN_DISABLED can be set to True to turn off authentication check when testing.
# This variable is also used by flask-login
if current_app.config.get("LOGIN_DISABLED"):
return fn(*args, **kwargs)
if not current_user.is_authenticated:
return current_app.login_manager.unauthorized()
if not current_user.is_member_of_one_group(groups):
......
app/defaults.py
View file @ 534cf643
......@@ -23,8 +23,10 @@ defaults = [
models.DeviceType(name="PhysicalMachine"),
models.DeviceType(name="VirtualMachine"),
models.DeviceType(name="Network"),
models.DeviceType(name="MicroTCA"),
models.DeviceType(name="MTCA-AMC"),
models.DeviceType(name="MTCA-IFC"),
models.DeviceType(name="MTCA-MCH"),
models.DeviceType(name="MTCA-RTM"),
models.DeviceType(name="VME"),
models.DeviceType(name="PLC"),
models.Tag(name="IOC", admin_only=False),
]
app/extensions.py
View file @ 534cf643
......@@ -21,7 +21,6 @@ from flask_debugtoolbar import DebugToolbarExtension
from flask_redis import FlaskRedis
from flask_session import Session
from flask_caching import Cache
from raven.contrib.flask import Sentry
convention = {
......@@ -44,4 +43,3 @@ toolbar = DebugToolbarExtension()
session_redis_store = FlaskRedis(config_prefix="SESSION_REDIS")
fsession = Session()
cache = Cache()
sentry = Sentry()
app/factory.py
View file @ 534cf643
......@@ -12,9 +12,11 @@ Create the WSGI application.
import logging
import sqlalchemy as sa
import rq_dashboard
import sentry_sdk
from flask import Flask
from whitenoise import WhiteNoise
from elasticsearch import Elasticsearch
from sentry_sdk.integrations.flask import FlaskIntegration
from . import settings, models
from .extensions import (
db,
......@@ -28,7 +30,6 @@ from .extensions import (
session_redis_store,
fsession,
cache,
sentry,
)
from .admin.views import (
AdminModelView,
......@@ -47,6 +48,7 @@ from .api.user import bp as user_api
from .api.inventory import bp as inventory_api
from .api.network import bp as network_api
from .commands import register_cli
from ._version import __version__
from . import utils
......@@ -60,28 +62,17 @@ def create_app(config=None):
app.jinja_env.filters["datetimeformat"] = utils.format_datetime
app.jinja_env.filters["toyaml"] = utils.pretty_yaml
app.jinja_env.globals["__version__"] = __version__
if app.config["SENTRY_DSN"]:
# CSENTRY_ENVIRONMENT can be overwritten in the local settings
# We can't define SENTRY_ENVIRONMENT before loading them
app.config["SENTRY_ENVIRONMENT"] = app.config["CSENTRY_ENVIRONMENT"]
sentry.init_app(
app, dsn=app.config["SENTRY_DSN"], logging=True, level=logging.ERROR
sentry_sdk.init(
dsn=app.config["SENTRY_DSN"],
environment=app.config["CSENTRY_ENVIRONMENT"],
send_default_pii=True,
integrations=[FlaskIntegration()],
)
if not app.debug:
# Log to stderr
handler = logging.StreamHandler()
handler.setFormatter(
logging.Formatter(
"%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
)
)
# Set app logger level to DEBUG
# otherwise only WARNING and above are propagated
app.logger.setLevel(logging.DEBUG)
handler.setLevel(logging.DEBUG)
app.logger.addHandler(handler)
app.logger.setLevel(logging.DEBUG)
app.logger.info("CSEntry created!")
# Remove variables that contain a password
settings_to_display = {
......@@ -104,6 +95,9 @@ def create_app(config=None):
)
app.logger.info(f"Settings:\n{settings_string}")
# Force RQ_DASHBOARD_REDIS_URL to RQ_REDIS_URL
app.config["RQ_DASHBOARD_REDIS_URL"] = app.config["RQ_REDIS_URL"]
db.init_app(app)
migrate.init_app(app)
login_manager.init_app(app)
......@@ -141,7 +135,6 @@ def create_app(config=None):
admin.add_view(AdminModelView(models.Interface, db.session))
admin.add_view(AdminModelView(models.Mac, db.session))
admin.add_view(AdminModelView(models.Cname, db.session))
admin.add_view(AdminModelView(models.Tag, db.session))
admin.add_view(TaskAdmin(models.Task, db.session, endpoint="tasks"))
app.register_blueprint(main)
......@@ -154,11 +147,7 @@ def create_app(config=None):
app.register_blueprint(network_api, url_prefix="/api/v1/network")
app.register_blueprint(rq_dashboard.blueprint, url_prefix="/rq")
app.wsgi_app = WhiteNoise(app.wsgi_app, root="static/")
app.wsgi_app.add_files(
root="/opt/conda/envs/csentry/lib/python3.6/site-packages/flask_bootstrap/static/",
prefix="bootstrap/",
)
app.wsgi_app = WhiteNoise(app.wsgi_app, root=app.config["CSENTRY_STATIC_DIR"])
register_cli(app)
......
app/fields.py
View file @ 534cf643
......@@ -32,11 +32,13 @@ class YAMLField(TextAreaField):
return yaml.safe_dump(self.data, default_flow_style=False) if self.data else ""
def process_formdata(self, valuelist):
if valuelist:
if valuelist and valuelist[0].strip() != "":
try:
self.data = yaml.safe_load(valuelist[0])
except yaml.YAMLError:
raise ValueError("This field contains invalid YAML")
if not isinstance(self.data, dict):
raise ValueError("This field shall only contain key-value-pairs")
else:
self.data = None
......
app/inventory/forms.py
View file @ 534cf643
......@@ -31,7 +31,10 @@ class ItemForm(CSEntryForm):
"ICS id",
validators=[
validators.InputRequired(),
validators.Regexp(ICS_ID_RE),
validators.Regexp(
ICS_ID_RE,
message="The ICS id shall be composed of 3 letters and 3 digits.",
),
Unique(models.Item, "ics_id"),
],
)
......@@ -48,7 +51,7 @@ class ItemForm(CSEntryForm):
parent_id = SelectField("Parent", coerce=utils.coerce_to_str_or_none)
host_id = SelectField("Host", coerce=utils.coerce_to_str_or_none)
stack_member = NoValidateSelectField(
"Stack member", coerce=utils.coerce_to_str_or_none, choices=[]
"Stack member", coerce=utils.coerce_to_int_or_none, choices=[]
)
mac_addresses = StringField(
"MAC addresses",
......
app/inventory/views.py
View file @ 534cf643
......@@ -46,14 +46,14 @@ def list_items():
@login_required
def _generate_excel_file():
task = current_user.launch_task(
"generate_items_excel_file", func="generate_items_excel_file", timeout=180
"generate_items_excel_file", func="generate_items_excel_file", job_timeout=180
)
db.session.commit()
return utils.redirect_to_job_status(task.id)
@bp.route("/items/create", methods=("GET", "POST"))
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def create_item():
# The following keys are stored in the session to easily create
# several identical items
......@@ -102,7 +102,7 @@ def view_item(ics_id):
@bp.route("/items/comment/<ics_id>", methods=("GET", "POST"))
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def comment_item(ics_id):
item = models.Item.query.filter_by(ics_id=ics_id).first_or_404()
form = CommentForm()
......@@ -116,7 +116,7 @@ def comment_item(ics_id):
@bp.route("/items/comment/edit/<comment_id>", methods=("GET", "POST"))
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def edit_comment(comment_id):
comment = models.ItemComment.query.get_or_404(comment_id)
form = EditCommentForm(request.form, obj=comment)
......@@ -136,7 +136,7 @@ def edit_comment(comment_id):
@bp.route("/items/comment/delete", methods=["POST"])
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def delete_comment():
comment = models.ItemComment.query.get_or_404(request.form["comment_id"])
ics_id = comment.item.ics_id
......@@ -150,7 +150,7 @@ def delete_comment():
@bp.route("/items/edit/<ics_id>", methods=("GET", "POST"))
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def edit_item(ics_id):
item = models.Item.query.filter_by(ics_id=ics_id).first_or_404()
mac_addresses = " ".join([str(mac) for mac in item.macs])
......@@ -210,10 +210,6 @@ def attributes_favorites():
@bp.route("/_retrieve_attributes_favorites")
@login_required
def retrieve_attributes_favorites():
if current_user not in db.session:
# If the current user is cached, it won't be in the sqlalchemy session
# Add it to access the user favorite attributes relationship
db.session.add(current_user)
data = [
(
favorite.base64_image(),
......@@ -227,7 +223,7 @@ def retrieve_attributes_favorites():
@bp.route("/attributes/<kind>", methods=("GET", "POST"))
@login_groups_accepted("admin", "create")
@login_groups_accepted("admin", "inventory")
def attributes(kind):
form = AttributeForm()
if form.validate_on_submit():
......@@ -238,7 +234,7 @@ def attributes(kind):
db.session.add(new_model)
try:
db.session.commit()
except sa.exc.IntegrityError as e:
except sa.exc.IntegrityError:
db.session.rollback()
flash(f"{form.name.data} already exists! {kind} not created.", "error")
else:
......@@ -275,10 +271,6 @@ def update_favorites(kind):
Add or remove the attribute from the favorites when the
checkbox is checked/unchecked in the attributes table
"""
if current_user not in db.session:
# If the current user is cached, it won't be in the sqlalchemy session
# Add it to access the user favorite attributes relationship
db.session.add(current_user)
try:
model = getattr(models, kind)
except AttributeError:
......