Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • andersharrisson/csentry
  • ics-infrastructure/csentry
2 results
Show changes
Commits on Source (625)
525 additional commits have been omitted to prevent performance issues.
Hide whitespace changes
Inline Side-by-side
Showing
with 2355 additions and 84 deletions
.dockerignore
View file @ 534cf643
.git
*.swp
__pycache__
**/*.swp
**/__pycache__
settings*.cfg
data
postgres
Jenkinsfile
Makefile
docker-compose*.yml
.gitlab-ci.yml
.cache
.coverage
.pytest_cache
docs/_build
.env 0 → 100644
View file @ 534cf643
POSTGRES_USER=ics
POSTGRES_PASSWORD=icspwd
POSTGRES_DB=csentry_db
PGDATA_VOLUME=./data/postgres
ELASTIC_DATA_VOLUME=./data/elastic
.flake8 0 → 100644
View file @ 534cf643
[flake8]
# E501: let black handle line length
# W503 is incompatible with PEP 8
ignore = E501,W503
.gitignore
View file @ 534cf643
......@@ -3,3 +3,16 @@
__pycache__
*.pyc
settings*.cfg
/data
.cache
.coverage
/docs/_build
.scannerwork
coverage.xml
junit.xml
.pytest_cache
.tower_cli.cfg
/app/static/files/*
!/app/static/files/.empty
.vscode
.mypy_cache
.gitlab-ci.yml 0 → 100644
View file @ 534cf643
---
include:
- remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/PreCommit.gitlab-ci.yml"
- remote: "https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/SonarScanner.gitlab-ci.yml"
variables:
CONTAINER_TEST_IMAGE: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME"
CONTAINER_RELEASE_IMAGE: "$CI_REGISTRY_IMAGE:latest"
CONTAINER_CACHE_IMAGE: "$CI_REGISTRY_IMAGE:master"
POSTGRES_USER: ics
POSTGRES_PASSWORD: icspwd
POSTGRES_DB: csentry_db_test
stages:
- check
- build
- test
- analyse
- release
- deploy
default:
tags:
- docker
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
build:
stage: build
image: docker:latest
script:
- docker pull "$CONTAINER_CACHE_IMAGE" || true
- docker build --pull --cache-from "$CONTAINER_CACHE_IMAGE" -t "$CONTAINER_TEST_IMAGE" .
- docker push "$CONTAINER_TEST_IMAGE"
test:
stage: test
image: "$CONTAINER_TEST_IMAGE"
services:
- postgres:10
- redis:4.0
- name: docker.elastic.co/elasticsearch/elasticsearch:7.3.2
alias: elasticsearch
command: ["bin/elasticsearch", "-Ediscovery.type=single-node"]
before_script:
- pip install -r requirements-dev.txt
script:
- pytest --junitxml=junit.xml --cov-report=xml:coverage.xml --cov-report=term --cov=app -v
artifacts:
paths:
- junit.xml
- coverage.xml
reports:
junit: junit.xml
expire_in: 1 hour
release-image:
stage: release
image: docker:latest
dependencies: []
script:
- docker pull "$CONTAINER_TEST_IMAGE"
- docker tag "$CONTAINER_TEST_IMAGE" "$CONTAINER_RELEASE_IMAGE"
- docker push "$CONTAINER_RELEASE_IMAGE"
only:
- tags
deploy-dev:
stage: deploy
image: registry.esss.lu.se/ics-docker/tower-cli
before_script: []
dependencies: []
script:
- >
tower-cli job launch
-h torn.tn.esss.lu.se
-t ${TOWER_OAUTH_TOKEN}
-J deploy-csentry-dev
-e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
environment:
name: dev
url: https://csentry-lab-01.cslab.esss.lu.se
except:
- master
- tags
deploy-staging:
stage: deploy
image: registry.esss.lu.se/ics-docker/tower-cli
before_script: []
dependencies: []
script:
- >
tower-cli job launch
-h torn.tn.esss.lu.se
-t ${TOWER_OAUTH_TOKEN}
-J deploy-csentry-staging
-e "csentry_tag=$CI_COMMIT_REF_NAME" --monitor
environment:
name: staging
url: https://csentry-test.esss.lu.se
only:
- master
- tags
pages:
stage: deploy
image: "$CONTAINER_RELEASE_IMAGE"
dependencies: []
before_script:
- pip install -r requirements-dev.txt
script:
- sphinx-build -M html docs docs/_build
- mv docs/_build/html public
artifacts:
paths:
- public
# CI_COMMIT_TAG is used in docs/conf.py
# Don't forget to update if not only deploying docs for tags
only:
- tags
deploy-production:
stage: deploy
image: registry.esss.lu.se/ics-docker/tower-cli
before_script: []
dependencies: []
script:
- >
tower-cli job launch
-h torn.tn.esss.lu.se
-t ${TOWER_OAUTH_TOKEN}
-J deploy-csentry
-e "csentry_tag=$CI_COMMIT_TAG" --monitor
environment:
name: production
url: https://csentry.esss.lu.se
only:
- tags
when: manual
.pre-commit-config.yaml 0 → 100644
View file @ 534cf643
repos:
- repo: https://github.com/psf/black
rev: 22.6.0
hooks:
- id: black
- repo: https://github.com/pycqa/flake8
rev: 4.0.1
hooks:
- id: flake8
CHANGELOG.rst 0 → 100644
View file @ 534cf643
Changelog
=========
Version 2021.04.06
------------------
- Validate group names (INFRA-3135)
- Add warning text for creating vm (INFRA-3292)
- Paginate history in ansible groups and hosts (INFRA-3136)
Version 2020.11.25
------------------
- Add documentation about database versioning
- Set display_name to username when empty (INFRA-2909)
- Improve logging (INFRA-2899)
Version 2020.11.13
------------------
- Increased hostname character limit from 20 -> 24 (INFRA-2808)
- Use SSL by default for LDAP connection (INFRA-2805)
- Use SonarScanner.gitlab-ci.yml template (INFRA-2832)
Version 2020.10.30
------------------
- Fix inventory export to excel (INFRA-2780)
- Add developer documentation
Version 2020.10.27
------------------
- Update Python and requirements (INFRA-2649)
- Update RQ and rq-dashboard (INFRA-2649 / INFRA-2754)
- Fix code smells (INFRA-2729)
- Add auditor group (INFRA-2728)
- Update Sphinx and switch to sphinx_rtd_theme (INFRA-2742)
Version 2020.10.06
------------------
- Make sensitive hosts visible for members of the scope (INFRA-2508)
- Only hide sensitive networks not part of the user scope (INFRA-2437)
- Automatically set network_scope group as parent of network groups (INFRA-2639)
Version 2020.07.08
------------------
- Add view to edit network scope (INFRA-2316)
- Add API endpoint to patch network scope (INFRA-2316)
Version 2020.06.12
------------------
- Change MTCA-IOxOS to MTCA-IFC (INFRA-2200)
- Add description field on Interface (INFRA-2112)
Version 2020.04.16
------------------
- Allow /network/networks API endpoint for normal users (INFRA-2013)
- Update Bootstrap to 4.4.1 (INFRA-2021)
- Update DataTables (INFRA-2021)
- Add tooltip on search box (INFRA-1893)
Version 2020.03.17
------------------
- Add extra badges to README and documentation
- Fix get_hosts for normal users (INFRA-1888)
Version 2020.03.09
------------------
- Trigger core services update on host change (INFRA-1846)
- Add DELETE method on all API endpoints (INFRA-1853)
- Remove create_user API endpoint (INFRA-1854)
Version 2020.03.05
------------------
- Allow users to view networks (INFRA-1809)
- Add HOSTNAME Ansible group type (INFRA-1844)
- Remove IOC git repository creation (INFRA-1839)
- Make IP optional when creating interface via the API (INFRA-1833)
- Introduce new device types for MTCA (INFRA-1835)
Version 2020.02.25
------------------
- Fix inventory update (INFRA-1792)
Version 2020.02.24
------------------
- Add app version in navbar (INFRA-1789)
- Allow to pass LDAP settings as env variables (INFRA-1788)
- Implement server side processing for Ansible groups (INFRA-1790)
Version 0.29.0 (2020-02-03)
---------------------------
- Add MAC address column to hosts table (INFRA-1669)
- Restrict view of networks and scopes to admin only (INFRA-1671)
- Add sensitive field to Network class (INFRA-1671)
- Filter out sensitive hosts for non admin users (INFRA-1671)
- Add API endpoint and view to update network (INFRA-1724)
- Make pages title more explicit (INFRA-1726)
Version 0.28.0 (2019-12-19)
---------------------------
- Fix exception when setting new stack_member (INFRA-1648)
- Prevent recursive dependency loop in Ansible groups (INFRA-1622)
- Prevent network and scope overlapping (INFRA-1627)
- Add netmask to network view (INFRA-1660)
Version 0.27.1 (2019-12-04)
---------------------------
- Fix regexp for ICS id validation (INFRA-1569)
- Allow normal users to edit hosts with no interface (INFRA-1604)
Version 0.27.0 (2019-10-30)
---------------------------
- Increase interface name max length (INFRA-1324)
- Add endpoints to patch host and interface (INFRA-1406)
- Limit networks to same scope for extra interfaces (INFRA-1297)
Version 0.26.1 (2019-09-18)
---------------------------
- Fix inventory update not triggered (INFRA-1290)
Version 0.26.0 (2019-09-18)
---------------------------
- Decouple inventory and core services update (INFRA-1290)
- Add broadcast address to network view (INFRA-1291)
- Add view network scope page (INFRA-1292)
- Make ansible_vars in host searchable (INFRA-1295)
- Remove Generate ZTP configuration function (INFRA-1299)
Version 0.25.1 (2019-08-30)
---------------------------
- Prevent input of invalid Ansible variables (INFRA-1221)
- Update black and pre-commit config
- Add pytest.ini to speed test discovery
Version 0.25.0 (2019-08-22)
---------------------------
- Implement user access on network scope instead of domain (INFRA-1228)
- Create IOC repositories with same path as name (INFRA-1230)
Version 0.24.1 (2019-07-10)
---------------------------
- Fix sorting of items with some null stack_member (INFRA-1112)
Version 0.24.0 (2019-07-10)
---------------------------
- Add serial_number and stack_member to hosts json model (INFRA-1111)
Version 0.23.0 (2019-06-14)
---------------------------
- Increase RQ default timeout (INFRA-1051)
Version 0.22.0 (2019-06-14)
---------------------------
- Make vlan optional in NetworkScope and Network (INFRA-1049)
- Update reverse dependencies of failed tasks (INFRA-1051)
Version 0.21.1 (2019-05-13)
---------------------------
- Fix users synchronization (INFRA-1026)
- Remove caching of user retrieval (INFRA-1025)
Version 0.21.0 (2019-05-10)
---------------------------
- Improve documentation about ansible-vault
- Fix IOC host indexation (INFRA-1015)
- Allow users to delete their host (INFRA-1018)
- Fix job.status DeprecationWarning (INFRA-1021)
- Allow service users to login (INFRA-1022)
Version 0.20.2 (2019-05-08)
---------------------------
- Fix IOC repository creation (INFRA-1015)
- Save ansible var when setting boot profile (INFRA-1016)
Version 0.20.1 (2019-04-25)
---------------------------
- Save the device_type_id in the session (INFRA-987)
- Fix exception when no network selected (INFRA-810)
- Remove graylog support in uwsgi (INFRA-981)
- Replace raven with sentry-sdk (INFRA-979)
Version 0.20.0 (2019-04-05)
---------------------------
- Add API endpoint to search hosts (INFRA-931)
- Initialize IOC repository on GitLab (INFRA-932)
- Limit the max number of elements returned per page (INFRA-942)
- Allow to set the boot profile from CSEntry (INFRA-943)
Version 0.19.2 (2019-03-27)
---------------------------
- Fix model update in admin interface (INFRA-908)
- Increase timeout for core services update (INFRA-895)
- Add depends_on field on Task (INFRA-895)
Version 0.19.1 (2019-03-18)
---------------------------
- Fix excel file download (INFRA-890)
- Skip post install for windows VM creation (INFRA-877)
Version 0.19.0 (2019-03-18)
---------------------------
- Add osversion field to CreateVMForm (INFRA-877)
- Fix link to AWX job id for workflow jobs (INFRA-886)
- Allow users to create VM/VIOC (INFRA-775)
- Trigger AWX inventory update on database change (INFRA-887)
Version 0.18.2 (2019-03-07)
---------------------------
- Fix post install job trigger (INFRA-870)
Version 0.18.1 (2019-03-07)
---------------------------
- Clean extra vars passed for VM and VIOC creation (INFRA-870)
Version 0.18.0 (2019-03-05)
---------------------------
- Add gateway field to network table (INFRA-809)
- Add view network page (INFRA-860)
- Remove tags and add is_ioc field to host table (INFRA-862)
- Allow to launch a workflow job (INFRA-867)
Version 0.17.1 (2019-02-07)
---------------------------
- Catch exception raised by Unique Validator (INFRA-777)
- Only check the main interface in dynamic groups (INFRA-784)
- Make sure the first interface is the main one (INFRA-785)
Version 0.17.0 (2019-01-28)
---------------------------
- Implement new layout (INFRA-763)
- Fix case insensitive search (INFRA-770)
- Add post install job after VM creation (INFRA-769)
Version 0.16.0 (2019-01-23)
---------------------------
- Explicitely define the elasticsearch mapping (INFRA-723)
- Add favicon (INFRA-725)
- Pass disk size in inventory when creating VM (INFRA-759)
Version 0.15.1 (2018-12-10)
---------------------------
- Fix extra interface creation (INFRA-697)
Version 0.15.0 (2018-11-30)
---------------------------
- Disable DHCP & DNS update on host modification (INFRA-673)
- Use different groups for inventory and network (INFRA-578)
- Add network permissions per domain (INFRA-578)
- Move mac API to inventory endpoint (INFRA-674)
- Add button to delete Ansible group (INFRA-678)
- Save VM memory and cores as ansible variables (INFRA-640)
- Return host fqdn in Ansible groups (INFRA-640)
- Add extra fields to the API (INFRA-640)
- Update Python to 3.7.1 and dependencies (INFRA-684)
- Fix WhiteNoise used static directory (INFRA-683)
- Fix flask subcommand name (INFRA-687)
Version 0.14.0 (2018-11-08)
---------------------------
- Fix TypeError in after_commit (INFRA-613)
- Add zabbix plugin to uwsgi (INFRA-614)
- Allow to edit and delete comments (INFRA-618)
- Make mac addresses unique by interface (INFRA-639)
- Replace bootstrap-select with selectize (INFRA-644)
- Make host/interface/cname unique (INFRA-245)
- Switch to new template to deploy VM in proxmox (INFRA-651)
Version 0.13.0 (2018-10-12)
---------------------------
- Implement full text search for items (INFRA-575)
- Implement server side processing and full text search for network hosts (INFRA-595)
- Enable datatables state saving (INFRA-608)
- Compile uwsgi with graylog2 plugin (INFRA-576)
- Fix javascript error: Cannot read property '_buttons' of undefined (INFRA-584)
- Fix LDAPInvalidFilterErrorldap3 (INFRA-550)
- Fix IndexError when mail is set to an empty list (INFRA-610)
Version 0.12.0 (2018-09-28)
---------------------------
- Update DataTables with bootstrap 4 theme (INFRA-545)
- Implement alert flashing from javascript (INFRA-545)
- Add button to export items to excel file (INFRA-545)
- Add button to delete hosts (INFRA-557)
- Allow to delete hosts and interfaces via the API (INFRA-570)
- Display hosts with no interface on list hosts page (INFRA-569)
- Ignore old "lost" deferred tasks (INFRA-559)
Version 0.11.4 (2018-09-24)
---------------------------
- Replace AWX username/password with oauth token for deployment
- Allow to resize the CodeMirror editor (INFRA-543)
- Add yaml constructor to support "!vault" tag (INFRA-544)
- Fix TowerCLIError when ztp_stack is empty (INFRA-547)
- Fix AttributeError when creating host with group (INFRA-548)
- Enable sentry user feedback for crash reports
Version 0.11.3 (2018-09-14)
---------------------------
- Add sentry integration (INFRA-525)
- Pass sensitive information via environment variables
Version 0.11.2 (2018-09-13)
---------------------------
- Add versioning on Host and AnsibleGroup (INFRA-500)
- Disable artifact downloads in release and deploy stages
Version 0.11.1 (2018-08-21)
---------------------------
- Update tower-cli version for deployment
- Add host FQDN in host view (INFRA-459)
- Pass ztp_stack var to trigger_ztp_configuration (INFRA-458)
- Update Ansible groups documentation
Version 0.11.0 (2018-08-16)
---------------------------
- Add Ansible parent - child groups
- Add Ansible dynamic groups
- Remove flask-bootstrap dependency
- Update to bootstrap 4.1.3
- Switch to python-slim image and update requirements
- Replace jwt decorators with flask-login
Version 0.10.2 (2018-07-23)
---------------------------
- Allow retrieving interfaces by network name (INFRA-424)
Version 0.10.1 (2018-07-17)
---------------------------
- Wrap long lines in Ansible groups table
- Add model field to /networks/hosts endpoint (INFRA-414)
- Add model field to /networks/interfaces endpoint (INFRA-414)
- Add trigger ZTP configuration task (INFRA-414)
Version 0.10.0 (2018-07-16)
---------------------------
- Use sqlalchemy events hook to trigger tasks
- Add Ansible variables on the host table (INFRA-412)
- Add Ansible groups table and views (INFRA-412)
- Add new api endpoint /network/groups (INFRA-412)
Version 0.9.0 (2018-07-06)
--------------------------
- Redirect to the view host page when creating a host (INFRA-402)
- Save tasks (background jobs) to the database (INFRA-403)
- Add blueprint to view tasks (INFRA-403)
- Redirect to the task view page when creating a VM
- Display host creation date and user on view host page
- Use black for source code formatting
- Add pre-commit hooks for code formatting and linting
- Update documentation
Version 0.8.1
-------------
- Fix VM creation (memory shall be passed in MB)
Version 0.8.0
-------------
- Hide interface name in create host form (INFRA-287)
- Allow to link several items to one host (INFRA-267)
- Add stack_member field to item (INFRA-267)
- Add extra device types (INFRA-302)
- Add IOC tag (INFRA-302)
- Select the IOC tag by default based on device type (INFRA-302)
- Use bootstrap-select for tags selection
- Use the last IP as network gateway (INFRA-339)
- Fix default selected tags in edit interface view (INFRA-344)
- Use CamelCase for CSEntry device types and tags (INFRA-334)
- Add RQ to process jobs in the background
- Add RQ Dashboard blueprint (admin only)
- Automatically trigger update of TN core services on host or interface change
- Add Create VM button (admin only for now)
- Allow to easily identify staging server (INFRA-347)
- Update documentation
Version 0.7.0
-------------
- Add search in model description from inventory page
- Add model description to view item (INFRA-280)
- Add device_type table (INFRA-281)
- Add device_type to interfaces API endpoint (INFRA-277)
- Add attributes favorites page (INFRA-283)
- Sort network names in the register new host form (INFRA-284)
- Clear error messages on form fields on change (INFRA-285)
Version 0.6.8
-------------
- return username instead of display name in the API (INFRA-241)
- add /network/cnames API endpoint
- update default available IPs range
Version 0.6.7
-------------
- accept a list of LDAP groups
Version 0.6.6
-------------
- add link to documentation
- add Action tab next to Attributes
- switch to centos-miniconda3 base image
Version 0.6.5
-------------
- add /network/domains API endpoint
- add interfaces filtering by domain
- add netmask field in network json representation
- increase QRCode size
Version 0.6.4
-------------
- add API endpoint to retrieve the current user profile
- add Submit action QRCode to create an item
- update scanner instructions
- update documentation
Version 0.6.3
-------------
- add CHANGELOG
- add sphinx dependency to build documentation
- add stage to deploy documentation to GitLab pages
- move QR codes to attributes table
- improve documentation
- add manual job to deploy to production
Dockerfile
View file @ 534cf643
FROM continuumio/miniconda3:latest
FROM python:3.8-slim as base
RUN groupadd -r ics && useradd -r -g ics ics
# Add ESS specific debian repository mirror
RUN echo "deb https://artifactory.esss.lu.se/artifactory/debian-mirror stable main" | tee /etc/apt/sources.list.d/ess-debian-mirror.list
WORKDIR /app
# Install Python dependencies in an intermediate image
# as some requires a compiler (psycopg2)
FROM base as builder
# Install dependencies required to compile some Python packages
# Taken from https://github.com/docker-library/python/blob/master/3.6/stretch/slim/Dockerfile
# For psycopg2: libpq-dev
# For pillow: libjpeg-dev libpng-dev libtiff-dev
RUN apt-get update \
&& apt-get install -yq --no-install-recommends \
gcc \
libbz2-dev \
libc6-dev \
libexpat1-dev \
libffi-dev \
libjpeg-dev \
libgdbm-dev \
liblzma-dev \
libncursesw5-dev \
libpcre3-dev \
libpng-dev \
libpq-dev \
libreadline-dev \
libsqlite3-dev \
libssl-dev \
libtiff-dev \
make \
tk-dev \
wget \
xz-utils \
zlib1g-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt /requirements.txt
RUN python -m venv /venv \
&& . /venv/bin/activate \
&& pip install --no-cache-dir --upgrade pip \
&& pip install --no-cache-dir -r /requirements.txt
ARG CSENTRY_BUILD
COPY requirements-dev.txt /requirements-dev.txt
RUN if [ "$CSENTRY_BUILD" = "DEV" ] ; then /venv/bin/pip install --no-cache-dir -r /requirements-dev.txt ; fi
FROM base
# Install inventory requirements
COPY environment.yml /app/environment.yml
RUN conda config --add channels conda-forge \
&& conda env create -n inventory -f environment.yml \
&& rm -rf /opt/conda/pkgs/*
RUN groupadd -r -g 1000 csi \
&& useradd --no-log-init -r -g csi -u 1000 csi
COPY --chown=csi:csi --from=builder /venv /venv
# Install libraries for psycopg2 and pillow
# Shall be the same as the one linked to when compiling in builder image!
RUN apt-get update \
&& apt-get install -yq --no-install-recommends \
libjpeg62-turbo \
libpng16-16 \
libtiff5 \
libpq5 \
libpcre3 \
zlib1g \
git \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
RUN rm /etc/apt/sources.list.d/ess-debian-mirror.list
COPY --chown=csi:csi . /app/
WORKDIR /app
RUN echo "__version__ = \"$(git describe)\"" > app/_version.py
# Install the app
COPY . /app/
RUN chown -R ics:ics /app/*
ENV PATH /venv/bin:$PATH
EXPOSE 8000
CMD ["flask", "run", "--host", "0.0.0.0", "--port", "8000"]
# activate the inventory environment
ENV PATH /opt/conda/envs/inventory/bin:$PATH
USER csi
LICENSE 0 → 100644
View file @ 534cf643
BSD 2-Clause License
Copyright (c) 2017, European Spallation Source ERIC
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Makefile 0 → 100644
View file @ 534cf643
.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image docs
OWNER := registry.esss.lu.se/ics-infrastructure
GIT_TAG := $(shell git describe --always)
IMAGE := csentry
help:
# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
@echo "CSEntry"
@echo "======="
@echo
@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
build: ## build the latest image
docker build -t $(OWNER)/$(IMAGE):latest .
tag: ## tag the latest image with the git tag
docker tag $(OWNER)/$(IMAGE):latest $(OWNER)/$(IMAGE):$(GIT_TAG)
push: ## push the latest and git tag image
docker push $(OWNER)/$(IMAGE):$(GIT_TAG)
docker push $(OWNER)/$(IMAGE):latest
clean: ## remove the image with git tag and the test database
-docker rmi $(OWNER)/$(IMAGE):$(GIT_TAG)
-docker rm -f csentry_postgres
-docker rm -f csentry_redis
refresh: ## pull the latest image from Docker Hub
# skip if error: image might not be on dockerhub yet
-docker pull $(OWNER)/$(IMAGE):latest
release: refresh \
build \
tag \
push
release: ## build, tag, and push all stacks
db: ## start postgres and redis for development
docker-compose up -d postgres redis elasticsearch worker
init_db: ## initialize the dev database
docker-compose run --rm web flask db upgrade head
docker-compose run --rm web flask create-defaults
upgrade_db: ## upgrade the dev database
docker-compose run --rm web flask db upgrade head
test: ## run the tests (on current directory)
docker-compose -f docker-compose.yml run --rm web pytest --cov=app -v
db_test: ## start required containers for test
# Pass docker-compose.yml to skip docker-compose.override.yml (db not mounted as volume for speed)
docker-compose -f docker-compose.yml up -d postgres redis elasticsearch worker
test_image: ## run the tests (on the latest image)
# Pass docker-compose.yml to skip docker-compose.override.yml
docker-compose -f docker-compose.yml run --rm web
run_uwsgi: ## run the application with uwsgi (to test prod env)
docker-compose run -p 8000:8000 --rm web uwsgi --master --http 0.0.0.0:8000 --http-keepalive --manage-script-name --mount /csentry=wsgi.py --callable app --uid conda --processes 2 -b 16384
docs: ## run the tests (on current directory)
docker run --rm -v $(shell pwd):/app registry.esss.lu.se/ics-infrastructure/csentry:master sphinx-build -M html docs docs/_build
README.rst
View file @ 534cf643
ICS inventory
=============
CSEntry
=======
ICS inventory proof of concept.
.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
:target: https://github.com/ambv/black
.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=alert_status
:target: https://sonarqube.esss.lu.se/dashboard?id=csentry
.. image:: https://sonarqube.esss.lu.se/api/project_badges/measure?project=csentry&metric=ncloc
:target: https://sonarqube.esss.lu.se/dashboard?id=csentry
.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/pipeline.svg
.. image:: https://gitlab.esss.lu.se/ics-infrastructure/csentry/badges/master/coverage.svg
Control System Entry web server.
Development
......@@ -11,36 +24,69 @@ You can use docker for development:
1. Clone the repository
2. Create the database::
2. Create the database. Data will be stored under "./data" by default.
You can export the PGDATA_VOLUME variable to use another directory::
$ mkdir /data/pgdata
$ export PGDATA_VOLUME=/data/pgdata
# Start only postgres so it has time to initialize
$ docker-compose up -d postgres
# Create the database
$ docker-compose run web flask initdb
or
$ make db
# Initialize the database
$ docker-compose run --rm web flask db upgrade head
$ docker-compose run --rm web flask create_defaults
or
$ make init_db
4. Start the application::
3. Start the application::
$ docker-compose up
5. Open your browser and go to `http://localhost:8000
4. Open your browser and go to `http://localhost:8000
<http://localhost:8000>`_.
5. To run the tests::
$ make test
Once the database has been created, you only need to run `docker-compose
up` to start the app.
Testing
-------
By default docker-compose reads both docker-compose.yml and docker-compose.override.yml.
In docker-compose.override.yml, the current directory is mounted as a volume.
This is great for development.
To test the built image, we should only use the docker-compose.yml (and ignore
docker-compose.override.yml).
1. Create the database::
# Start only postgres so it has time to initialize
$ docker-compose -f docker-compose.yml up -d postgres
or
$ make db_image
2. Run the tests::
$ docker-compose -f docker-compose.yml run --rm web
or
$ make test_image
Backup & restore
----------------
To dump the database::
$ docker run --rm --link inventory_postgres:postgres --net inventory_default -e PGPASSWORD="<inventory_password>"
postgres:9.6 pg_dump -h postgres -U inventory inventory_db | gzip > inventory_db.dump.gz
$ docker run --rm --link csentry_postgres:postgres --net csentry_default -e PGPASSWORD="<csentry_password>"
postgres:10 pg_dump -h postgres -U ics csentry_db | gzip > csentry_db.sql.gz
To restore the database::
$ gunzip -c inventory_db.dump.g | docker run --rm --link inventory_postgres:postgres --net inventory_default
-e PGPASSWORD="<inventory_password>" -i postgres:9.6 psql -h postgres -U inventory inventory_db
$ gunzip -c csentry_db.sql.gz | docker run --rm --link csentry_postgres:postgres --net csentry_default
-e PGPASSWORD="<csentry_password>" -i postgres:10 psql -h postgres -U ics csentry_db
app/_version.py 0 → 100644
View file @ 534cf643
__version__ = "dev"
app/users/__init__.py → app/admin/__init__.py
View file @ 534cf643
File moved
app/admin/views.py 0 → 100644
View file @ 534cf643
# -*- coding: utf-8 -*-
"""
app.admin.views
~~~~~~~~~~~~~~~
This module customizes the admin views.
:copyright: (c) 2017 European Spallation Source ERIC
:license: BSD 2-Clause, see LICENSE for more details.
"""
from wtforms import validators, fields
from flask_admin.contrib import sqla
from flask_admin.model.form import converts
from flask_login import current_user
from ..validators import IPNetwork, ICS_ID_RE
# Monkey patch flask-admin Unique validator to disable it
# Flask-admin automatically adds an unique validator that queries
# the database. For some fields like INET and MACADDR, if the value
# is invalid the database raises an exception. It's nicer to let the
# field validator catches the incorrect format.
# It's also better to let the database catch unicity error anyway.
sqla.form.Unique.__call__ = lambda x, y, z: None
# Add custom model converter for CIText type
# See https://github.com/flask-admin/flask-admin/issues/1196
class AppAdminModelConverter(sqla.form.AdminModelConverter):
@converts("CIText")
def conv_CIText(self, field_args, **extra):
return fields.TextAreaField(**field_args)
@converts("sqlalchemy.dialects.postgresql.base.CIDR")
def conv_PGCidr(self, field_args, **extra):
field_args["validators"].append(IPNetwork())
return fields.StringField(**field_args)
class AdminModelView(sqla.ModelView):
model_form_converter = AppAdminModelConverter
# Replace TextAreaField (default for Text) with StringField
form_overrides = {"name": fields.StringField}
def is_accessible(self):
return current_user.is_authenticated and current_user.is_admin
class UserAdmin(AdminModelView):
can_create = False
can_edit = False
can_delete = False
class TokenAdmin(AdminModelView):
can_create = False
can_edit = False
class ItemAdmin(AdminModelView):
# Replace TextAreaField (default for Text) with StringField
form_overrides = {"ics_id": fields.StringField, "serial_number": fields.StringField}
form_args = {
"ics_id": {
"label": "ICS id",
"validators": [
validators.Regexp(
ICS_ID_RE, message="ICS id shall match [A-Z]{3}[0-9]{3}"
)
],
"filters": [lambda x: x or None],
}
}
class NetworkAdmin(AdminModelView):
# Replace TextAreaField (default for Text) with StringField
form_overrides = {"vlan_name": fields.StringField}
class TaskAdmin(AdminModelView):
column_display_pk = True
can_create = False
app/api/inventory.py 0 → 100644
View file @ 534cf643
# -*- coding: utf-8 -*-
"""
app.api.inventory
~~~~~~~~~~~~~~~~~
This module implements the inventory API.
:copyright: (c) 2017 European Spallation Source ERIC
:license: BSD 2-Clause, see LICENSE for more details.
"""
from flask import Blueprint, jsonify, request, current_app
from flask_login import login_required
from .. import utils, models
from ..decorators import login_groups_accepted
from .utils import commit, create_generic_model, get_generic_model, delete_generic_model
bp = Blueprint("inventory_api", __name__)
def get_item_by_id_or_ics_id(id_):
"""Retrieve item by id or ICS id"""
try:
item_id = int(id_)
except ValueError:
# Assume id_ is an ics_id
item = models.Item.query.filter_by(ics_id=id_).first()
else:
item = models.Item.query.get(item_id)
if item is None:
raise utils.CSEntryError(f"Item id '{id_}' not found", status_code=404)
return item
@bp.route("/items")
@login_required
def get_items():
"""Return items
.. :quickref: Inventory; Get items
"""
return get_generic_model(models.Item, order_by=models.Item.created_at)
@bp.route("/items/<id_>")
@login_required
def get_item(id_):
r"""Retrieve item by id or ICS id
.. :quickref: Inventory; Get item by id or ICS id
:param id\_: ICS id or primary key of the item
"""
item = get_item_by_id_or_ics_id(id_)
return jsonify(item.to_dict())
@bp.route("/items", methods=["POST"])
@login_groups_accepted("admin", "inventory")
def create_item():
"""Register a new item
.. :quickref: Inventory; Create new item
:jsonparam serial_number: serial number
:jsonparam ics_id: (optional) ICS id (temporary one generated if not present)
:jsonparam quantity: (optional) number of items [default: 1]
:jsonparam manufacturer: (optional) name of the manufacturer
:jsonparam model: (optional) name of the model
:jsonparam location: (optional) name of the location
:jsonparam status: (optional) name of the status
:jsonparam parent_id: (optional) parent id
:jsonparam host_id: (optional) host id
"""
# People should assign an ICS id to a serial number when creating
# an item so ics_id should also be a mandatory field.
# But there are existing items (in confluence and JIRA) that we want to
# import and associate after they have been created.
# In that case a temporary id is automatically assigned.
return create_generic_model(models.Item, mandatory_fields=("serial_number",))
@bp.route("/items/<id_>", methods=["PATCH"])
@login_groups_accepted("admin", "inventory")
def patch_item(id_):
r"""Patch an existing item
.. :quickref: Inventory; Update existing item
:param id\_: ICS id or primary key of the item
:jsonparam ics_id: ICS id - only allowed if the current one is temporary
:jsonparam manufacturer: Item's manufacturer
:jsonparam model: Item's model
:jsonparam location: Item's location
:jsonparam status: Item's status
:jsonparam parent: Item's parent
"""
data = request.get_json()
if data is None:
raise utils.CSEntryError("Body should be a JSON object")
if not data:
raise utils.CSEntryError("At least one field is required", status_code=422)
for key in data.keys():
if key not in (
"ics_id",
"manufacturer",
"model",
"location",
"status",
"parent",
):
raise utils.CSEntryError(f"Invalid field '{key}'", status_code=422)
item = get_item_by_id_or_ics_id(id_)
# Only allow to set ICS id if the current id is a temporary one
if item.ics_id.startswith(current_app.config["TEMPORARY_ICS_ID"]):
item.ics_id = data.get("ics_id", item.ics_id)
elif "ics_id" in data:
raise utils.CSEntryError("'ics_id' can't be changed", status_code=422)
item.manufacturer = utils.convert_to_model(
data.get("manufacturer", item.manufacturer), models.Manufacturer
)
item.model = utils.convert_to_model(data.get("model", item.model), models.Model)
item.location = utils.convert_to_model(
data.get("location", item.location), models.Location
)
item.status = utils.convert_to_model(data.get("status", item.status), models.Status)
parent_ics_id = data.get("parent")
if parent_ics_id is not None:
parent = models.Item.query.filter_by(ics_id=parent_ics_id).first()
if parent is not None:
item.parent_id = parent.id
# Update location and status with those from parent
item.location = parent.location
item.status = parent.status
# Update all children status and location
for child in item.children:
child.location = item.location
child.status = item.status
commit()
return jsonify(item.to_dict())
@bp.route("/items/<int:item_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_item(item_id):
"""Delete an item
.. :quickref: Inventory; Delete an item
:param item_id: item primary key
"""
return delete_generic_model(models.Item, item_id)
@bp.route("/items/<id_>/comments")
@login_required
def get_item_comments(id_):
r"""Get item comments
.. :quickref: Inventory; Get item comments
:param id\_: ICS id or primary key of the item
"""
item = get_item_by_id_or_ics_id(id_)
return jsonify([comment.to_dict() for comment in item.comments])
@bp.route("/items/<id_>/comments", methods=["POST"])
@login_groups_accepted("admin", "inventory")
def create_item_comment(id_):
r"""Create a comment on item
.. :quickref: Inventory; Create comment on item
:param id\_: ICS id or primary key of the item
:jsonparam body: comment body
"""
item = get_item_by_id_or_ics_id(id_)
return create_generic_model(
models.ItemComment, mandatory_fields=("body",), item_id=item.id
)
@bp.route("/items/comments/<int:comment_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_comment(comment_id):
"""Delete an item comment
.. :quickref: Inventory; Delete an item comment
:param comment_id: comment primary key
"""
return delete_generic_model(models.ItemComment, comment_id)
@bp.route("/actions")
@login_required
def get_actions():
"""Get actions
.. :quickref: Inventory; Get actions
"""
return get_generic_model(models.Action)
@bp.route("/manufacturers")
@login_required
def get_manufacturers():
"""Get manufacturers
.. :quickref: Inventory; Get manufacturers
"""
return get_generic_model(models.Manufacturer)
@bp.route("/manufacturers", methods=["POST"])
@login_groups_accepted("admin", "inventory")
def create_manufacturer():
"""Create a new manufacturer
.. :quickref: Inventory; Create new manufacturer
:jsonparam name: manufacturer name
:jsonparam description: (optional) description
"""
return create_generic_model(models.Manufacturer)
@bp.route("/manufacturers/<int:manufacturer_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_manufacturer(manufacturer_id):
"""Delete a manufacturer
.. :quickref: Inventory; Delete a manufacturer
:param manufacturer_id: manufacturer primary key
"""
return delete_generic_model(models.Manufacturer, manufacturer_id)
@bp.route("/models")
@login_required
def get_models():
"""Get models
.. :quickref: Inventory; Get models
"""
return get_generic_model(models.Model)
@bp.route("/models", methods=["POST"])
@login_groups_accepted("admin", "inventory")
def create_model():
"""Create a new model
.. :quickref: Inventory; Create new model
:jsonparam name: model name
:jsonparam description: (optional) description
"""
return create_generic_model(models.Model)
@bp.route("/models/<int:model_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_model(model_id):
"""Delete a model
.. :quickref: Inventory; Delete a model
:param model_id: model primary key
"""
return delete_generic_model(models.Model, model_id)
@bp.route("/locations")
@login_required
def get_locations():
"""Get locations
.. :quickref: Inventory; Get locations
"""
return get_generic_model(models.Location)
@bp.route("/locations", methods=["POST"])
@login_groups_accepted("admin", "inventory")
def create_locations():
"""Create a new location
.. :quickref: Inventory; Create new location
:jsonparam name: location name
:jsonparam description: (optional) description
"""
return create_generic_model(models.Location)
@bp.route("/locations/<int:location_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_location(location_id):
"""Delete a location
.. :quickref: Inventory; Delete a location
:param location_id: location primary key
"""
return delete_generic_model(models.Location, location_id)
@bp.route("/statuses")
@login_required
def get_status():
"""Get statuses
.. :quickref: Inventory; Get statuses
"""
return get_generic_model(models.Status)
@bp.route("/statuses", methods=["POST"])
@login_groups_accepted("admin", "inventory")
def create_status():
"""Create a new status
.. :quickref: Inventory; Create new status
:jsonparam name: status name
:jsonparam description: (optional) description
"""
return create_generic_model(models.Status)
@bp.route("/statuses/<int:status_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_status(status_id):
"""Delete a status
.. :quickref: Inventory; Delete a status
:param status_id: status primary key
"""
return delete_generic_model(models.Status, status_id)
@bp.route("/macs")
@login_required
def get_macs():
"""Return mac addresses
.. :quickref: Inventory; Get mac addresses
"""
return get_generic_model(models.Mac, order_by=models.Mac.address)
@bp.route("/macs", methods=["POST"])
@login_groups_accepted("admin", "inventory")
def create_macs():
"""Create a new mac address
.. :quickref: Inventory; Create new mac address
:jsonparam address: MAC address
:jsonparam item_id: (optional) linked item primary key
"""
return create_generic_model(models.Mac, mandatory_fields=("address",))
app/api/items.py deleted 100644 → 0
View file @ 5f41066b
# -*- coding: utf-8 -*-
"""
app.api.items
~~~~~~~~~~~~~
This module implements the items API.
:copyright: (c) 2017 European Spallation Source ERIC
:license: BSD 2-Clause, see LICENSE for more details.
"""
from flask import (current_app, Blueprint, jsonify, request)
from flask_jwt_extended import create_access_token, jwt_required
from flask_ldap3_login import AuthenticationResponseStatus
from ..extensions import ldap_manager, db
from ..models import User
bp = Blueprint('api', __name__)
@bp.route('/login', methods=['POST'])
def login():
try:
username = request.get_json()['username']
password = request.get_json()['password']
except (TypeError, KeyError):
return jsonify({'msg': 'Bad Request'}), 400
response = ldap_manager.authenticate(username, password)
if response.status == AuthenticationResponseStatus.success:
current_app.logger.debug(f'{username} successfully logged in')
user = User.query.filter_by(username=username).first()
if user is None:
# LDAP user not in database yet - create it
user = User(username, name=response.user_info['cn'], email=response.user_info['mail'])
db.session.add(user)
db.session.commit()
payload = {'access_token': create_access_token(identity=user.id)}
return jsonify(payload), 200
return jsonify({'msg': 'Invalid credentials'}), 401
@bp.route('/items')
@jwt_required
def get_items():
sql = """SELECT item.id, item.name, vendor.name, model.name, location.name, status.name
FROM item
INNER JOIN vendor ON vendor.id = item.vendor_id
INNER JOIN model ON model.id = item.model_id
INNER JOIN location ON location.id = item.location_id
INNER JOIN status ON status.id = item.status_id
"""
result = db.engine.execute(sql)
data = [[item for item in row] for row in result]
return jsonify(data=data)
app/api/network.py 0 → 100644
View file @ 534cf643
# -*- coding: utf-8 -*-
"""
app.api.network
~~~~~~~~~~~~~~~
This module implements the network API.
:copyright: (c) 2017 European Spallation Source ERIC
:license: BSD 2-Clause, see LICENSE for more details.
"""
from flask import Blueprint, request
from flask_login import login_required, current_user
from wtforms import ValidationError
from .. import models
from ..decorators import login_groups_accepted
from ..utils import CSEntryError, validate_ip
from . import utils
bp = Blueprint("network_api", __name__)
@bp.route("/scopes")
@login_groups_accepted("admin", "auditor")
def get_scopes():
"""Return network scopes
.. :quickref: Network; Get network scopes
"""
return utils.get_generic_model(
models.NetworkScope, order_by=models.NetworkScope.name
)
@bp.route("/scopes", methods=["POST"])
@login_groups_accepted("admin")
def create_scope():
"""Create a new network scope
.. :quickref: Network; Create new network scope
:jsonparam name: network scope name
:jsonparam first_vlan: network scope first vlan
:jsonparam last_vlan: network scope last vlan
:jsonparam supernet: network scope supernet
:jsonparam domain_id: primary key of the default domain
:jsonparam description: (optional) description
"""
return utils.create_generic_model(
models.NetworkScope,
mandatory_fields=("name", "first_vlan", "last_vlan", "supernet", "domain_id"),
)
@bp.route("/scopes/<int:scope_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_scope(scope_id):
"""Delete a network scope
.. :quickref: Network; Delete a network scope
:param scope_id: network scope primary key
"""
return utils.delete_generic_model(models.NetworkScope, scope_id)
@bp.route("/scopes/<int:scope_id>", methods=["PATCH"])
@login_groups_accepted("admin")
def patch_scope(scope_id):
r"""Patch an existing network scope
.. :quickref: Network; Update an existing network scope
:param scope_id: network scope primary key
:jsonparam name: network scope name
:jsonparam description: description
:jsonparam first_vlan: network scope first vlan
:jsonparam last_vlan: network scope last vlan
:jsonparam supernet: network scope supernet
:jsonparam domain: name of the default domain
"""
allowed_fields = (
("name", str, None),
("description", str, None),
("first_vlan", int, None),
("last_vlan", int, None),
("supernet", str, None),
("domain", models.Domain, "name"),
)
return utils.update_generic_model(models.NetworkScope, scope_id, allowed_fields)
@bp.route("/networks")
@login_groups_accepted("admin", "auditor", "network")
def get_networks():
"""Return networks
.. :quickref: Network; Get networks
"""
query = models.Network.query
if not (current_user.is_admin or current_user.is_auditor):
sensitive_networks = (
query.filter(models.Network.sensitive.is_(True))
.join(models.Network.scope)
.filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
)
query = (
query.filter(models.Network.sensitive.is_(False))
.union(sensitive_networks)
.distinct(models.Network.id)
.from_self()
)
query = query.order_by(models.Network.address)
return utils.get_generic_model(model=models.Network, base_query=query)
@bp.route("/networks", methods=["POST"])
@login_groups_accepted("admin")
def create_network():
"""Create a new network
.. :quickref: Network; Create new network
:jsonparam vlan_name: vlan name
:jsonparam vlan_id: vlan id
:jsonparam address: vlan address
:jsonparam first_ip: first IP of the allowed range
:jsonparam last_ip: last IP of the allowed range
:jsonparam gateway: gateway IP
:jsonparam scope: network scope name
:jsonparam domain_id: (optional) primary key of the domain [default: scope domain]
:jsonparam admin_only: (optional) boolean to restrict the network to admin users [default: False]
:type admin_only: bool
:jsonparam sensitive: hide the network and all hosts if True (for non admin) [default: False]
:type sensitive: bool
:jsonparam description: (optional) description
"""
return utils.create_generic_model(
models.Network,
mandatory_fields=(
"vlan_name",
"vlan_id",
"address",
"first_ip",
"last_ip",
"gateway",
"scope",
),
)
@bp.route("/networks/<int:network_id>", methods=["PATCH"])
@login_groups_accepted("admin")
def patch_network(network_id):
r"""Patch an existing network
.. :quickref: Network; Update an existing network
:param network_id: network primary key
:jsonparam vlan_name: vlan name
:jsonparam address: vlan address
:jsonparam first_ip: first IP of the allowed range
:jsonparam last_ip: last IP of the allowed range
:jsonparam gateway: gateway IP
:jsonparam domain: domain name
:jsonparam admin_only: boolean to restrict the network to admin users
:type admin_only: bool
:jsonparam sensitive: hide the network and all hosts if True (for non admin)
:type sensitive: bool
:jsonparam description: description
"""
# The method currently doesn't allow to update the network_scope
# as this will have an impact on the address
allowed_fields = (
("vlan_name", str, None),
("address", str, None),
("first_ip", str, None),
("last_ip", str, None),
("gateway", str, None),
("domain", models.Domain, "name"),
("admin_only", bool, None),
("sensitive", bool, None),
("description", str, None),
)
return utils.update_generic_model(models.Network, network_id, allowed_fields)
@bp.route("/networks/<int:network_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_network(network_id):
"""Delete a network
.. :quickref: Network; Delete a network
:param network_id: network primary key
"""
return utils.delete_generic_model(models.Network, network_id)
@bp.route("/interfaces")
@login_required
def get_interfaces():
"""Return interfaces
.. :quickref: Network; Get interfaces
"""
query = models.Interface.query
query = query.join(models.Interface.network).order_by(models.Interface.ip)
if not (current_user.is_admin or current_user.is_auditor):
sensitive_interfaces = (
query.filter(models.Network.sensitive.is_(True))
.join(models.Network.scope)
.filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
)
query = (
query.filter(models.Network.sensitive.is_(False))
.union(sensitive_interfaces)
.from_self()
.join(models.Interface.network)
.order_by(models.Interface.ip)
)
domain = request.args.get("domain", None)
if domain is not None:
query = query.join(models.Network.domain).filter(models.Domain.name == domain)
return utils.get_generic_model(model=None, query=query)
network = request.args.get("network", None)
if network is not None:
query = query.filter(models.Network.vlan_name == network)
return utils.get_generic_model(model=None, query=query)
return utils.get_generic_model(model=models.Interface, base_query=query)
@bp.route("/interfaces", methods=["POST"])
@login_groups_accepted("admin", "network")
def create_interface():
"""Create a new interface
.. :quickref: Network; Create new interface
:jsonparam network: network name
:jsonparam ip: (optional) interface IP - IP will be assigned automatically if not given
:jsonparam name: interface name
:jsonparam host: host name
:jsonparam mac: (optional) MAC address
"""
# The validate_interfaces method from the Network class is called when
# setting interface.network. This is why we don't pass network_id here
# but network (as vlan_name string)
# Same for host
data = request.get_json()
# Check that the user has the permissions to create an interface on this network
try:
network = models.Network.query.filter_by(
vlan_name=data["network"]
).first_or_404()
except Exception:
# If we can't get a network, an error will be raised in create_generic_model
pass
else:
if not current_user.has_access_to_network(network):
raise CSEntryError("User doesn't have the required group", status_code=403)
return utils.create_generic_model(
models.Interface, mandatory_fields=("network", "name", "host")
)
@bp.route("/interfaces/<int:interface_id>", methods=["PATCH"])
@login_groups_accepted("admin", "network")
def patch_interface(interface_id):
r"""Patch an existing interface
.. :quickref: Network; Update an existing interface
:param interface_id: interface primary key
:jsonparam ip: interface IP
:jsonparam name: interface name
:jsonparam network: network name
:jsonparam mac: MAC address
"""
interface = models.Interface.query.get_or_404(interface_id)
# User shall have access to both the current and new network (if provided)
if not current_user.has_access_to_network(interface.network):
raise CSEntryError("User doesn't have the required group", status_code=403)
data = utils.get_json_body()
if "network" in data:
new_network = models.Network.query.filter_by(
vlan_name=data["network"]
).first_or_404()
if not current_user.has_access_to_network(new_network):
raise CSEntryError("User doesn't have the required group", status_code=403)
# Ensure that the IP is in the network (current one or new one if passed)
# This is required because validate_interfaces is only called when a new network
# is assigned. So the check is needed even when new_network == interface.network
# (same network given in argument)
if "ip" in data:
try:
if "network" in data:
validate_ip(data["ip"], new_network)
else:
validate_ip(data["ip"], interface.network)
except ValidationError as e:
raise CSEntryError(str(e), status_code=422)
# The method doesn't allow to update the interface host
# I don't think it makes much sense (and an interface shall start by the host name)
# To add a new cname, use create_cname
allowed_fields = (
("ip", str, None),
("name", str, None),
("network", models.Network, "vlan_name"),
("mac", str, None),
)
return utils.update_generic_model(models.Interface, interface_id, allowed_fields)
@bp.route("/interfaces/<int:interface_id>", methods=["DELETE"])
@login_groups_accepted("admin", "network")
def delete_interface(interface_id):
"""Delete an interface
.. :quickref: Network; Delete an interface
:param interface_id: interface primary key
"""
return utils.delete_generic_model(models.Interface, interface_id)
@bp.route("/groups")
@login_required
def get_ansible_groups():
"""Return ansible groups
.. :quickref: Network; Get Ansible groups
"""
return utils.get_generic_model(
models.AnsibleGroup, order_by=models.AnsibleGroup.name
)
@bp.route("/groups", methods=["POST"])
@login_groups_accepted("admin")
def create_ansible_groups():
"""Create a new Ansible group
.. :quickref: Network; Create new Ansible group
:jsonparam name: group name
:jsonparam vars: (optional) Ansible variables
"""
return utils.create_generic_model(models.AnsibleGroup, mandatory_fields=("name",))
@bp.route("/groups/<int:group_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_ansible_group(group_id):
"""Delete an Ansible group
.. :quickref: Network; Delete an Ansible group
:param group_id: Ansible group primary key
"""
return utils.delete_generic_model(models.AnsibleGroup, group_id)
@bp.route("/hosts")
@login_required
def get_hosts():
"""Return hosts
.. :quickref: Network; Get hosts
"""
query = models.Host.query
if not (current_user.is_admin or current_user.is_auditor):
# Note that hosts without interface will be filtered out
# by this query. This is not an issue as hosts should always
# have an interface. They are useless otherwise.
non_sensitive_hosts = (
query.join(models.Host.interfaces)
.join(models.Interface.network)
.filter(models.Network.sensitive.is_(False))
)
sensitive_hosts = (
query.join(models.Host.interfaces)
.join(models.Interface.network)
.filter(models.Network.sensitive.is_(True))
.join(models.Network.scope)
.filter(models.NetworkScope.name.in_(current_user.csentry_network_scopes))
)
query = (
non_sensitive_hosts.union(sensitive_hosts)
.distinct(models.Host.id)
.from_self()
)
query = query.order_by(models.Host.name)
return utils.get_generic_model(model=models.Host, base_query=query)
@bp.route("/hosts/search")
@login_required
def search_hosts():
"""Search hosts
.. :quickref: Network; Search hosts
:query q: the search query
"""
return utils.search_generic_model(models.Host, filter_sensitive=True)
@bp.route("/hosts", methods=["POST"])
@login_groups_accepted("admin", "network")
def create_host():
"""Create a new host
.. :quickref: Network; Create new host
:jsonparam name: hostname
:jsonparam device_type: Physical|Virtual|...
:jsonparam is_ioc: True|False (optional)
:jsonparam description: (optional) description
:jsonparam items: (optional) list of items ICS id linked to the host
:jsonparam ansible_vars: (optional) Ansible variables
:jsonparam ansible_groups: (optional) list of Ansible groups names
"""
return utils.create_generic_model(
models.Host, mandatory_fields=("name", "device_type")
)
@bp.route("/hosts/<int:host_id>", methods=["PATCH"])
@login_groups_accepted("admin", "network")
def patch_host(host_id):
r"""Patch an existing host
.. :quickref: Network; Update an existing host
:param host_id: host primary key
:jsonparam device_type: Physical|Virtual|...
:jsonparam is_ioc: True|False
:jsonparam description: description
:jsonparam items: list of items ICS id linked to the host
:jsonparam ansible_vars: Ansible variables
:jsonparam ansible_groups: list of Ansible groups names
"""
host = models.Host.query.get_or_404(host_id)
if not current_user.has_access_to_network(host.main_network):
raise CSEntryError("User doesn't have the required group", status_code=403)
# The method currently doesn't allow to update the host name
# If we do, we have to update all the linked interface name as well!
# Interfaces shall always start by the host name
allowed_fields = (
("device_type", models.DeviceType, "name"),
("is_ioc", bool, None),
("description", str, None),
("items", [models.Item], "ics_id"),
("ansible_vars", dict, None),
("ansible_groups", [models.AnsibleGroup], "name"),
)
return utils.update_generic_model(models.Host, host_id, allowed_fields)
@bp.route("/hosts/<int:host_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_host(host_id):
"""Delete a host
.. :quickref: Network; Delete a host
:param host_id: host primary key
"""
return utils.delete_generic_model(models.Host, host_id)
@bp.route("/domains")
@login_required
def get_domains():
"""Return domains
.. :quickref: Network; Get domains
"""
return utils.get_generic_model(models.Domain, order_by=models.Domain.name)
@bp.route("/domains", methods=["POST"])
@login_groups_accepted("admin")
def create_domain():
"""Create a new domain
.. :quickref: Network; Create new domain
:jsonparam name: domain name
"""
return utils.create_generic_model(models.Domain, mandatory_fields=("name",))
@bp.route("/domains/<int:domain_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_domain(domain_id):
"""Delete a domain
.. :quickref: Network; Delete a domain
:param domain_id: domain primary key
"""
return utils.delete_generic_model(models.Domain, domain_id)
@bp.route("/cnames")
@login_required
def get_cnames():
"""Return cnames
.. :quickref: Network; Get cnames
"""
domain = request.args.get("domain", None)
if domain is not None:
query = models.Cname.query
query = (
query.join(models.Cname.interface)
.join(models.Interface.network)
.join(models.Network.domain)
.filter(models.Domain.name == domain)
)
query = query.order_by(models.Cname.name)
return utils.get_generic_model(model=None, query=query)
return utils.get_generic_model(models.Cname, order_by=models.Cname.name)
@bp.route("/cnames", methods=["POST"])
@login_groups_accepted("admin")
def create_cname():
"""Create a new cname
.. :quickref: Network; Create new cname
:jsonparam name: full cname
:jsonparam interface_id: primary key of the associated interface
"""
return utils.create_generic_model(
models.Cname, mandatory_fields=("name", "interface_id")
)
@bp.route("/cnames/<int:cname_id>", methods=["DELETE"])
@login_groups_accepted("admin")
def delete_cname(cname_id):
"""Delete a cname
.. :quickref: Network; Delete a cname
:param cname_id: cname primary key
"""
return utils.delete_generic_model(models.Cname, cname_id)
app/api/user.py 0 → 100644
View file @ 534cf643
# -*- coding: utf-8 -*-
"""
app.api.user
~~~~~~~~~~~~
This module implements the user API.
:copyright: (c) 2017 European Spallation Source ERIC
:license: BSD 2-Clause, see LICENSE for more details.
"""
from flask import current_app, Blueprint, jsonify, request
from flask_ldap3_login import AuthenticationResponseStatus
from flask_login import login_required, current_user
from ..extensions import ldap_manager
from ..decorators import login_groups_accepted
from .. import utils, tokens, models
from .utils import get_generic_model
bp = Blueprint("user_api", __name__)
@bp.route("/users")
@login_groups_accepted("admin", "auditor")
def get_users():
"""Return users information
.. :quickref: User; Get users information
"""
return get_generic_model(models.User, order_by=models.User.username)
@bp.route("/profile")
@login_required
def get_user_profile():
"""Return the current user profile
.. :quickref: User; Get current user profile
"""
return jsonify(current_user.to_dict()), 200
@bp.route("/login", methods=["POST"])
def login():
"""Return a JSON Web Token
.. :quickref: User; Get a token
:jsonparam username: username to login
:jsonparam password: password
"""
data = request.get_json()
if data is None:
raise utils.CSEntryError("Body should be a JSON object")
try:
username = data["username"]
password = data["password"]
except KeyError:
raise utils.CSEntryError(
"Missing mandatory field (username or password)", status_code=422
)
response = ldap_manager.authenticate(username, password)
if response.status == AuthenticationResponseStatus.success:
current_app.logger.debug(f"{username} successfully logged in")
user = ldap_manager._save_user(
response.user_dn, response.user_id, response.user_info, response.user_groups
)
payload = {"access_token": tokens.generate_access_token(identity=user.id)}
return jsonify(payload), 200
raise utils.CSEntryError("Invalid credentials", status_code=401)
app/api/utils.py 0 → 100644
View file @ 534cf643
# -*- coding: utf-8 -*-
"""
app.api.utils
~~~~~~~~~~~~~
This module implements useful functions for the API.
:copyright: (c) 2017 European Spallation Source ERIC
:license: BSD 2-Clause, see LICENSE for more details.
"""
import urllib.parse
import sqlalchemy as sa
from flask import current_app, jsonify, request
from flask_login import current_user
from flask_sqlalchemy import Pagination
from ..extensions import db
from .. import utils
def commit():
try:
db.session.commit()
except sa.exc.SQLAlchemyError as e:
db.session.rollback()
raise utils.CSEntryError(str(e), status_code=422)
def build_pagination_header(pagination, base_url, **kwargs):
"""Return the X-Total-Count and Link header information
:param pagination: flask_sqlalchemy Pagination class instance
:param base_url: request base_url
:param kwargs: extra query string parameters (without page and per_page)
:returns: dict with X-Total-Count and Link keys
"""
header = {"X-Total-Count": pagination.total}
links = []
if pagination.page > 1:
params = urllib.parse.urlencode(
{"per_page": pagination.per_page, "page": 1, **kwargs}
)
links.append(f'<{base_url}?{params}>; rel="first"')
if pagination.has_prev:
params = urllib.parse.urlencode(
{"per_page": pagination.per_page, "page": pagination.prev_num, **kwargs}
)
links.append(f'<{base_url}?{params}>; rel="prev"')
if pagination.has_next:
params = urllib.parse.urlencode(
{"per_page": pagination.per_page, "page": pagination.next_num, **kwargs}
)
links.append(f'<{base_url}?{params}>; rel="next"')
if pagination.pages > pagination.page:
params = urllib.parse.urlencode(
{"per_page": pagination.per_page, "page": pagination.pages, **kwargs}
)
links.append(f'<{base_url}?{params}>; rel="last"')
if links:
header["Link"] = ", ".join(links)
return header
def get_generic_model(model, order_by=None, base_query=None, query=None):
"""Return data from model as json
:param model: model class
:param order_by: column to order the result by (not used if base_query or query is passed)
:param query: optional base_query to use [default: model.query]
:param query: optional query to use (for more complex queries)
:returns: data from model as json
"""
kwargs = request.args.to_dict()
page = int(kwargs.pop("page", 1))
per_page = int(kwargs.pop("per_page", 20))
# Remove recursive from kwargs so that it doesn't get passed
# to query.filter_by in get_query
recursive = kwargs.pop("recursive", "false").lower() == "true"
if query is None:
if base_query is None:
base_query = model.query
if order_by is None:
order_by = getattr(model, "name")
base_query = base_query.order_by(order_by)
query = utils.get_query(base_query, model, **kwargs)
pagination = query.paginate(
page, per_page, max_per_page=current_app.config["MAX_PER_PAGE"]
)
data = [item.to_dict(recursive=recursive) for item in pagination.items]
# Re-add recursive to kwargs so that it's part of the pagination url
kwargs["recursive"] = recursive
header = build_pagination_header(pagination, request.base_url, **kwargs)
return jsonify(data), 200, header
def search_generic_model(model, filter_sensitive=False):
"""Return filtered data from model as json
:param model: model class
:param bool filter_sensitive: filter out sensitive data if set to True
:returns: filtered data from model as json
"""
kwargs = request.args.to_dict()
page = int(kwargs.pop("page", 1))
per_page = int(kwargs.pop("per_page", 20))
per_page = min(per_page, current_app.config["MAX_PER_PAGE"])
search = kwargs.get("q", "*")
instances, nb_filtered = model.search(
search, page=page, per_page=per_page, filter_sensitive=filter_sensitive
)
current_app.logger.debug(
f'Found {nb_filtered} {model.__tablename__}(s) when searching "{search}"'
)
data = [instance.to_dict(recursive=True) for instance in instances]
pagination = Pagination(None, page, per_page, nb_filtered, None)
header = build_pagination_header(pagination, request.base_url, **kwargs)
return jsonify(data), 200, header
def get_json_body():
"""Return the json body from the current request
Raise a CSEntryError if the body is not a json object
"""
data = request.get_json()
if data is None:
raise utils.CSEntryError("Body should be a JSON object")
if not data:
raise utils.CSEntryError("At least one field is required", status_code=422)
return data
def create_generic_model(model, mandatory_fields=("name",), **kwargs):
"""Create an instance of the model
:param model: model class
:param mandatory_fields: list of fields that shall be passed in the body
:param kwargs: extra fields to use
:returns: representation of the created instance as json
"""
data = get_json_body()
data.update(kwargs)
for mandatory_field in mandatory_fields:
if mandatory_field not in data:
raise utils.CSEntryError(
f"Missing mandatory field '{mandatory_field}'", status_code=422
)
try:
instance = model(**data)
except TypeError as e:
message = str(e).replace("__init__() got an ", "")
raise utils.CSEntryError(message, status_code=422)
except ValueError as e:
raise utils.CSEntryError(str(e), status_code=422)
db.session.add(instance)
commit()
current_app.logger.info(
f"New {model.__tablename__} created by {current_user}: {instance.to_dict()}"
)
return jsonify(instance.to_dict()), 201
def delete_generic_model(model, primary_key):
"""Delete the model based on the primary_key
:param model: model class
:param primary_key: primary key of the instance to delete
"""
instance = model.query.get_or_404(primary_key)
db.session.delete(instance)
commit()
current_app.logger.info(
f"{model.__tablename__} {instance} deleted by {current_user}"
)
return jsonify(), 204
def update_generic_model(model, primary_key, allowed_fields):
"""Update the model based on the primary_key
:param model: model class
:param primary_key: primary key of the instance to update
:param allowed_fields: list of fields that can be updated
:returns: representation of the updated model as json
"""
data = get_json_body()
# Use a list and not a set because the order is important
allowed_keys = [field[0] for field in allowed_fields]
for key in data:
if key not in allowed_keys:
raise utils.CSEntryError(f"Invalid field '{key}'", status_code=422)
instance = model.query.get_or_404(primary_key)
data = utils.convert_to_models(data, allowed_fields)
try:
# Loop on allowed_keys and not data to respect
# the order in which the fields are set
# setting ip before network is important for Interface
for key in allowed_keys:
if key in data:
setattr(instance, key, data[key])
except Exception as e:
raise utils.CSEntryError(str(e), status_code=422)
commit()
current_app.logger.info(
f"{model.__tablename__} {primary_key} updated by {current_user}: {instance.to_dict()}"
)
return jsonify(instance.to_dict()), 200
app/commands.py 0 → 100644
View file @ 534cf643
# -*- coding: utf-8 -*-
"""
app.commands
~~~~~~~~~~~~
This module defines extra flask commands.
:copyright: (c) 2018 European Spallation Source ERIC
:license: BSD 2-Clause, see LICENSE for more details.
"""
import click
import ldap3
import redis
import rq
import sqlalchemy as sa
import sentry_sdk
from flask import current_app
from sentry_sdk.integrations.rq import RqIntegration
from .extensions import db, ldap_manager
from .defaults import defaults
from .tasks import TaskWorker
from . import models, utils, tokens
def disable_user(user):
"""Clear users'groups, email and tokens"""
user.groups = []
user.email = ""
# Revoke all user's tokens
for token in user.tokens:
db.session.delete(token)
def sync_user(connection, user):
"""Synchronize the user from the database with information from the LDAP server"""
search_attr = current_app.config.get("LDAP_USER_LOGIN_ATTR")
object_filter = current_app.config.get("LDAP_USER_OBJECT_FILTER")
search_filter = f"(&{object_filter}({search_attr}={user.username}))"
connection.search(
search_base=ldap_manager.full_user_search_dn,
search_filter=search_filter,
search_scope=getattr(ldap3, current_app.config.get("LDAP_USER_SEARCH_SCOPE")),
attributes=current_app.config.get("LDAP_GET_USER_ATTRIBUTES"),
)
results = [
result for result in connection.response if result["type"] == "searchResEntry"
]
if len(results) == 1:
ldap_user = results[0]
# OU=InActiveUsers is specific to ESS AD
if "OU=InActiveUsers" in ldap_user["dn"]:
current_app.logger.info(f"{user} is inactive. User disabled.")
disable_user(user)
else:
attributes = ldap_user["attributes"]
user.display_name = (
utils.attribute_to_string(attributes["cn"]) or user.username
)
user.email = utils.attribute_to_string(attributes["mail"])
groups = ldap_manager.get_user_groups(
dn=ldap3.utils.conv.escape_filter_chars(ldap_user["dn"]),
_connection=connection,
)
user.groups = sorted(
[utils.attribute_to_string(group["cn"]) for group in groups]
)
current_app.logger.info(f"{user} updated")
elif len(results) == 0:
current_app.logger.warning(f"{user} not found! User disabled.")
disable_user(user)
else:
current_app.logger.warning(f"Too many results for {user}!")
current_app.logger.warning(f"results: {results}")
return user
def sync_users():
"""Synchronize all users from the database with information the LDAP server"""
current_app.logger.info(
"Synchronize database with information from the LDAP server"
)
try:
connection = ldap_manager.connection
except ldap3.core.exceptions.LDAPException as e:
current_app.logger.warning(f"Failed to connect to the LDAP server: {e}")
return
for user in models.User.query.all():
sync_user(connection, user)
db.session.commit()
def clean_deferred_tasks():
"""Set all deferred tasks to failed"""
for task in (
models.Task.query.filter_by(status=models.JobStatus.DEFERRED)
.order_by(models.Task.created_at)
.all()
):
if task.depends_on is None or task.depends_on.status == models.JobStatus.FAILED:
current_app.logger.info(
f"Set deferred task {task.id} ({task.name}) to failed"
)
task.status = models.JobStatus.FAILED
db.session.commit()
def register_cli(app):
@app.cli.command()
def create_defaults():
"""Create the database default values"""
for instance in defaults:
db.session.add(instance)
try:
db.session.commit()
except sa.exc.IntegrityError:
db.session.rollback()
app.logger.debug(f"{instance} already exists")
@app.cli.command()
def clean_deferred():
"""Set deferred tasks to failed if the task it depends on failed"""
clean_deferred_tasks()
@app.cli.command()
def syncusers():
"""Synchronize all users from the database with information the LDAP server"""
sync_users()
@app.cli.command()
def delete_expired_tokens():
"""Prune database from expired tokens"""
tokens.prune_database()
@app.cli.command()
def maintenance():
"""Run maintenance commands"""
sync_users()
tokens.prune_database()
@app.cli.command()
def runworker():
"""Run RQ worker"""
redis_url = current_app.config["RQ_REDIS_URL"]
redis_connection = redis.from_url(redis_url)
with rq.Connection(redis_connection):
worker = TaskWorker(["high", "normal", "low"])
if current_app.config["SENTRY_DSN"]:
sentry_sdk.init(
current_app.config["SENTRY_DSN"],
environment=current_app.config["CSENTRY_ENVIRONMENT"],
integrations=[RqIntegration()],
)
worker.work()
@app.cli.command()
@click.option(
"--delete/--no-delete",
default=True,
help="Delete and recreate the index [default: True]",
)
@click.option(
"--name", default="all", help="name of the class to reindex [default: all]"
)
def reindex(delete, name):
"""Initialize the elasticsearch index"""
if delete:
current_app.elasticsearch.indices.delete("*", ignore=404)
if name == "all":
models.Item.reindex(delete)
models.Host.reindex(delete)
models.AnsibleGroup.reindex(delete)
return
try:
getattr(models, name).reindex(delete)
except AttributeError:
click.echo(f"The class {name} can't be indexed")