diff --git a/Makefile b/Makefile index 71d4894e01fd57351d7064ea94979eb1bf78a687..add1a505f8336a72bd4f70edfb7c96f4b8728244 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image +.PHONY: help build tag push refresh release db init_db upgrade_db test db_image test_image docs OWNER := registry.esss.lu.se/ics-infrastructure GIT_TAG := $(shell git describe --always) @@ -60,3 +60,6 @@ test_image: ## run the tests (on the latest image) run_uwsgi: ## run the application with uwsgi (to test prod env) docker-compose run -p 8000:8000 --rm web uwsgi --master --http 0.0.0.0:8000 --http-keepalive --manage-script-name --mount /csentry=wsgi.py --callable app --uid conda --processes 2 -b 16384 + +docs: ## run the tests (on current directory) + docker run --rm -v $(shell pwd):/app registry.esss.lu.se/ics-infrastructure/csentry:master sphinx-build -M html docs docs/_build diff --git a/app/decorators.py b/app/decorators.py index 66048e29f0b6173386b5ff92b0e0f25215e31f56..805dafc87e4f32dc2f8af1539c65f72621c12ab5 100644 --- a/app/decorators.py +++ b/app/decorators.py @@ -21,6 +21,7 @@ def login_groups_accepted(*groups): This can be used for users logged in using a cookie (web UI) or JWT (API). Example:: + @bp.route('/models', methods=['POST']) @login_groups_accepted('admin', 'inventory') def create_model(): diff --git a/app/main/views.py b/app/main/views.py index c84b7e2b9ac4cfcd0e8a764439e5470eab0df77d..8b84fc583277b36bc7c1f60ba1a8935f128a5382 100644 --- a/app/main/views.py +++ b/app/main/views.py @@ -99,6 +99,7 @@ def pop_rq_connection(exception=None): @bp.route("/") @login_required def index(): + """Return the application index""" return render_template("index.html") diff --git a/app/tasks.py b/app/tasks.py index 758d5af9afd3205135f2a6db180b2471be57fc3d..28a79937fc19d4f5441f4f13c92f3267975947d1 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -92,12 +92,12 @@ class TaskWorker(Worker): def launch_awx_job(resource="job", **kwargs): - """Launch an AWX job + r"""Launch an AWX job job_template or inventory_source shall be passed as keyword argument :param resource: job|workflow_job|inventory_source - :param **kwargs: keyword arguments passed to launch the job + :param \*\*kwargs: keyword arguments passed to launch the job :returns: A dictionary with information from resource.monitor """ rq_job = get_current_job() diff --git a/app/user/views.py b/app/user/views.py index 9dcae105f6d81aa45fd31b7e77a86088d1e7a126..67b7909987d07c43b240c35a6bf424c3e6c63614 100644 --- a/app/user/views.py +++ b/app/user/views.py @@ -29,6 +29,7 @@ bp = Blueprint("user", __name__) @bp.route("/login", methods=["GET", "POST"]) def login(): + """Login page""" form = LDAPLoginForm(request.form) if form.validate_on_submit(): login_user(form.user, remember=form.remember_me.data) @@ -39,6 +40,7 @@ def login(): @bp.route("/logout") @login_required def logout(): + """Logout endpoint""" logout_user() return redirect(url_for("user.login")) @@ -46,6 +48,7 @@ def logout(): @bp.route("/profile", methods=["GET", "POST"]) @login_required def profile(): + """User profile""" # Try to get the generated token from the session token = session.pop("generated_token", None) form = TokenForm(request.form) @@ -70,6 +73,7 @@ def profile(): @bp.route("/tokens/revoke", methods=["POST"]) @login_required def revoke_token(): + """Endpoint to revoke a token""" token_id = request.form["token_id"] jti = request.form["jti"] try: diff --git a/docs/development/deployment.rst b/docs/development/deployment.rst new file mode 100644 index 0000000000000000000000000000000000000000..801cda440fe0d5b7c9644fd771dd197262706fe5 --- /dev/null +++ b/docs/development/deployment.rst @@ -0,0 +1,31 @@ +Deployment +========== + +Deployment is performed using Ansible_ and docker_. The application is made of the following containers: + +- csentry_web to run the main Flask_ application with uwsgi_ +- csentry_workers_<index> to run RQ_ workers (same image as the main application) +- csentry_postgres to run PostgreSQL_ +- csentry_elasticsearch to run Elasticsearch_ +- csentry_redis to run Redis_ + +Refer to the `CSEntry Ansible role`_ for details. + +All application `default settings <https://gitlab.esss.lu.se/ics-infrastructure/csentry/-/blob/master/app/settings.py>`_ +can be overridden using a local `settings.cfg` file. +For deployment, this local file is defined in the `CSEntry Ansible playbook`_: + +- `config/settings-prod.cfg <https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-csentry/-/blob/master/config/settings-prod.cfg>`_ for production +- `config/settings-test.cfg <https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-csentry/-/blob/master/config/settings-test.cfg>`_ for staging + + +.. _docker: https://www.docker.com +.. _Ansible: https://docs.ansible.com/ansible/latest/index.html +.. _Flask: https://flask.palletsprojects.com +.. _uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/ +.. _PostgreSQL: https://www.postgresql.org +.. _Redis: https://redis.io +.. _Elasticsearch: https://www.elastic.co/elasticsearch/ +.. _RQ: https://python-rq.org +.. _CSEntry Ansible role: https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-role-csentry +.. _CSEntry Ansible playbook: https://gitlab.esss.lu.se/ics-ansible-galaxy/ics-ans-csentry diff --git a/docs/development/design.rst b/docs/development/design.rst new file mode 100644 index 0000000000000000000000000000000000000000..e27ce0666ba5a96b88eaf12c5a7f631df4b510a5 --- /dev/null +++ b/docs/development/design.rst @@ -0,0 +1,173 @@ +Design +====== + +CSEntry is a web application developed using Flask_, one of the most popular Python web frameworks. +Many principles follow the concepts described in `The Flask Mega-Tutorial`_ from `Miguel Grinberg <https://blog.miguelgrinberg.com/index>`_. + +The application relies on: + +- PostgreSQL_ as main database +- Redis_ for caching and for queuing jobs and processing them in the background with workers using RQ_ +- Elasticsearch_ for search + +Database +-------- + +PostgreSQL_ is used as the main database. +Interactions with the database are perfomed using SQLAlchemy_ ORM, which maps SQL tables to Python classes. +All models are defined in the :mod:`app.models` file. + +Database migrations are performed using alembic_. +When the database is modified, a new migration script can be autogenerated using:: + + docker-compose run --rm web flask db migrate -m "revision message" + +Resulting script should be modified according to the desired behaviour. Looking at previously migration scripts +under the `migration/versions <https://gitlab.esss.lu.se/ics-infrastructure/csentry/-/tree/master/migrations/versions>`_ +directory can help. + + +Full-text Search +---------------- + +Search is performed using Elasticsearch_ which provides very powerful capabilities. +Implementation was inspired by `The Flask Mega-Tutorial Part XVI <https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-xvi-full-text-search>`_. + +.. note:: + + Elasticsearch index has to be kept in sync with the Postgres database. This is done using SQLAlchemy event listeners. + +The `List items <https://csentry.esss.lu.se/inventory/items>`_ and `List hosts <https://csentry.esss.lu.se/network/hosts>`_ pages +use Elasticsearch_ to display the paginated list of results. +If the index is empty or not up-to-date, missing items/hosts won't be displayed (even if they are in the postgres database). + +To make a class searchable, it should be a subclass of the :class:`~app.models.SearchableMixin` class and define the fields to index in the `__mapping__` class attribute. +By default, the object will be kept in sync in the index thanks to the event listeners defined in the :class:`~app.models.SearchableMixin` class. +Note that if you define a field that isn't a database column, you have to make sure it is kept up-to-date in the Elasticsearch index. +An example is the :attr:`~app.models.Host.sensitive` field on the :class:`~app.models.Host` class. It's a property that comes from the network. +Updating the sensitive field on a network won't trigger an update of the Host objects. +A specific event listener :meth:`~app.models.update_host_sensitive_field` had to be implemented in that case. + + + +Background Tasks +---------------- + +A web application should reply to a client as fast as possible. +Long running tasks can't be performed in the main process and block the response. + +Solving this is usually done using a task queue, a mechanism to send tasks to workers. +In CSEntry, background tasks are run using RQ_, which is backed by Redis_. + +A subclass of RQ_ `Worker` is used to keep tasks results in the Postgres database: :class:`~app.tasks.TaskWorker`. +This allows the user to see the `tasks <https://csentry.esss.lu.se/task/tasks>`_ that he created. +Admin users have access to all tasks. + +The workers use the same docker image as the main application. +They are started using the `flask runworker` subcommand, giving them access to the same settings as the web application. + +Some tasks are triggered automatically, based on SQLAlchemy event listeners, to keep the csentry inventory in sync in AWX for example. +Some are triggered manually, like to create a VM for example. + +Most tasks are used to trigger templates in AWX via the API. +But they can be used for internal processing as well, like reindexing Ansible groups in Elasticsearch_. + +Permissions +----------- + +Using CSEntry requires each user to login with his/her ESS username. +Any logged user has read access to most information, except `network scopes <https://csentry.esss.lu.se/network/scopes>`_ +and sensitive `networks <https://csentry.esss.lu.se/network/networks>`_. + +Write access requires to be part of a specific group. +There are several roles/groups in CSEntry. +Each internal group has to be mapped to a list of LDAP groups. + +Internal groups ++++++++++++++++ + +admin +~~~~~ + +Admin users have full power and access to the application. +They can modify anything via the `admin interface <https://csentry.esss.lu.se/admin/>`_. +They can also monitor background jobs via the `RQ Dashboard <https://csentry.esss.lu.se/rq/>`_. + +auditor +~~~~~~~ + +Auditor users have read-only access to everything in the application (including sensitive networks). +They don't have any write access. + +inventory +~~~~~~~~~ + +Users part of the inventory group have read and write access +to the `Inventory <https://csentry.esss.lu.se/inventory/items>`_ to register items. + +network +~~~~~~~ + +*network* is used to gather groups based on the network scope. It shouldn't be mapped to a LDAP group. +Only network scope groups should. + +If a user is part of a network scope group, it is added automatically to the *network* group. +The user will have write access to all hosts in this scope, including on sensitive networks. +He will still have read-only access to hosts on admin only networks. + +Admin users have automatically access to all network scopes. + +Configuration ++++++++++++++ + +The *admin*, *auditor* and *inventory* groups mapping shall be defined in the :attr:`~app.settings.CSENTRY_LDAP_GROUPS` dictionary:: + + CSENTRY_LDAP_GROUPS = { + "admin": ["LDAP admin group"], + "auditor": ["another group"], + "inventory": ["group1", "group2"], + } + +For networks, groups based on the network scope name shall be defined +in the :attr:`~app.settings.CSENTRY_NETWORK_SCOPES_LDAP_GROUPS` dictionary:: + + CSENTRY_NETWORK_SCOPES_LDAP_GROUPS = { + "TechnicalNetwork": ["group-tn"], + "LabNetworks": ["group-lab", "group-tn"], + } + +With the above settings, a user part of *group-tn* will have access to both the `TechnicalNetwork` and `LabNetworks` scopes. +While a user part of the *group-lab* will only have access to the `LabNetworks` scope. +If a network scope isn't defined in this dictionary, only admin users will have access to it. + +Usage ++++++ + +Every endpoint should be protected using either the `login_required` or :meth:`~app.decorators.login_groups_accepted` decorator. +The first will give access to any logged user. With the second, a list of internal groups should be given. + +To restrict access to admin users only:: + + @login_groups_accepted("admin") + def create_domain(): + ... + + +When using the *network* group, an additional check inside the function is required to check that the current user +has the proper access (based on the network):: + + @login_groups_accepted("admin", "network") + def edit_interface(name): + interface = models.Interface.query.filter_by(name=name).first_or_404() + if not current_user.has_access_to_network(interface.network): + abort(403) + + +.. _Flask: https://flask.palletsprojects.com +.. _The Flask Mega-Tutorial: https://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-i-hello-world +.. _PostgreSQL: https://www.postgresql.org +.. _SQLAlchemy: https://www.sqlalchemy.org +.. _Redis: https://redis.io +.. _Elasticsearch: https://www.elastic.co/elasticsearch/ +.. _RQ: https://python-rq.org +.. _alembic: https://alembic.sqlalchemy.org diff --git a/docs/development/endpoints.rst b/docs/development/endpoints.rst new file mode 100644 index 0000000000000000000000000000000000000000..b99da5cb83cd031a2fb8c07138a550295277bad4 --- /dev/null +++ b/docs/development/endpoints.rst @@ -0,0 +1,34 @@ +Endpoints +========= + +Main blueprint +-------------- + +.. autoflask:: wsgi:app + :modules: app.main.views + :include-empty-docstring: + :order: path + +User blueprint +-------------- + +.. autoflask:: wsgi:app + :modules: app.user.views + :include-empty-docstring: + :order: path + +Inventory blueprint +------------------- + +.. autoflask:: wsgi:app + :modules: app.inventory.views + :include-empty-docstring: + :order: path + +Network blueprint +----------------- + +.. autoflask:: wsgi:app + :modules: app.network.views + :include-empty-docstring: + :order: path \ No newline at end of file diff --git a/docs/development/implementation/commands.rst b/docs/development/implementation/commands.rst new file mode 100644 index 0000000000000000000000000000000000000000..35f3493ea1b1482907f4cbe7dabbe26e0137f186 --- /dev/null +++ b/docs/development/implementation/commands.rst @@ -0,0 +1,3 @@ +.. automodule:: app.commands + :members: + :undoc-members: diff --git a/docs/development/implementation/decorators.rst b/docs/development/implementation/decorators.rst new file mode 100644 index 0000000000000000000000000000000000000000..4f36ae5a600f4f2d5019d553cee8a0e8aabee60e --- /dev/null +++ b/docs/development/implementation/decorators.rst @@ -0,0 +1,3 @@ +.. automodule:: app.decorators + :members: + :undoc-members: diff --git a/docs/development/implementation/index.rst b/docs/development/implementation/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..ce3818a98251360a78b0420bd4bd3e85ec527bd5 --- /dev/null +++ b/docs/development/implementation/index.rst @@ -0,0 +1,17 @@ +Implementation +============== + +The main modules are described below. +Full source code is available in `GitLab <https://gitlab.esss.lu.se/ics-infrastructure/csentry>`_. + +.. toctree:: + :maxdepth: 2 + + models + commands + decorators + search + tasks + user_view + inventory_view + network_view diff --git a/docs/development/implementation/inventory_view.rst b/docs/development/implementation/inventory_view.rst new file mode 100644 index 0000000000000000000000000000000000000000..c01a5aba066c86348df0005c89be520a8de69f9d --- /dev/null +++ b/docs/development/implementation/inventory_view.rst @@ -0,0 +1,3 @@ +.. automodule:: app.inventory.views + :members: + :undoc-members: diff --git a/docs/development/implementation/models.rst b/docs/development/implementation/models.rst new file mode 100644 index 0000000000000000000000000000000000000000..99654669b9f1896ad6de9dd7401bef4934893cc9 --- /dev/null +++ b/docs/development/implementation/models.rst @@ -0,0 +1,3 @@ +.. automodule:: app.models + :members: + :undoc-members: diff --git a/docs/development/implementation/network_view.rst b/docs/development/implementation/network_view.rst new file mode 100644 index 0000000000000000000000000000000000000000..7215f903ac6c71014d1f34bbe9e53845d2fb3f9b --- /dev/null +++ b/docs/development/implementation/network_view.rst @@ -0,0 +1,3 @@ +.. automodule:: app.network.views + :members: + :undoc-members: diff --git a/docs/development/implementation/search.rst b/docs/development/implementation/search.rst new file mode 100644 index 0000000000000000000000000000000000000000..bcc470426d9dcecfc6642d1a74e542afe3134a18 --- /dev/null +++ b/docs/development/implementation/search.rst @@ -0,0 +1,3 @@ +.. automodule:: app.search + :members: + :undoc-members: diff --git a/docs/development/implementation/tasks.rst b/docs/development/implementation/tasks.rst new file mode 100644 index 0000000000000000000000000000000000000000..1d8d1a1d183269a443a14ade78e52d29a9a9bda0 --- /dev/null +++ b/docs/development/implementation/tasks.rst @@ -0,0 +1,3 @@ +.. automodule:: app.tasks + :members: + :undoc-members: diff --git a/docs/development/implementation/user_view.rst b/docs/development/implementation/user_view.rst new file mode 100644 index 0000000000000000000000000000000000000000..c4712c38740f12e6bfc1b1e4c77027c38437ea4e --- /dev/null +++ b/docs/development/implementation/user_view.rst @@ -0,0 +1,3 @@ +.. automodule:: app.user.views + :members: + :undoc-members: diff --git a/docs/index.rst b/docs/index.rst index becba50e5b1a7c3928cc8526fd851301368e1889..0f036589cdca03a6d36db9cd6df7f16302985720 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -45,6 +45,7 @@ Please use the navigation sidebar on the left to begin. .. toctree:: :hidden: + :caption: User documentation :maxdepth: 3 inventory @@ -53,3 +54,13 @@ Please use the navigation sidebar on the left to begin. profile api changelog + +.. toctree:: + :hidden: + :caption: Developer documentation + :maxdepth: 3 + + development/design + development/deployment + development/endpoints + development/implementation/index