diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3ec6b9b135935f6425161cbcc18bdb3947eb6bfb..a88b56132eb4c7971bd08a32d01f20d1b00655cd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -46,6 +46,8 @@ test:
services:
- postgres:10
- redis:4.0
+ - name: docker.elastic.co/elasticsearch/elasticsearch:6.4.1
+ alias: elasticsearch
before_script:
- pip install -r requirements-dev.txt
script:
diff --git a/app/commands.py b/app/commands.py
index 678ebaa04f019599e430edf9167c7dddf0dc8d0d..f1af4d2a8b67c7194779ea117bb7843dd9be35db 100644
--- a/app/commands.py
+++ b/app/commands.py
@@ -115,3 +115,9 @@ def register_cli(app):
)
register_sentry(client, worker)
worker.work()
+
+ @app.cli.command()
+ def reindex():
+ """Initialize the elasticsearch index"""
+ current_app.elasticsearch.indices.delete("*", ignore=404)
+ models.Item.reindex()
diff --git a/app/inventory/views.py b/app/inventory/views.py
index ef61248334e06b94fb707da87c515f446ce3047d..820dcd171842d7d56eb461bb5a0b7cf318eafb3b 100644
--- a/app/inventory/views.py
+++ b/app/inventory/views.py
@@ -30,90 +30,10 @@ from .. import utils, models
bp = Blueprint("inventory", __name__)
-@bp.route("/_retrieve_items")
+@bp.route("/_retrieve_items", methods=["POST"])
@login_required
def retrieve_items():
- # Get the parameters from the query string sent by datatables
- draw = int(request.args.get("draw", 0))
- start = int(request.args.get("start", 0))
- per_page = int(request.args.get("length", 20))
- search = request.args.get("search[value]", "")
- order_column = int(request.args.get("order[0][column]", 3))
- if request.args.get("order[0][dir]") == "desc":
- order_dir = sa.desc
- else:
- order_dir = sa.asc
- # Total number of items before filtering
- nb_items_total = db.session.query(sa.func.count(models.Item.id)).scalar()
- # Construct the query
- query = models.Item.query
- if search:
- if "%" not in search:
- search = f"%{search}%"
- q1 = query.filter(
- sa.or_(
- models.Item.ics_id.like(search), models.Item.serial_number.like(search)
- )
- )
- q2 = query.join(models.Item.manufacturer).filter(
- models.Manufacturer.name.like(search)
- )
- q3 = query.join(models.Item.model).filter(
- sa.or_(
- models.Model.name.like(search), models.Model.description.like(search)
- )
- )
- q4 = query.join(models.Item.location).filter(models.Location.name.like(search))
- q5 = query.join(models.Item.status).filter(models.Status.name.like(search))
- q6 = query.join(models.Item.comments).filter(
- models.ItemComment.body.like(search)
- )
- query = q1.union(q2).union(q3).union(q4).union(q5).union(q6)
- nb_items_filtered = query.order_by(None).count()
- else:
- nb_items_filtered = nb_items_total
- # Construct the order_by query
- columns = (
- "id",
- "ics_id",
- "created_at",
- "updated_at",
- "serial_number",
- "quantity",
- "manufacturer",
- "model",
- "location",
- "status",
- "parent",
- )
- query = query.order_by(order_dir(getattr(models.Item, columns[order_column])))
- # Limit and offset the query
- if per_page != -1:
- query = query.limit(per_page)
- query = query.offset(start)
- data = [
- [
- item.id,
- item.ics_id,
- utils.format_field(item.created_at),
- utils.format_field(item.updated_at),
- item.serial_number,
- item.quantity,
- utils.format_field(item.manufacturer),
- utils.format_field(item.model),
- utils.format_field(item.location),
- utils.format_field(item.status),
- utils.format_field(item.parent),
- ]
- for item in query.all()
- ]
- response = {
- "draw": draw,
- "recordsTotal": nb_items_total,
- "recordsFiltered": nb_items_filtered,
- "data": data,
- }
- return jsonify(response)
+ return utils.retrieve_data_for_datatables(request.values, models.Item)
@bp.route("/items")
diff --git a/app/models.py b/app/models.py
index 4832add6af07bc80d2a48b4bb82479538bdbf421..875410d588f9ac46858fd3f30b71fbf73e0a9028 100644
--- a/app/models.py
+++ b/app/models.py
@@ -15,6 +15,7 @@ import string
import qrcode
import itertools
import urllib.parse
+import elasticsearch
import sqlalchemy as sa
from enum import Enum
from sqlalchemy.ext.declarative import declared_attr
@@ -36,7 +37,7 @@ from .validators import (
DEVICE_TYPE_RE,
TAG_RE,
)
-from . import utils
+from . import utils, search
make_versioned(plugins=[FlaskUserPlugin()])
@@ -319,6 +320,62 @@ class User(db.Model, UserMixin):
}
+class SearchableMixin(object):
+ """Add search capability to a class"""
+
+ @classmethod
+ def search(cls, query, page, per_page, sort=None):
+ try:
+ ids, total = search.query_index(
+ cls.__tablename__ + current_app.config["ELASTICSEARCH_INDEX_SUFFIX"],
+ query,
+ page,
+ per_page,
+ sort,
+ )
+ except elasticsearch.ElasticsearchException as e:
+ # Invalid query
+ current_app.logger.warning(e)
+ return cls.query.filter_by(id=0), 0
+ if total == 0:
+ return cls.query.filter_by(id=0), 0
+ when = [(value, i) for i, value in enumerate(ids)]
+ return (
+ cls.query.filter(cls.id.in_(ids)).order_by(db.case(when, value=cls.id)),
+ total,
+ )
+
+ @classmethod
+ def after_flush(cls, session, flush_context):
+ # Trigger the elasticsearch index update
+ # We don't use the after_commit event because calling
+ # model.to_dict() often requires to load some relationships
+ # which is not possible in the session state during after_commit
+ for obj in itertools.chain(session.new, session.dirty):
+ if isinstance(obj, SearchableMixin):
+ search.add_to_index(
+ obj.__tablename__
+ + current_app.config["ELASTICSEARCH_INDEX_SUFFIX"],
+ obj,
+ )
+ for obj in session.deleted:
+ if isinstance(obj, SearchableMixin):
+ search.remove_from_index(
+ obj.__tablename__
+ + current_app.config["ELASTICSEARCH_INDEX_SUFFIX"],
+ obj,
+ )
+
+ @classmethod
+ def reindex(cls):
+ """Force to reindex all instances of the class"""
+ for obj in cls.query:
+ search.add_to_index(
+ cls.__tablename__ + current_app.config["ELASTICSEARCH_INDEX_SUFFIX"],
+ obj,
+ )
+
+
class Token(db.Model):
"""Table to store valid tokens"""
@@ -438,7 +495,7 @@ class CreatedMixin:
}
-class Item(CreatedMixin, db.Model):
+class Item(CreatedMixin, SearchableMixin, db.Model):
__versioned__ = {
"exclude": [
"created_at",
@@ -1340,3 +1397,5 @@ def before_flush(session, flush_context, instances):
# required by sqlalchemy_continuum
sa.orm.configure_mappers()
ItemVersion = version_class(Item)
+# Set SQLAlchemy event listeners
+db.event.listen(db.session, "after_flush", SearchableMixin.after_flush)
diff --git a/app/search.py b/app/search.py
new file mode 100644
index 0000000000000000000000000000000000000000..051692dc37efd4e0dbdcb198954a8eb50cb0c88d
--- /dev/null
+++ b/app/search.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+app.search
+~~~~~~~~~~
+
+This module implements the search interface.
+
+:copyright: (c) 2018 European Spallation Source ERIC
+:license: BSD 2-Clause, see LICENSE for more details.
+
+"""
+from flask import current_app
+
+
+def add_to_index(index, model):
+ """Add a model instance to an index"""
+ if not current_app.elasticsearch:
+ return
+ body = model.to_dict()
+ # We remove the id key because it's used as the id of the document.
+ # Using the model id as document id allows to easily retrieve
+ # the model linked to a document
+ del body["id"]
+ current_app.elasticsearch.index(
+ index=index,
+ doc_type="_doc",
+ id=model.id,
+ body=body,
+ refresh=current_app.config["ELASTICSEARCH_REFRESH"],
+ )
+
+
+def remove_from_index(index, model):
+ """Remove a model from the index"""
+ if not current_app.elasticsearch:
+ return
+ current_app.elasticsearch.delete(
+ index=index,
+ doc_type="_doc",
+ id=model.id,
+ refresh=current_app.config["ELASTICSEARCH_REFRESH"],
+ )
+
+
+def query_index(index, query, page=1, per_page=20, sort=None):
+ """Run a search query on the index
+
+ Return a BaseQuery (with the ids found) and the total number of hits as a tuple
+ """
+ if not current_app.elasticsearch:
+ return [], 0
+ kwargs = {
+ "index": index,
+ "doc_type": "_doc",
+ "q": query,
+ "from_": (page - 1) * per_page,
+ "size": per_page,
+ }
+ if sort is not None:
+ kwargs["sort"] = sort
+ search = current_app.elasticsearch.search(**kwargs)
+ ids = [int(hit["_id"]) for hit in search["hits"]["hits"]]
+ return ids, search["hits"]["total"]
diff --git a/app/settings.py b/app/settings.py
index 254a6e7a2b725daa725836bc757c9d8d9b18d357..fa946af37d433065ebcba4f29505302a54e5fe7b 100644
--- a/app/settings.py
+++ b/app/settings.py
@@ -37,6 +37,11 @@ REDIS_URL = "redis://redis:6379/2"
QUEUES = ["default"]
ELASTICSEARCH_URL = "http://elasticsearch:9200"
+ELASTICSEARCH_INDEX_SUFFIX = "-dev"
+# Shall only be set to "true" for testing to make
+# documents visible for search immediately
+# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-refresh.html
+ELASTICSEARCH_REFRESH = "false"
LDAP_HOST = "esss.lu.se"
LDAP_BASE_DN = "DC=esss,DC=lu,DC=se"
diff --git a/app/static/js/items.js b/app/static/js/items.js
index 5eeccef6249f919644b0176639e5377c8aeda139..060647b55bf6ad67d77d47e51ea329db10ec6aa6 100644
--- a/app/static/js/items.js
+++ b/app/static/js/items.js
@@ -1,5 +1,14 @@
$(document).ready(function() {
+ function render_item_link(data) {
+ // render funtion to create link to Item view page
+ if ( data === null ) {
+ return data;
+ }
+ var url = $SCRIPT_ROOT + "/inventory/items/view/" + data;
+ return '<a href="'+ url + '">' + data + '</a>';
+ }
+
// scroll up to avoid having the form input
// hidden under the navbar
if (location.hash == "#body") {
@@ -59,35 +68,34 @@ $(document).ready(function() {
"<'row'<'col-sm-12'tr>>" +
"<'row'<'col-sm-12 col-md-5'i><'col-sm-12 col-md-7'p>>",
"ajax": {
- "url": $SCRIPT_ROOT + "/inventory/_retrieve_items"
+ "url": $SCRIPT_ROOT + "/inventory/_retrieve_items",
+ "type": "POST"
},
"processing": true,
"serverSide": true,
"searchDelay": 500,
- "order": [[3, 'desc']],
"orderMulti": false,
+ "aaSorting": [],
"pagingType": "full_numbers",
"pageLength": 20,
- "lengthMenu": [[20, 50, 100, -1], [20, 50, 100, "All"]],
- "columnDefs": [
- {
- "targets": [0],
- "visible": false,
- "searchable": false
- },
- {
- "targets": [6, 7, 8, 9],
- "orderable": false
+ "lengthMenu": [[20, 50, 100], [20, 50, 100]],
+ "columns": [
+ { data: 'ics_id',
+ render: function(data, type, row) {
+ return render_item_link(data);
+ }
},
- {
- "targets": [1, 10],
- "render": function(data, type, row) {
- // render funtion to create link to Item view page
- if ( data === null ) {
- return data;
- }
- var url = $SCRIPT_ROOT + "/inventory/items/view/" + data;
- return '<a href="'+ url + '">' + data + '</a>';
+ { data: 'created_at' },
+ { data: 'updated_at' },
+ { data: 'serial_number' },
+ { data: 'quantity' },
+ { data: 'manufacturer' },
+ { data: 'model' },
+ { data: 'location' },
+ { data: 'status' },
+ { data: 'parent',
+ render: function(data, type, row) {
+ return render_item_link(data);
}
}
]
diff --git a/app/templates/inventory/items.html b/app/templates/inventory/items.html
index 31f59bfcfd8731a4428cb0ebe9dc784f670f6529..7c13e2861018d85af1eddf6904d17214d1601703 100644
--- a/app/templates/inventory/items.html
+++ b/app/templates/inventory/items.html
@@ -22,7 +22,6 @@
<table id="items_table" class="table table-bordered table-hover table-sm" style="width:100%">
<thead>
<tr>
- <th>Id</th>
<th>ICS id</th>
<th>Created</th>
<th>Updated</th>
diff --git a/app/utils.py b/app/utils.py
index a70a9e1f99274b61b9997a6ccd80b1ec0ed5d707..268160abefc8901c4b7223a6dbe2c13a710b1bbd 100644
--- a/app/utils.py
+++ b/app/utils.py
@@ -20,6 +20,7 @@ from pathlib import Path
from flask import current_app, jsonify, url_for
from flask.globals import _app_ctx_stack, _request_ctx_stack
from flask_login import current_user
+from .extensions import db
def fetch_current_user_id():
@@ -279,3 +280,51 @@ def unique_filename(filename):
break
nb += 1
return unique
+
+
+def retrieve_data_for_datatables(values, model):
+ """Return the filtered data of model to datatables
+
+ This function is supposed to be called when using datatables
+ with serverSide processing.
+
+ :param values: a `~werkzeug.datastructures.MultiDict`, typically request.values
+ :param model: class of the model to query
+ :return: json object required by datatables
+ """
+ # Get the parameters from the post data sent by datatables
+ draw = values.get("draw", 0, type=int)
+ per_page = values.get("length", 20, type=int)
+ # page starts at 1 in elasticsearch
+ page = int(values.get("start", 0, type=int) / per_page) + 1
+ search = values.get("search[value]", "")
+ if search == "":
+ search = "*"
+ order_column = values.get("order[0][column]")
+ if order_column is None:
+ sort = None
+ else:
+ name = values.get(f"columns[{order_column}][data]")
+ order_dir = values.get("order[0][dir]", "asc")
+ # For all fields of type text, sorting should be done
+ # on .keyword in elasticsearch
+ # quantity (in items table) is of type integer
+ # It's hard-coded here for now. If needed we can try to find a
+ # generic way to find this information.
+ if name == "quantity":
+ sort = f"{name}:{order_dir}"
+ else:
+ sort = f"{name}.keyword:{order_dir}"
+ instances, nb_filtered = model.search(
+ search, page=page, per_page=per_page, sort=sort
+ )
+ data = [instance.to_dict() for instance in instances]
+ # Total number of items before filtering
+ nb_total = db.session.query(sa.func.count(model.id)).scalar()
+ response = {
+ "draw": draw,
+ "recordsTotal": nb_total,
+ "recordsFiltered": nb_filtered,
+ "data": data,
+ }
+ return jsonify(response)
diff --git a/docs/_static/inventory/export_to_excel.png b/docs/_static/inventory/export_to_excel.png
new file mode 100644
index 0000000000000000000000000000000000000000..ce6831139b5d8132d7223e3c8403c9b713032582
Binary files /dev/null and b/docs/_static/inventory/export_to_excel.png differ
diff --git a/docs/_static/inventory/search_exists_parent.png b/docs/_static/inventory/search_exists_parent.png
new file mode 100644
index 0000000000000000000000000000000000000000..3599f315900b8d6144f73f3cceac91107171f7db
Binary files /dev/null and b/docs/_static/inventory/search_exists_parent.png differ
diff --git a/docs/_static/inventory/search_ics_id_field.png b/docs/_static/inventory/search_ics_id_field.png
new file mode 100644
index 0000000000000000000000000000000000000000..fcd0e9287062ef69311be65fa9436b4b7d044c09
Binary files /dev/null and b/docs/_static/inventory/search_ics_id_field.png differ
diff --git a/docs/_static/inventory/search_ics_id_multiple.png b/docs/_static/inventory/search_ics_id_multiple.png
new file mode 100644
index 0000000000000000000000000000000000000000..bc7a5645522d2c1000d346c908169e734fc4b9e1
Binary files /dev/null and b/docs/_static/inventory/search_ics_id_multiple.png differ
diff --git a/docs/_static/inventory/search_juniper.png b/docs/_static/inventory/search_juniper.png
new file mode 100644
index 0000000000000000000000000000000000000000..a7ea6e18d1bc037beeafdd6156dd873be7c323fe
Binary files /dev/null and b/docs/_static/inventory/search_juniper.png differ
diff --git a/docs/_static/inventory/search_juniper_not_rats.png b/docs/_static/inventory/search_juniper_not_rats.png
new file mode 100644
index 0000000000000000000000000000000000000000..ea1d5efab0ec9edf69d6bafb337de2e30d6b73c4
Binary files /dev/null and b/docs/_static/inventory/search_juniper_not_rats.png differ
diff --git a/docs/_static/inventory/search_quote_mac.png b/docs/_static/inventory/search_quote_mac.png
new file mode 100644
index 0000000000000000000000000000000000000000..ee10090aa20d38e42ecc65c694e71ecf182b5b72
Binary files /dev/null and b/docs/_static/inventory/search_quote_mac.png differ
diff --git a/docs/_static/inventory/search_quote_tag.png b/docs/_static/inventory/search_quote_tag.png
new file mode 100644
index 0000000000000000000000000000000000000000..390be85a52812034e34f6198ede5922d339d81af
Binary files /dev/null and b/docs/_static/inventory/search_quote_tag.png differ
diff --git a/docs/_static/inventory/search_serial_number.png b/docs/_static/inventory/search_serial_number.png
new file mode 100644
index 0000000000000000000000000000000000000000..9af6aaa3094e4cb6e240e14adca08cb1d5566205
Binary files /dev/null and b/docs/_static/inventory/search_serial_number.png differ
diff --git a/docs/inventory.rst b/docs/inventory.rst
index dd3cd23f70b803becf57cb9fe4941ae8a61836c4..e6044ea0e55e47375b3b90e162b521d42c8f4e56 100644
--- a/docs/inventory.rst
+++ b/docs/inventory.rst
@@ -15,25 +15,78 @@ An item can have a parent.
Only **ICS id** and **serial number** fields are mandatory.
The **serial number** is a free text field.
-The *items* page of the inventory lists all existing items. You can find a specific item or goup of items
-by using the search box.
-The search is performed on the following fields:
+The *items* page of the inventory lists all existing items.
- - ICS id
- - serial number
- - manufacturer name
- - model name
- - location name
- - status name
- - item comments
+.. image:: _static/inventory.png
-Note that the search is case-sensitive!
+Search
+------
-.. image:: _static/inventory.png
+You can find a specific item or group of items by using the search box.
+Elasticsearch is used as the full-text search engine and allows to use a query string “mini-languageâ€.
+
+- To perform a free text search, simply enter a text string. For example, if you’re looking for Juniper devices, enter ``juniper``. Note that the search is case insensitive.
+
+.. image:: _static/inventory/search_juniper.png
+
+- You can search a serial number or ICS id.
+
+.. image:: _static/inventory/search_serial_number.png
+
+.. image:: _static/inventory/search_ics_id_multiple.png
+
+- As you can see on the above screenshot, by default the search is performed on all fields. The id ``AAA997`` was found both in the ics_id and parent fields. You can restrict the search to a specific field: ``ics_id:AAA997``
+
+.. image:: _static/inventory/search_ics_id_field.png
+
+- By default, all terms are optional, as long as one term matches. A search for ``foo bar baz`` will find any document that contains one or more of foo or bar or baz. Boolean operators can be used to provide more control. The preferred operators are + (this term must be present) and - (this term must not be present). For example to search all juniper devices not at RATS: ``juniper -RATS``
+
+.. image:: _static/inventory/search_juniper_not_rats.png
+
+- You can search items where a field has any non-null value. To get items with a parent: ``_exists_:parent``
+
+.. image:: _static/inventory/search_exists_parent.png
+
+- To search a string that includes reserved characters (+ - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /), you have to use double quotes. Here is how to search for an old JIRA TAG ``"TAG-250"`` or mac adress ``"7c:e2:ca:64:dd:61"``.
+
+.. image:: _static/inventory/search_quote_tag.png
+
+.. image:: _static/inventory/search_quote_mac.png
+
+The complete list of fields that can be searched is the following:
+
+ - created_at
+ - updated_at
+ - user
+ - ics_id
+ - serial_number
+ - quantity
+ - manufacturer
+ - model
+ - location
+ - status
+ - parent
+ - children
+ - macs
+ - host
+ - stack_member
+ - history
+ - comments
+
+Check the elasticsearch `query string syntax`_ for more details.
+
+
+Export items to Excel
+---------------------
You can export all items to an excel file by clicking on the *Excel* button.
Note that it might take some time depending on the number of items in the database.
+.. image:: _static/inventory/export_to_excel.png
+
+View an item
+-------------
+
Clicking on the **ICS id** of an item will take you to the *View item* page.
.. image:: _static/view_item.png
@@ -123,3 +176,6 @@ Scan the following codes to change the Xenon 1900 scanner settings:
.. image:: _static/xenon/usb-serial.png
:align: center
+
+
+.. _query string syntax: https://www.elastic.co/guide/en/elasticsearch/reference/6.4/query-dsl-query-string-query.html#query-string-syntax
diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py
index 7bace7fd3ce3cac2079cf717bd08e4b8a5078eb9..1ff32a3d36832fd8a37859322d3e4bc85eea1d89 100644
--- a/tests/functional/conftest.py
+++ b/tests/functional/conftest.py
@@ -15,6 +15,7 @@ from pytest_factoryboy import register
from flask_ldap3_login import AuthenticationResponse, AuthenticationResponseStatus
from app.factory import create_app
from app.extensions import db as _db
+from app.models import SearchableMixin
from . import common, factories
register(factories.UserFactory)
@@ -44,6 +45,8 @@ def app(request):
"TESTING": True,
"WTF_CSRF_ENABLED": False,
"SQLALCHEMY_DATABASE_URI": "postgresql://ics:icspwd@postgres/csentry_db_test",
+ "ELASTICSEARCH_INDEX_SUFFIX": "-test",
+ "ELASTICSEARCH_REFRESH": "true",
"CSENTRY_LDAP_GROUPS": {
"admin": ["CSEntry Admin"],
"create": ["CSEntry User", "CSEntry Consultant"],
@@ -104,10 +107,16 @@ def session(db, request):
session.expire_all()
session.begin_nested()
+ # We have to register the after_flush event because we use a specific
+ # session to run the tests (not the same used in models.py)
+ db.event.listen(session(), "after_flush", SearchableMixin.after_flush)
db.session = session
yield session
+ # ELASTICSEARCH_INDEX_SUFFIX is set to "-test"
+ # Delete all "*-test" indices after each test
+ db.app.elasticsearch.indices.delete("*-test", ignore=404)
session.remove()
transaction.rollback()
connection.close()
diff --git a/tests/functional/test_search.py b/tests/functional/test_search.py
new file mode 100644
index 0000000000000000000000000000000000000000..feb15a2bccd71fe2091251d21dddc86c08026861
--- /dev/null
+++ b/tests/functional/test_search.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+tests.functional.test_search
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This module defines search tests.
+
+:copyright: (c) 2017 European Spallation Source ERIC
+:license: BSD 2-Clause, see LICENSE for more details.
+
+"""
+import elasticsearch
+import pytest
+from app import search
+
+
+class MyModel:
+ def __init__(self, id, name, description=""):
+ self.id = id
+ self.name = name
+ self.description = description
+
+ def to_dict(self):
+ return {"id": self.id, "name": self.name, "description": self.description}
+
+
+def test_add_to_index(db):
+ model1 = MyModel(2, "foo", "This is a test")
+ search.add_to_index("index-test", model1)
+ res = db.app.elasticsearch.get(index="index-test", doc_type="_doc", id=2)
+ assert res["_source"] == {"name": "foo", "description": "This is a test"}
+
+
+def test_remove_from_index(db):
+ model1 = MyModel(3, "hello world!")
+ search.add_to_index("index-test", model1)
+ res = db.app.elasticsearch.search(index="index-test", q="*")
+ assert res["hits"]["total"] == 1
+ search.remove_from_index("index-test", model1)
+ res = db.app.elasticsearch.search(index="index-test", q="*")
+ assert res["hits"]["total"] == 0
+
+
+def test_remove_from_index_non_existing():
+ model1 = MyModel(1, "hello world!")
+ with pytest.raises(elasticsearch.NotFoundError):
+ search.remove_from_index("index-test", model1)
+
+
+def test_query_index():
+ model1 = MyModel(1, "Python", "Python is my favorite language")
+ search.add_to_index("index-test", model1)
+ model1 = MyModel(2, "Java", "Your should switch to Python!")
+ search.add_to_index("index-test", model1)
+ # Test query all
+ ids, total = search.query_index("index-test", "*")
+ assert sorted(ids) == [1, 2]
+ assert total == 2
+ # Test query string
+ ids, total = search.query_index("index-test", "java")
+ assert ids == [2]
+ assert total == 1
+ ids, total = search.query_index("index-test", "python")
+ assert sorted(ids) == [1, 2]
+ # Test query specific field
+ ids, total = search.query_index("index-test", "name:python")
+ assert ids == [1]
+ # Test query sort
+ ids, total = search.query_index("index-test", "*", sort="name.keyword")
+ assert ids == [2, 1]
diff --git a/tests/functional/test_web.py b/tests/functional/test_web.py
index 12878b0cc5343e03032a49419f6cb5aa0cb608e1..cd78c59e981eee46aefb112db2a32eca10676d24 100644
--- a/tests/functional/test_web.py
+++ b/tests/functional/test_web.py
@@ -46,10 +46,8 @@ def test_index(logged_client):
assert b"User RO" in response.data
-@pytest.mark.parametrize(
- "url", ["/", "/inventory/items", "/inventory/_retrieve_items", "/network/networks"]
-)
-def test_protected_url(url, client):
+@pytest.mark.parametrize("url", ["/", "/inventory/items", "/network/networks"])
+def test_protected_url_get(url, client):
response = client.get(url)
assert response.status_code == 302
assert "/user/login" in response.headers["Location"]
@@ -58,16 +56,105 @@ def test_protected_url(url, client):
assert response.status_code == 200
+@pytest.mark.parametrize("url", ["/inventory/_retrieve_items"])
+def test_protected_url_post(url, client):
+ response = client.post(url)
+ assert response.status_code == 302
+ assert "/user/login" in response.headers["Location"]
+ login(client, "user_ro", "userro")
+ response = client.post(url)
+ assert response.status_code == 200
+
+
def test_retrieve_items(logged_client, item_factory):
- response = logged_client.get("/inventory/_retrieve_items")
+ response = logged_client.post("/inventory/_retrieve_items")
assert response.get_json()["data"] == []
serial_numbers = ("12345", "45678")
for sn in serial_numbers:
item_factory(serial_number=sn)
- response = logged_client.get("/inventory/_retrieve_items")
+ response = logged_client.post("/inventory/_retrieve_items")
+ items = response.get_json()["data"]
+ assert set(serial_numbers) == set(item["serial_number"] for item in items)
+ assert len(items[0]) == 18
+
+
+def test_retrieve_items_pagination(logged_client, item_factory):
+ for sn in range(1000, 1030):
+ item_factory(serial_number=sn)
+ response = logged_client.post(
+ "/inventory/_retrieve_items", data={"draw": "50", "length": 10, "start": 0}
+ )
+ r = response.get_json()
+ assert r["draw"] == 50
+ assert r["recordsTotal"] == 30
+ assert r["recordsFiltered"] == 30
+ assert len(r["data"]) == 10
+ serial_numbers = [item["serial_number"] for item in r["data"]]
+ response = logged_client.post(
+ "/inventory/_retrieve_items", data={"draw": "51", "length": 10, "start": 10}
+ )
+ serial_numbers.extend(
+ [item["serial_number"] for item in response.get_json()["data"]]
+ )
+ response = logged_client.post(
+ "/inventory/_retrieve_items", data={"draw": "52", "length": 10, "start": 20}
+ )
+ serial_numbers.extend(
+ [item["serial_number"] for item in response.get_json()["data"]]
+ )
+ assert sorted(serial_numbers) == list(str(i) for i in range(1000, 1030))
+
+
+def test_retrieve_items_filter(logged_client, item_factory):
+ for sn in range(1000, 1010):
+ item_factory(serial_number=sn)
+ response = logged_client.post(
+ "/inventory/_retrieve_items",
+ data={
+ "draw": "50",
+ "length": 20,
+ "start": 0,
+ "search[value]": "serial_number:1005",
+ },
+ )
+ r = response.get_json()
+ assert r["recordsTotal"] == 10
+ assert r["recordsFiltered"] == 1
+ assert len(r["data"]) == 1
+ assert r["data"][0]["serial_number"] == "1005"
+
+
+def test_retrieve_items_sort(logged_client, item_factory):
+ serial_numbers = ["AAA001", "AAB034", "AAA100", "AAB001"]
+ for sn in serial_numbers:
+ item_factory(serial_number=sn)
+ response = logged_client.post(
+ "/inventory/_retrieve_items",
+ data={
+ "draw": "50",
+ "length": 20,
+ "start": 0,
+ "order[0][column]": "3",
+ "columns[3][data]": "serial_number",
+ },
+ )
+ items = response.get_json()["data"]
+ assert sorted(serial_numbers) == [item["serial_number"] for item in items]
+ response = logged_client.post(
+ "/inventory/_retrieve_items",
+ data={
+ "draw": "50",
+ "length": 20,
+ "start": 0,
+ "order[0][column]": "3",
+ "order[0][dir]": "desc",
+ "columns[3][data]": "serial_number",
+ },
+ )
items = response.get_json()["data"]
- assert set(serial_numbers) == set(item[4] for item in items)
- assert len(items[0]) == 11
+ assert sorted(serial_numbers, reverse=True) == [
+ item["serial_number"] for item in items
+ ]
def test_generate_random_mac(logged_client):