Switch to server-side processing for items table

When there are more than a 1000 items in the table, retrieving and
displaying all items takes about 7 seconds.
This is every time we refresh the items page, which is too slow.

app/inventory/views.py

@@ -24,20 +24,87 @@ bp = Blueprint('inventory', __name__)
 @bp.route('/_retrieve_items')
 @login_required
 def retrieve_items():
-    items = models.Item.query.order_by(models.Item.created_at)
-    data = [[item.id,
-             item.ics_id,
-             utils.format_field(item.created_at),
-             utils.format_field(item.updated_at),
-             item.serial_number,
-             utils.format_field(item.manufacturer),
-             utils.format_field(item.model),
-             utils.format_field(item.location),
-             utils.format_field(item.status),
-             utils.format_field(item.parent),
-             '\n'.join([comment.body for comment in item.comments]),
-             ] for item in items]
-    return jsonify(data=data)
+    # Get the parameters from the query string sent by datatables
+    draw = int(request.args.get('draw'))
+    start = int(request.args.get('start', 0))
+    per_page = int(request.args.get('length', 20))
+    search = request.args.get('search[value]', '')
+    order_column = int(request.args.get('order[0][column]'))
+    if request.args.get('order[0][dir]') == 'desc':
+        order_dir = sa.desc
+    else:
+        order_dir = sa.asc
+    # Total number of items before filtering
+    nb_items_total = db.session.query(sa.func.count(models.Item.id)).scalar()
+    # Construct the query
+    query = models.Item.query
+    if search:
+        if '%' not in search:
+            search = f'%{search}%'
+        q1 = query.filter(
+            sa.or_(
+                models.Item.ics_id.like(search),
+                models.Item.serial_number.like(search),
+            )
+        )
+        q2 = query.join(models.Item.manufacturer).filter(
+            models.Manufacturer.name.like(search))
+        q3 = query.join(models.Item.model).filter(
+            models.Model.name.like(search))
+        q4 = query.join(models.Item.location).filter(
+            models.Location.name.like(search))
+        q5 = query.join(models.Item.status).filter(
+            models.Status.name.like(search))
+        q6 = query.join(models.Item.comments).filter(
+            models.ItemComment.body.like(search))
+        query = q1.union(q2).union(q3).union(q4).union(q5).union(q6)
+        nb_items_filtered = query.order_by(None).count()
+    else:
+        nb_items_filtered = nb_items_total
+    # Construct the order_by query
+    columns = ('id',
+               'ics_id',
+               'created_at',
+               'updated_at',
+               'serial_number',
+               'manufacturer',
+               'model',
+               'location',
+               'status',
+               'parent',
+               )
+    field = getattr(models.Item, columns[order_column])
+    if 4 < order_column < 9:
+        query = query.join(field)
+        relationship_class = getattr(models, columns[order_column].title())
+        relationship_field = getattr(relationship_class, 'name')
+        query = query.order_by(order_dir(relationship_field))
+    else:
+        query = query.order_by(order_dir(getattr(models.Item, columns[order_column])))
+    # Limit and offset the query
+    if per_page != -1:
+        query = query.limit(per_page)
+    query = query.offset(start)
+    data = [
+        [item.id,
+         item.ics_id,
+         utils.format_field(item.created_at),
+         utils.format_field(item.updated_at),
+         item.serial_number,
+         utils.format_field(item.manufacturer),
+         utils.format_field(item.model),
+         utils.format_field(item.location),
+         utils.format_field(item.status),
+         utils.format_field(item.parent),
+         ] for item in query.all()
+    ]
+    response = {
+        'draw': draw,
+        'recordsTotal': nb_items_total,
+        'recordsFiltered': nb_items_filtered,
+        'data': data
+    }
+    return jsonify(response)
 
 
 @bp.route('/items')

app/static/js/items.js

@@ -47,14 +47,14 @@ $(document).ready(function() {
   });
 
   var items_table =  $("#items_table").DataTable({
-    "ajax": function(data, callback, settings) {
-      $.getJSON(
-        $SCRIPT_ROOT + "/inventory/_retrieve_items",
-        function(json) {
-          callback(json);
-        });
+    "ajax": {
+      "url": $SCRIPT_ROOT + "/inventory/_retrieve_items"
     },
+    "processing": true,
+    "serverSide": true,
+    "searchDelay": 500,
     "order": [[3, 'desc']],
+    "orderMulti": false,
     "pagingType": "full_numbers",
     "pageLength": 20,
     "lengthMenu": [[20, 50, 100, -1], [20, 50, 100, "All"]],
@@ -74,12 +74,7 @@ $(document).ready(function() {
           var url = $SCRIPT_ROOT + "/inventory/items/view/" + data;
           return '<a href="'+ url + '">' + data + '</a>';
         }
-      },
-      {
-        "targets": [10],
-        "visible": false,
-        "searchable": true
-      },
+      }
     ]
   });
 

app/templates/inventory/items.html

@@ -31,7 +31,6 @@
         <th>Location</th>
         <th>Status</th>
         <th>Parent</th>
-        <th>Comments</th>
       </tr>
     </thead>
   </table>