Add RQ to process jobs in the background

Makefile

@@ -38,7 +38,7 @@ release: refresh \
 release: ## build, tag, and push all stacks
 
 db: ## start postgres and redis for development
-	docker-compose up -d postgres redis
+	docker-compose up -d postgres redis worker
 
 init_db: ## initialize the dev database
 	docker-compose run --rm web flask db upgrade head

app/commands.py

@@ -10,6 +10,8 @@ This module defines extra flask commands.
 
 """
 import ldap3
+import redis
+import rq
 import sqlalchemy as sa
 from flask import current_app
 from .extensions import db, ldap_manager
@@ -88,3 +90,12 @@ def register_cli(app):
         """Run maintenance commands"""
         sync_users()
         tokens.prune_database()
+
+    @app.cli.command()
+    def runworker():
+        """Run RQ worker"""
+        redis_url = current_app.config['RQ_REDIS_URL']
+        redis_connection = redis.from_url(redis_url)
+        with rq.Connection(redis_connection):
+            worker = rq.Worker(current_app.config['QUEUES'])
+            worker.work()

app/main/views.py

@@ -10,8 +10,10 @@ This module implements the main blueprint.
 
 """
 import os
-from flask import Blueprint, render_template, jsonify, request, current_app
+import redis
+from flask import Blueprint, render_template, jsonify, g, current_app
 from flask_login import login_required
+from rq import push_connection, pop_connection
 from .. import utils
 
 bp = Blueprint('main', __name__)
@@ -58,6 +60,24 @@ def modified_static_file(endpoint, values):
             values['m'] = int(os.stat(file_path).st_mtime)
 
 
+def get_redis_connection():
+    redis_connection = getattr(g, '_redis_connection', None)
+    if redis_connection is None:
+        redis_url = current_app.config['RQ_REDIS_URL']
+        redis_connection = g._redis_connection = redis.from_url(redis_url)
+    return redis_connection
+
+
+@bp.before_app_request
+def push_rq_connection():
+    push_connection(get_redis_connection())
+
+
+@bp.teardown_app_request
+def pop_rq_connection(exception=None):
+    pop_connection()
+
+
 @bp.route('/')
 @login_required
 def index():

app/settings.py

@@ -30,6 +30,8 @@ SESSION_TYPE = 'redis'
 REDIS_URL = 'redis://redis:6379/0'
 CACHE_TYPE = 'redis'
 CACHE_REDIS_URL = 'redis://redis:6379/1'
+RQ_REDIS_URL = 'redis://redis:6379/2'
+QUEUES = ['default']
 
 LDAP_HOST = 'esss.lu.se'
 LDAP_BASE_DN = 'DC=esss,DC=lu,DC=se'

docker-compose.override.yml

@@ -11,6 +11,12 @@ services:
       - "8000:8000"
     volumes:
       - .:/app
+  worker:
+    environment:
+      LOCAL_SETTINGS: /app/settings.cfg
+      FLASK_APP: /app/wsgi.py
+    volumes:
+      - .:/app
   postgres:
     ports:
       - "5432:5432"

docker-compose.yml

@@ -7,6 +7,12 @@ services:
     depends_on:
       - postgres
       - redis
+  worker:
+    image: registry.esss.lu.se/ics-infrastructure/csentry:master
+    container_name: csentry_worker
+    command: flask runworker
+    depends_on:
+      - redis
   postgres:
     image: postgres:10
     container_name: csentry_postgres

environment.yml

@@ -104,6 +104,7 @@ dependencies:
   - flask-session==0.3.1
   - inflection==0.3.1
   - pytest-factoryboy==2.0.1
+  - rq==0.10.0
   - sphinxcontrib-httpdomain==1.6.0
   - sqlalchemy-citext==1.3.post0
   - sqlalchemy-continuum==1.3.3