From 68abc67c23a27ac171fa66cfba6dea40a8ea2765 Mon Sep 17 00:00:00 2001
From: Benjamin Bertrand <benjamin.bertrand@esss.se>
Date: Sun, 11 Feb 2018 22:39:57 +0100
Subject: [PATCH] Add RQ to process jobs in the background
---
Makefile | 2 +-
app/commands.py | 11 +++++++++++
app/main/views.py | 22 +++++++++++++++++++++-
app/settings.py | 2 ++
docker-compose.override.yml | 6 ++++++
docker-compose.yml | 6 ++++++
environment.yml | 1 +
7 files changed, 48 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
index 8683aa1..78966b9 100644
--- a/Makefile
+++ b/Makefile
@@ -38,7 +38,7 @@ release: refresh \
release: ## build, tag, and push all stacks
db: ## start postgres and redis for development
- docker-compose up -d postgres redis
+ docker-compose up -d postgres redis worker
init_db: ## initialize the dev database
docker-compose run --rm web flask db upgrade head
diff --git a/app/commands.py b/app/commands.py
index 5c25236..3134399 100644
--- a/app/commands.py
+++ b/app/commands.py
@@ -10,6 +10,8 @@ This module defines extra flask commands.
"""
import ldap3
+import redis
+import rq
import sqlalchemy as sa
from flask import current_app
from .extensions import db, ldap_manager
@@ -88,3 +90,12 @@ def register_cli(app):
"""Run maintenance commands"""
sync_users()
tokens.prune_database()
+
+ @app.cli.command()
+ def runworker():
+ """Run RQ worker"""
+ redis_url = current_app.config['RQ_REDIS_URL']
+ redis_connection = redis.from_url(redis_url)
+ with rq.Connection(redis_connection):
+ worker = rq.Worker(current_app.config['QUEUES'])
+ worker.work()
diff --git a/app/main/views.py b/app/main/views.py
index 02bf0b2..dcc9375 100644
--- a/app/main/views.py
+++ b/app/main/views.py
@@ -10,8 +10,10 @@ This module implements the main blueprint.
"""
import os
-from flask import Blueprint, render_template, jsonify, request, current_app
+import redis
+from flask import Blueprint, render_template, jsonify, g, current_app
from flask_login import login_required
+from rq import push_connection, pop_connection
from .. import utils
bp = Blueprint('main', __name__)
@@ -58,6 +60,24 @@ def modified_static_file(endpoint, values):
values['m'] = int(os.stat(file_path).st_mtime)
+def get_redis_connection():
+ redis_connection = getattr(g, '_redis_connection', None)
+ if redis_connection is None:
+ redis_url = current_app.config['RQ_REDIS_URL']
+ redis_connection = g._redis_connection = redis.from_url(redis_url)
+ return redis_connection
+
+
+@bp.before_app_request
+def push_rq_connection():
+ push_connection(get_redis_connection())
+
+
+@bp.teardown_app_request
+def pop_rq_connection(exception=None):
+ pop_connection()
+
+
@bp.route('/')
@login_required
def index():
diff --git a/app/settings.py b/app/settings.py
index 97c63c3..41bee74 100644
--- a/app/settings.py
+++ b/app/settings.py
@@ -30,6 +30,8 @@ SESSION_TYPE = 'redis'
REDIS_URL = 'redis://redis:6379/0'
CACHE_TYPE = 'redis'
CACHE_REDIS_URL = 'redis://redis:6379/1'
+RQ_REDIS_URL = 'redis://redis:6379/2'
+QUEUES = ['default']
LDAP_HOST = 'esss.lu.se'
LDAP_BASE_DN = 'DC=esss,DC=lu,DC=se'
diff --git a/docker-compose.override.yml b/docker-compose.override.yml
index 2551d30..c013955 100644
--- a/docker-compose.override.yml
+++ b/docker-compose.override.yml
@@ -11,6 +11,12 @@ services:
- "8000:8000"
volumes:
- .:/app
+ worker:
+ environment:
+ LOCAL_SETTINGS: /app/settings.cfg
+ FLASK_APP: /app/wsgi.py
+ volumes:
+ - .:/app
postgres:
ports:
- "5432:5432"
diff --git a/docker-compose.yml b/docker-compose.yml
index df27b22..f765829 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,6 +7,12 @@ services:
depends_on:
- postgres
- redis
+ worker:
+ image: registry.esss.lu.se/ics-infrastructure/csentry:master
+ container_name: csentry_worker
+ command: flask runworker
+ depends_on:
+ - redis
postgres:
image: postgres:10
container_name: csentry_postgres
diff --git a/environment.yml b/environment.yml
index 1695721..2b306cc 100644
--- a/environment.yml
+++ b/environment.yml
@@ -104,6 +104,7 @@ dependencies:
- flask-session==0.3.1
- inflection==0.3.1
- pytest-factoryboy==2.0.1
+ - rq==0.10.0
- sphinxcontrib-httpdomain==1.6.0
- sqlalchemy-citext==1.3.post0
- sqlalchemy-continuum==1.3.3
--
GitLab