From a8b4c0f91c54ccf72554316a8d0802adcca75e1b Mon Sep 17 00:00:00 2001
From: Benjamin Bertrand <benjamin.bertrand@esss.se>
Date: Mon, 1 Apr 2019 18:41:32 +0200
Subject: [PATCH] Initialize IOC repository on GitLab
This is temporary. The repository should be created by IOCFactory.
JIRA INFRA-932 #action In Progress
---
.env | 1 +
app/models.py | 10 ++++++++
app/settings.py | 9 +++++++
app/tasks.py | 49 +++++++++++++++++++++++++++++++++++++
app/utils.py | 17 +++++++++++++
docker-compose.override.yml | 1 +
6 files changed, 87 insertions(+)
diff --git a/.env b/.env
index 29cca1b..cce874f 100644
--- a/.env
+++ b/.env
@@ -3,3 +3,4 @@ POSTGRES_PASSWORD=icspwd
POSTGRES_DB=csentry_db
PGDATA_VOLUME=./data/postgres
ELASTIC_DATA_VOLUME=./data/elastic
+GL_ACCESS_TOKEN=xxxx
diff --git a/app/models.py b/app/models.py
index 4fd0936..df0edd3 100644
--- a/app/models.py
+++ b/app/models.py
@@ -1745,6 +1745,16 @@ def before_flush(session, flush_context, instances):
trigger_inventory_update(session)
+@sa.event.listens_for(Host.is_ioc, "set")
+def receive_host_is_ioc_set(target, value, oldvalue, initiator):
+ """Listen for the 'set' event on Host.is_ioc
+
+ Trigger repository creation in GitLab
+ """
+ if value is True and oldvalue is not True:
+ utils.trigger_ioc_repository_creation(target)
+
+
# call configure_mappers after defining all the models
# required by sqlalchemy_continuum
sa.orm.configure_mappers()
diff --git a/app/settings.py b/app/settings.py
index b0f759b..37e804c 100644
--- a/app/settings.py
+++ b/app/settings.py
@@ -122,6 +122,15 @@ VIOC_MEMORY_CHOICES = [2, 4, 8]
VIOC_DISK_CHOICES = [15, 50, 100, 250]
VIOC_OSVERSION_CHOICES = ["centos7"]
+# IOC repository creation
+IOC_REPOSITORY_CREATION_ENABLED = False
+IOC_REPOSITORY_GROUP_ID = 208
+IOC_REPOSITORY_VISIBILITY = "internal"
+IOC_REPOSITORY_GITLAB_URL = "https://gitlab.esss.lu.se"
+IOC_REPOSITORY_GITLAB_CI_YML = """---
+include: 'https://gitlab.esss.lu.se/ics-infrastructure/gitlab-ci-yml/raw/master/Ioc.gitlab-ci.yml'
+"""
+
# Sentry integration
CSENTRY_RELEASE = raven.fetch_git_sha(Path(__file__).parents[1])
# Leave to empty string to disable sentry integration
diff --git a/app/tasks.py b/app/tasks.py
index 6bef637..e0ad7c3 100644
--- a/app/tasks.py
+++ b/app/tasks.py
@@ -9,8 +9,10 @@ This module implements tasks to run.
:license: BSD 2-Clause, see LICENSE for more details.
"""
+import os
import time
import traceback
+import requests
import tower_cli
from datetime import datetime
from flask import current_app
@@ -226,3 +228,50 @@ def generate_items_excel_file():
pagination = pagination.next()
wb.save(full_path)
return full_path.name
+
+
+def create_ioc_repository(project_name):
+ """Create the IOC repository on GitLab
+
+ Return the response of the POST
+ """
+ api_url = current_app.config["IOC_REPOSITORY_GITLAB_URL"] + "/api/v4"
+ headers = {
+ "user-agent": "csentry",
+ "accept": "application/json",
+ "private-token": os.environ.get("GL_ACCESS_TOKEN"),
+ }
+ # The repository path can't contain "."
+ data = {
+ "namespace_id": current_app.config["IOC_REPOSITORY_GROUP_ID"],
+ "visibility": current_app.config["IOC_REPOSITORY_VISIBILITY"],
+ "name": project_name,
+ "path": project_name.lower().replace(".", "_"),
+ }
+ response = requests.post(api_url + "/projects", headers=headers, json=data)
+ try:
+ payload = response.json()
+ except ValueError:
+ payload = None
+ if response.status_code != 201:
+ current_app.logger.warning(
+ f"{project_name} repository couldn't be created: {payload}"
+ )
+ else:
+ commit_data = {
+ "branch": "master",
+ "commit_message": "Add .gitlab-ci.yml file",
+ "actions": [
+ {
+ "action": "create",
+ "file_path": ".gitlab-ci.yml",
+ "content": current_app.config["IOC_REPOSITORY_GITLAB_CI_YML"],
+ }
+ ],
+ }
+ response = requests.post(
+ api_url + f"/projects/{payload['id']}/repository/commits",
+ headers=headers,
+ json=commit_data,
+ )
+ return response
diff --git a/app/utils.py b/app/utils.py
index 6a20b77..d9005a8 100644
--- a/app/utils.py
+++ b/app/utils.py
@@ -301,6 +301,23 @@ def trigger_inventory_update():
return task
+def trigger_ioc_repository_creation(host):
+ """Trigger a job to create an IOC repository on GitLab"""
+ if not current_app.config["IOC_REPOSITORY_CREATION_ENABLED"]:
+ current_app.logger.info(
+ f"IOC repository creation is disabled. No job triggered for {host.fqdn}."
+ )
+ else:
+ current_app.logger.info(
+ f"Launch new job to create the IOC repository: {host.fqdn}"
+ )
+ kwargs = {"func": "create_ioc_repository", "project_name": host.fqdn}
+ task = current_user.launch_task(
+ "create_ioc_repository", queue_name="low", **kwargs
+ )
+ return task
+
+
def redirect_to_job_status(job_id):
"""
The answer to a client request, leading it to regularly poll a job status.
diff --git a/docker-compose.override.yml b/docker-compose.override.yml
index 0fe282a..79d4364 100644
--- a/docker-compose.override.yml
+++ b/docker-compose.override.yml
@@ -18,6 +18,7 @@ services:
environment:
LOCAL_SETTINGS: /app/settings.cfg
FLASK_APP: /app/wsgi.py
+ GL_ACCESS_TOKEN: ${GL_ACCESS_TOKEN}
volumes:
- .:/app
postgres:
--
GitLab