.gitlab-ci.yml

@@ -108,3 +108,15 @@ deploy-test-02:
   environment:
     name: test-02
     url: https://naming-test-02.cslab.esss.lu.se
+
+deploy-ce-test:
+  extends: .deploy
+  script:
+    - >
+      TOWER_OAUTH_TOKEN=$CE_TOWER_OAUTH_TOKEN awx job_templates launch deploy-ce-naming-test 
+      --extra_vars "naming_container_image_tag: $CI_COMMIT_REF_SLUG"
+      --job_tags ce-naming-backend
+      --monitor
+  environment:
+    name: ce-naming-test
+    url: https://ce-naming-test.cslab.esss.lu.se

CONFIGURATION.md

+### Configuration
+
+Configuration may be set using SpringBoot's configuration file (`application.properties`) and environment variables.
+
+##### Environment variables
+
+| Variable                            | Default                                     | Description                                                                                                            |
+|-------------------------------------|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------|
+| **Application**                     |                                             |                                                                                                                        |
+|                                     |                                             |                                                                                                                        |
+| NAMING_DATABASE_URL                 | jdbc:postgresql://postgres:5432/discs_names | JDBC URL for the database connection                                                                                   |
+| NAMING_DATABASE_USERNAME            | discs_names                                 | user name for the database connection                                                                                  |
+| NAMING_DATABASE_PASSWORD            | discs_names                                 | password for the database connection                                                                                   |
+| NAMING_LOGGING_LEVEL                | INFO                                        | log level for application                                                                                              |
+| NAMING_LOGGING_FILTER_INCLUDES      | org.openepics.names                         | comma-separated list of strings to filter stack trace for inclusion (ref. whitelist)                                   |
+| NAMING_LOGGING_FILTER_EXCLUDES      | org.openepics.names.util.ExceptionUtil      | comma-separated list of strings to filter stack trace for exclusion (ref. blacklist)                                   |
+| NAMING_LOGGING_STACKTRACE_LENGTH    | 20                                          | max number of stack trace elements to log, negative value will log entire stack trace                                  |
+| NAMING_MAIL_ADMINISTRATOR           |                                             | comma-separated list of administrator email addresses                                                                  |
+| NAMING_MAIL_NOTIFICATION            | false                                       | true/false if mail notications are to be sent                                                                          |
+| NAMING_MAIL_NOTIFICATION_NAMES      |                                             | comma-separated list of email addresses to which to send notications about changes for names                           |
+| NAMING_MAIL_NOTIFICATION_STRUCTURES |                                             | comma-separated list of email addresses to which to send notications about changes for structures                      |
+| NAMING_NOTIFY_SCHEDULE              | 0 0 5 * * *                                 | schedule for notification job, cron expression with six or seven fields to describe individual details of the schedule |
+| NAMING_SMTP_HOST                    | mail.esss.lu.se                             | DNS name of server that accepts connections for SMTP                                                                   |
+| NAMING_SMTP_PORT                    | 587                                         | port to use for SMTP, 587 for SMTP Secure                                                                              |
+| NAMING_SMTP_USERNAME                |                                             | username for SMTP server                                                                                               |
+| NAMING_SMTP_PASSWORD                |                                             | password for SMTP server                                                                                               |
+|                                     |                                             |                                                                                                                        |
+| **Springdoc, Swagger**              |                                             |                                                                                                                        |
+|                                     |                                             |                                                                                                                        |
+| NAMING_SWAGGER_URL                  | http://localhost:8080/swagger-ui.html       | default url for Swagger UI                                                                                             |
+| API_DOCS_PATH                       | /api-docs                                   | custom path of the OpenAPI documentation in JSON                                                                       |
+| SWAGGER_UI_PATH                     | /swagger-ui.html                            | custom path of the swagger-ui documentation in HTML                                                                    |
+|                                     |                                             |                                                                                                                        |
+| **Security**                        |                                             |                                                                                                                        |
+|                                     |                                             |                                                                                                                        |
+| NAMING_SECURITY_ENABLED             | false                                       | enable security for application                                                                                        |
+| AES_KEY                             | aes_secret_key                              | secret that is used to hash the tokens in the JWT token                                                                |
+| JWT_SECRET                          | secret_password_to_hash_token               | secret that is used sign the JWT token                                                                                 |
+| JWT_EXP_MIN                         | 240                                         | time interval until the JWT token is valid (in minutes)                                                                |
+| RBAC_SERVER_ADDRESS                 |                                             | RBAC server address used for log in users if NAMING_SECURITY_ENABLED is enabled                                        |

Dockerfile.integrationtest

+# ------------------------------------------------------------------------------
+# Copyright (C) 2021 European Spallation Source ERIC.
+# ------------------------------------------------------------------------------
+
+FROM openjdk:17
+
+# deployment unit
+COPY target/naming-backend-*.jar /naming/naming-backend.jar
+
+# code coverage
+COPY target/jacoco/jacocoagent.jar /naming/jacocoagent.jar
+
+CMD ["java", "-jar", "/naming/naming-backend.jar", "--spring.config.name=application"]

README.md

@@ -6,72 +6,73 @@ The purpose of Naming is to handle Naming of ESS wide physical and logical devic
 
 Naming backend is a web application implemented as a REST style web service backed by a relational database. The web service is implemented as a Spring Boot application and the database is available as PostgreSQL.
 
-Background and motivation, application and database design, and more, is available through content in docs folder.
+See `docs` and sub folders to learn how ESS Naming Convention is implemented in Naming backend.
 
 ### Environment
 
 Tools
-* Java  17   (openjdk)
-* Maven 3.8+ (e.g. 3.8.5)
-* PostgreSQL (9.6 client lib)
-* Docker     (docker, docker-compose)
-* Git
-
-Dependencies
-* postgresql
-* springdoc
-* springframework
-* testcontainers
-
-### Set up development environment and test
 
-#### Checkout and build
+* Java 17 - openjdk
+* Maven 3.8+
+* PostgreSQL - client lib 9.6
+* Docker - engine 18.06.0+ or later, compose 2.21.0 or later, compose file version 3.7 to be supported
+* Git
 
-* ```git clone https://gitlab.esss.lu.se/ics-software/naming-backend.git```
-* ```cd directory```
-* ```mvn clean install```
+More on configuration and setting of environment variables is available in [Configuration](CONFIGURATION.md).
 
-#### Start application (with database)
+### Start application
 
-Either
+Start the application and database via Docker. For first time users, the recommended way is to use demo content. See Note for more on configuration.
 
-* start the application and database
-  * ```docker-compose up --build```
+* build and run
+    * `mvn clean install`
+    * `docker-compose up --build`
+    * `docker-compose -f docker-compose-demo.yml up --build`
+*  browse to
+    * `http://localhost:8080/swagger-ui.html`
 
-or
+More on how to start the application is available in [Start application](START_APPLICATION.md).
 
-* make sure database available as in ```src/main/resources/application.properties```
-* start the application
-  * ```java -jar target/naming-backend-*.jar```
+### Integration tests with Docker containers
 
-Then
+Purpose of integration tests is to ensure REST API works as expected for parts and as a whole by running tests against running instances of application and database.
 
-* browse to ```http://localhost:8080/swagger-ui.html```
+See `src/test/java` and package `org.openepics.names.docker`.
 
-#### Integration tests with Docker containers
+Integration tests start docker containers for Naming backend and Postgresql database and run http requests (GET) and curl commands (POST, PUT, PATCH, DELETE) towards the application to test behavior (CRUD - create, read, update, delete) and replies are received and checked if content is as expected.
 
-See ```src/test/java``` and packages
-* ```org.openepics.names.docker```
-* ```org.openepics.names.docker.complex```
+There are tests for names and structures separately and in combination.
 
-JUnit tests start a docker container for the application (Naming backend) and another docker container for the database (Postgresql) through ```docker-compose-integrationtest.yml```.
+Integration tests can be run in IDE and via Maven.
 
 ```
-    @Container
-    public static final DockerComposeContainer<?> ENVIRONMENT =
-        new DockerComposeContainer<>(new File("docker-compose-integrationtest.yml"))
-            .waitingFor(ITUtil.NAMING, Wait.forLogMessage(".*Started NamingApplication.*", 1));
+mvn failsafe:integration-test -DskipITs=false
 ```
 
-Thereafter a number of http requests (GET) and curl commands (POST, PUT, PATCH, DELETE) are run towards the application to test behavior (CRUD - create, read, update, delete) and replies are received and checked if content is as expected.
+See
+* [How to run Integration test with Docker](src/test/resources/INTEGRATIONTEST_DOCKER_RUN.md)
+* [Tutorial for Integration test with Docker](src/test/resources/INTEGRATIONTEST_DOCKER_TUTORIAL.md)
+
+### Note
 
-#### Note
+* Configuration
+    * Application and database may be started separately, with or without Docker, in which case configuration set by environment variables, docker compose files and `application.properties` must reflect this. An example of this is hostname/ip address/port for database that is to be available for application. Configuration must not be in conflict with environment in which application and database run.
+* Pre-populated database to be available if data to be available at start of application. Otherwise database may be populated through `curl` at command line or Swagger UI.
+    * See `src/main/resources` and `src/test/resources` and sub folders for database scripts. This includes scripts for database schema, migration and example content.
+    * If no prior database is available, see integration tests for examples on how to populate database. E.g. tests may be debugged and database content may be examined at any debug point. It's possible to backup database at such debug point and later restore it outside test. In such way, database may be set to state of choice.
 
-* Pre-populated database to be available if data to be available at start of application. Otherwise database may be populated through ```curl``` at command line or Swagger UI.
-  * See ```src/main/resources``` and sub folders for database scripts.
-  * If no prior database is available, see integration tests for examples on how to populate database. E.g. tests may be debugged and database content may be examined at any debug point. It's possible to backup database at such debug point and later restore it outside test. In such way, database may be set to state of choice.
+ ```
+   postgres:
+     volumes:
+       - ./src/test/resources/data/db/dump-discs_names_namesit.sql:/docker-entrypoint-initdb.d/dump-discs_names_namesit.sql
+ ```
 
-#### Recommentation
+* Test
+    * Integration tests and code coverage for unit and integration tests are disabled by default and can be enabled separately and together.
+    * Code coverage for unit tests is enabled by `-Djacoco.skip=false`. Result is available in `target/site/jacoco` folder and includes code coverage execution data and reports.
 
-* Have local docker-compose file for database alone if there is need to backup or restore database.
+### Recommendation
 
+* Have local docker-compose files - `.gitignore` contains `docker-compose-local*`
+    * for application and database together that may contain local configuration
+    * for database alone if there is need to backup or restore database

START_APPLICATION.md

+### Start application
+
+For first time users, the recommended way to start the application and database is with Docker and demo content.
+
+##### With Docker
+
+To start the application with an existing database.
+
+```
+docker-compose up --build
+```
+
+To start the application with an empty database.
+
+```
+docker-compose -f docker-compose-integrationtest.yml up --build
+```
+
+To start the application with a database with demo content.
+
+```
+docker-compose -f docker-compose-demo.yml up --build
+```
+
+Browse to `http://localhost:8080/swagger-ui.html`.
+
+##### With `java -jar`
+
+Make sure the database is available as in `src/main/resources/application.properties`.
+
+To start the application.
+
+```
+java -jar target/naming-backend-*.jar
+java -jar -Dspring.datasource.url=jdbc:postgresql://localhost:5432/discs_names target/naming-backend-*.jar
+```
+
+Browse to `http://localhost:8080/swagger-ui.html`.

docker-compose-demo.yml

+# ------------------------------------------------------------------------------
+# Copyright (C) 2022 European Spallation Source ERIC.
+# ------------------------------------------------------------------------------
+
+version: '3.7'
+services:
+  naming:
+    container_name: "naming"
+    build: ./
+    networks:
+      - naming-net
+    ports:
+      - "8080:8080"
+    depends_on:
+      postgres: 
+        condition: service_healthy
+    command: "java -jar /naming/naming-backend.jar"
+  
+  postgres:
+    container_name: "postgres"
+    image: "postgres:9.6.7"
+    networks:
+      - naming-net
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_DB: discs_names
+      POSTGRES_USER: discs_names
+      POSTGRES_PASSWORD: discs_names
+      PGDATA: /var/lib/postgresql/data/pgdata
+    healthcheck: 
+      test: ["CMD-SHELL", "pg_isready -U discs_names"]
+      interval: 10s
+      timeout: 5s
+      retries: 10    
+    volumes:
+      - ./src/test/resources/data/db/dump-discs_names_namesit.sql:/docker-entrypoint-initdb.d/dump-discs_names_namesit.sql
+
+volumes:
+  naming-data:
+
+networks:
+  naming-net:
+    driver: bridge
+

docker-compose-integrationtest.yml

@@ -5,16 +5,31 @@
 version: '3.7'
 services:
   naming:
-    build: ./
+    build:
+      context: .
+      dockerfile: Dockerfile.integrationtest
+    networks:
+      - naming-net
     ports:
-      - "8080:8083"
+      - "8080:8080"
+    environment:
+      NAMING_LOGGING_LEVEL: TRACE
+      NAMING_MAIL_ADMINISTRATOR: abcd@efgh.ijkl
+      skipITCoverage: "true"
     depends_on:
       postgres: 
         condition: service_healthy
-    command: "java -jar /naming/naming-backend.jar"
-  
+    command: >
+      /bin/bash -c "
+        if [ ${skipITCoverage} == false ]; then
+          export JAVA_TOOL_OPTIONS=-javaagent:/naming/jacocoagent.jar=destfile=/naming/jacoco.exec,output=file,append=false
+        fi
+        java -jar /naming/naming-backend.jar"
+
   postgres:
     image: "postgres:9.6.7"
+    networks:
+      - naming-net
     ports:
       - "5432:5432"
     environment:
@@ -28,10 +43,23 @@ services:
       timeout: 5s
       retries: 10    
     volumes:
-      - ./src/test/resources/db/schema_migration/V1__Initial.sql:/docker-entrypoint-initdb.d/V1__Initial.sql
-      - ./src/test/resources/db/schema_migration/V2__Commit_Msg_to_Device.sql:/docker-entrypoint-initdb.d/V2__Commit_Msg_to_Device.sql
-      - ./src/test/resources/db/schema_migration/V3__Notification_CC_List.sql:/docker-entrypoint-initdb.d/V3__Notification_CC_List.sql
-      - ./src/test/resources/db/schema_migration/V4__Schema_data_migration.sql:/docker-entrypoint-initdb.d/V4__Schema_data_migration.sql
+      - ./src/main/resources/db/migration/V1__Initial.sql:/docker-entrypoint-initdb.d/V1__Initial.sql
+      - ./src/main/resources/db/migration/V2__Commit_Msg_to_Device.sql:/docker-entrypoint-initdb.d/V2__Commit_Msg_to_Device.sql
+      - ./src/main/resources/db/migration/V3__Notification_CC_List.sql:/docker-entrypoint-initdb.d/V3__Notification_CC_List.sql
+      - ./src/main/resources/db/migration/V4__Schema_data_migration.sql:/docker-entrypoint-initdb.d/V4__Schema_data_migration.sql
+      - ./src/main/resources/db/migration/V5.2__Data_transformation_status_deleted.sql:/docker-entrypoint-initdb.d/V5.2__Data_transformation_status_deleted.sql
+      - ./src/main/resources/db/migration/V6.2__Schema_data_migration_audit.sql:/docker-entrypoint-initdb.d/V6.2__Schema_data_migration_audit.sql
+      - ./src/main/resources/db/migration/V7__Remove_user_notification_cc_list.sql:/docker-entrypoint-initdb.d/V7__Remove_user_notification_cc_list.sql
 
 volumes:
   naming-data:
+
+networks:
+  naming-net:
+    driver: bridge
+
+
+# ------------------------------------------------------------------------------
+# Note
+# ------------------------------------------------------------------------------
+# container_name property is not supported by Testcontainers

docker-compose.yml

@@ -5,9 +5,12 @@
 version: '3.7'
 services:
   naming:
+    container_name: "naming"
     build: ./
+    networks:
+      - naming-net
     ports:
-      - "8080:8083"
+      - "8080:8080"
     depends_on:
       - postgres
     command: "java -jar /naming/naming-backend.jar"
@@ -15,6 +18,8 @@ services:
   postgres:
     container_name: "postgres"
     image: "postgres:9.6.7"
+    networks:
+      - naming-net
     ports:
       - "5432:5432"
     environment:
@@ -27,3 +32,7 @@ services:
 
 volumes:
   naming-data:
+
+networks:
+  naming-net:
+    driver: bridge

docs/developer/naming-convention-tool/developer/application/README.txt

-Developer documentation
-
-    Purpose to help understand application, database and in between.
-
-    Created on Ubuntu 18.04 LTS with LibreOffice (6.0.7.3)(Draw, Writer), GNOME Screenshot,  Krita (4.3.0).
-    Document ".odt" exported to ".pdf".