Merge branch 'packaging-configuration-refactoring' into 'development'

Packaging configuration refactoring

See merge request !6

.gitlab-ci.yml

@@ -7,10 +7,10 @@ stages:
 image: maven:3.6.3-openjdk-11
 
 variables:
-  MAVEN_CLI_OPTS: "--batch-mode"
   MAVEN_OPTS: "-Dmaven.repo.local=.m2/repository"
   MAVEN_CLI_OPTS_DATA_COLLECTION_POM_FILE: "--file data-collection/pom.xml"
   MAVEN_CLI_OPTS_WEBAPP_POM_FILE: "--file webapp/pom.xml"
+  # Maven command line options are defined in `.mvn/maven.config` so the same config is used when running from the command line.
 
 cache:
   paths:
@@ -19,40 +19,69 @@ cache:
 build-data-collection:
   stage: build
   script:
-    - mvn $MAVEN_CLI_OPTS_DATA_COLLECTION_POM_FILE $MAVEN_CLI_OPTS compile
+    - mvn $MAVEN_CLI_OPTS_DATA_COLLECTION_POM_FILE compile
 
 build-webapp:
   stage: build
   script:
-    - mvn $MAVEN_CLI_OPTS_WEBAPP_POM_FILE $MAVEN_CLI_OPTS compile
+    - mvn $MAVEN_CLI_OPTS_WEBAPP_POM_FILE compile
 
 test-data-collection:
   stage: test
   script:
-    - mvn $MAVEN_CLI_OPTS_DATA_COLLECTION_POM_FILE $MAVEN_CLI_OPTS test
+    - mvn $MAVEN_CLI_OPTS_DATA_COLLECTION_POM_FILE test
 
 test-webapp:
   stage: test
   script:
-    - mvn $MAVEN_CLI_OPTS_WEBAPP_POM_FILE $MAVEN_CLI_OPTS test
+    - mvn $MAVEN_CLI_OPTS_WEBAPP_POM_FILE test
 
-jar-data-collection:
+.jar-data-collection-common:
   stage: package
   script:
-    - mvn $MAVEN_CLI_OPTS_DATA_COLLECTION_POM_FILE $MAVEN_CLI_OPTS package
+    - mvn $MAVEN_CLI_OPTS_DATA_COLLECTION_POM_FILE package
   artifacts:
     paths:
       - data-collection/target/*.jar
+      - data-collection/target/lib/*.jar
 
-war-webapp:
+jar-data-collection:prod:
+  extends: .jar-data-collection-common
+  artifacts:
+    expire_in: 10 years
+  only:
+    - master
+
+jar-data-collection:dev:
+  extends: .jar-data-collection-common
+  artifacts:
+    expire_in: 30 days
+  except:
+    - master
+
+.war-webapp-common:
   stage: package
   script:
-    - mvn $MAVEN_CLI_OPTS_WEBAPP_POM_FILE $MAVEN_CLI_OPTS package
+    - mvn $MAVEN_CLI_OPTS_WEBAPP_POM_FILE package
   artifacts:
     paths:
       - webapp/target/*.war
 
+war-webapp:prod:
+  extends: .war-webapp-common
+  artifacts:
+    expire_in: 10 years
+  only:
+    - master
+
+war-webapp:dev:
+  extends: .war-webapp-common
+  artifacts:
+    expire_in: 30 days
+  except:
+    - master
+
 deploy-data-collection:
   stage: deploy
   script:
-    - echo "TODO implement automated deployment for data-collection"
+    - echo "TODO implement automated deployment"

.mvn/maven.config

+--batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true
\ No newline at end of file

.mvn/wrapper/MavenWrapperDownloader.java

+/*
+ * Copyright 2007-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.net.*;
+import java.io.*;
+import java.nio.channels.*;
+import java.util.Properties;
+
+public class MavenWrapperDownloader {
+
+    private static final String WRAPPER_VERSION = "0.5.6";
+    /**
+     * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
+     */
+    private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+        + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
+
+    /**
+     * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
+     * use instead of the default one.
+     */
+    private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
+            ".mvn/wrapper/maven-wrapper.properties";
+
+    /**
+     * Path where the maven-wrapper.jar will be saved to.
+     */
+    private static final String MAVEN_WRAPPER_JAR_PATH =
+            ".mvn/wrapper/maven-wrapper.jar";
+
+    /**
+     * Name of the property which should be used to override the default download url for the wrapper.
+     */
+    private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
+
+    public static void main(String args[]) {
+        System.out.println("- Downloader started");
+        File baseDirectory = new File(args[0]);
+        System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
+
+        // If the maven-wrapper.properties exists, read it and check if it contains a custom
+        // wrapperUrl parameter.
+        File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
+        String url = DEFAULT_DOWNLOAD_URL;
+        if(mavenWrapperPropertyFile.exists()) {
+            FileInputStream mavenWrapperPropertyFileInputStream = null;
+            try {
+                mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
+                Properties mavenWrapperProperties = new Properties();
+                mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
+                url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
+            } catch (IOException e) {
+                System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
+            } finally {
+                try {
+                    if(mavenWrapperPropertyFileInputStream != null) {
+                        mavenWrapperPropertyFileInputStream.close();
+                    }
+                } catch (IOException e) {
+                    // Ignore ...
+                }
+            }
+        }
+        System.out.println("- Downloading from: " + url);
+
+        File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
+        if(!outputFile.getParentFile().exists()) {
+            if(!outputFile.getParentFile().mkdirs()) {
+                System.out.println(
+                        "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
+            }
+        }
+        System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
+        try {
+            downloadFileFromURL(url, outputFile);
+            System.out.println("Done");
+            System.exit(0);
+        } catch (Throwable e) {
+            System.out.println("- Error downloading");
+            e.printStackTrace();
+            System.exit(1);
+        }
+    }
+
+    private static void downloadFileFromURL(String urlString, File destination) throws Exception {
+        if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
+            String username = System.getenv("MVNW_USERNAME");
+            char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
+            Authenticator.setDefault(new Authenticator() {
+                @Override
+                protected PasswordAuthentication getPasswordAuthentication() {
+                    return new PasswordAuthentication(username, password);
+                }
+            });
+        }
+        URL website = new URL(urlString);
+        ReadableByteChannel rbc;
+        rbc = Channels.newChannel(website.openStream());
+        FileOutputStream fos = new FileOutputStream(destination);
+        fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+        fos.close();
+        rbc.close();
+    }
+
+}

.mvn/wrapper/maven-wrapper.properties

+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

ci_settings.xml

+<settings xmlns="http://maven.apache.org/SETTINGS/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0 http://maven.apache.org/xsd/settings-1.1.0.xsd">
+  <servers>
+    <server>
+      <id>gitlab-maven</id>
+      <configuration>
+        <httpHeaders>
+          <property>
+            <name>Job-Token</name>
+            <value>${env.CI_JOB_TOKEN}</value>
+          </property>
+        </httpHeaders>
+      </configuration>
+    </server>
+  </servers>
+</settings>
\ No newline at end of file

webapp/collectdata.sh → data-collection/bin/collectdata.sh

 #!/bin/bash
 
-MRL_ROOT=..
+WORKDIR=$(pwd)
+CONFIG_FILENAME='collection.conf'
+CONFIG_FULLPATH="${WORKDIR}/${CONFIG_FILENAME}"
+
+if [ -e "${CONFIG_FULLPATH}" ]; then
+  source ${CONFIG_FULLPATH}
+else
+  echo "Config file not found (${CONFIG_FULLPATH}). Exiting."
+  exit 1
+fi
+
+# extract MySQL credentials from webapp config
 
-MYSQL_USER=root
-MYSQL_PASSWORD=password
-MYSQL_DB=OW2_MRL
+# Extract collector's output folder from its configuration file (.cfg)
 
 # Generate script to fetch data
 # (according to data sources configuration file)
 java -jar $MRL_ROOT/data-collection/target/ow2-mrl-data-0.0.4-SNAPSHOT.jar $MRL_ROOT/data-collection/src/main/resources/ow2_projects.cfg 1
 
-# Fetch data (call fetch_statistics.sh in path specified by configuration file)
+# Fetch data (call fetch_statistics.sh from path specified by configuration file)
 /tmp/MRL/fetch_statistics.sh
 
 # Parse data and generate CSV with metrics
@@ -27,4 +36,3 @@ then
   mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB -Bse "CREATE TABLE SpareMetric as select * from Metric;"
   mysql -u $MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB -Bse "LOAD DATA LOCAL INFILE '/tmp/MRL/MRL_thresholds.csv' REPLACE INTO TABLE Metric FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' (MetricName,Threshold1,Threshold2,Threshold3,Threshold4,Threshold5,Reverse);"
 fi
-

data-collection/conf/collection.conf.dist

+# This is a sample configuration file for collectdata.sh
+# It should be adapted to the environment.
+
+MRL_ROOT=</path/to/runtime/file.jar>
+MYSQL_CONFIG=</path/to/webapp/mysql.properties>

data-collection/conf/credentials.file.dist

+# This is an example of credentials definition
+# Syntax is: key: user:password
+
+# E.g. token for FusionDirectory SonarQube
+fusiondirectory.sonarqube.credentials: <SonarQube token 2f6e985a0bad785:>
\ No newline at end of file

data-collection/src/main/resources/ow2_projects.cfg → data-collection/conf/ow2_projects.cfg.dist

 # MRL metadata endpoints
-MRL.thresholds: https://projects.dev.ow2.org/get/ow2/Project_metrics_thresholds
+# e.g. https://projects.ow2.org/get/ow2/Project_metrics_thresholds
+MRL.thresholds: <MRL metadata API endpoints>
+
+# Output directory for data collection (generated scripts and collected data)
+# e.g. /tmp/MRL
+outputDir: </MRL/data/collection/directory/full/path>
+
+# Include secret credentials if any
+# e.g. /home/user/credentials.file
+include: </MRL/data/collection/credentials/configuration/file/full/path>
 
 # Scanned projects list
 projects: asm, authzforce, clif, docdoku, fractal, fusiondirectory, glpi, imixs-workflow, joram, knowage, lemonldap-ng, lutece, ocsinventory, proactive, rocket-chat, sat4j, seedstack, sympa, spoon, telosys, waarp, weblab, xwiki, zenroom
 
-# Output directory for data collection (script and data)
-outputDir: /tmp/MRL
-
 # Data access configuration, for each listed project
 
-# Include secret credentials if any (not committed into repo)
-# eg. omm.credentials: user:password
-include: ../credentials.secret
-
 # ASM (complete)
 asm.repo: https://gitlab.ow2.org/asm/asm
 asm.issues.endpoint: https://gitlab.ow2.org/api/v4/projects

data-collection/pom.xml

-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <artifactId>ow2-mrl-data</artifactId>
     <version>0.0.6-SNAPSHOT</version>
     <packaging>jar</packaging>
-    <name>MRL Data gathering</name>
-    <description>MRL Data gathering</description>
+    <name>MRL data gathering</name>
+    <description>MRL data gathering</description>
 
     <parent>
         <groupId>org.ow2.mrl</groupId>

data-collection/src/main/resources/logback.xml

 <configuration>
-
   <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
     <!-- encoders are assigned the type
          ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->