Merge branch 'ems/prepare-for-nebulous'

.gitignore

@@ -0,0 +1,15 @@
+target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr

README.md

@@ -0,0 +1 @@
+repo init

ems-core/.gitattributes

@@ -0,0 +1,9 @@
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
+# If a copy of the MPL was not distributed with this file, You can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+*.sh text eol=lf

ems-core/.gitignore

@@ -0,0 +1,10 @@
+.idea
+broker-client/*.js
+config-files/*.p12
+config-files/*.crt
+public_resources/**
+tcnative*
+ems.log
+server.pem
+.dev-*
+.flattened-pom.xml

ems-core/baguette-client-install/pom.xml

@@ -0,0 +1,66 @@
+<!--
+  ~ Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+  ~
+  ~ This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+  ~ Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+  ~ If a copy of the MPL was not distributed with this file, you can obtain one at
+  ~ https://www.mozilla.org/en-US/MPL/2.0/
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>gr.iccs.imu.ems</groupId>
+        <artifactId>ems-core</artifactId>
+        <version>${revision}</version>
+    </parent>
+
+    <artifactId>baguette-client-install</artifactId>
+    <name>EMS - Baguette Client install utilities</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>gr.iccs.imu.ems</groupId>
+            <artifactId>baguette-server</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.projectlombok/lombok -->
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.rauschig/jarchivelib -->
+        <dependency>
+            <groupId>org.rauschig</groupId>
+            <artifactId>jarchivelib</artifactId>
+            <version>1.2.0</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml -->
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.yaml/snakeyaml -->
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationProperties.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install;
+
+import gr.iccs.imu.ems.util.EmsConstant;
+import lombok.Data;
+import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+@Slf4j
+@Data
+@Configuration
+@ConfigurationProperties(prefix = EmsConstant.EMS_PROPERTIES_PREFIX + "baguette.client.install")
+public class ClientInstallationProperties implements InitializingBean {
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        log.debug("ClientInstallationProperties: {}", this);
+    }
+
+    enum INSTALLER_TYPE { DEFAULT_INSTALLER, JS_INSTALLER }
+
+    private final Map<String, List<String>> osFamilies = new LinkedHashMap<>();
+
+    private int workers = 1;
+    private INSTALLER_TYPE installerType = INSTALLER_TYPE.DEFAULT_INSTALLER;
+
+    private String baseDir;                     // EMS client home directory
+    private String rootCmd;                     // Root command (e.g. 'sudo', or 'echo ${NODE_SSH_PASSWORD} | sudo -S ')
+    private List<String> mkdirs;
+    private List<String> touchFiles;
+    private String checkInstalledFile;
+
+    private String downloadUrl;                 // Base URL of EMS server downloads
+    @ToString.Exclude
+    private String apiKey;                      // API Key for accessing EMS server downloads
+    private String installScriptUrl;
+    private String installScriptFile;
+
+    private String archiveSourceDir;            // the directory in server that will be archived (it must contain client configuration)
+    private String archiveDir;                  // the directory in server where client config. archive will be placed into
+    private String archiveFile;                 // name of the client configuration archive (in server)
+    private String clientConfArchiveFile;       // location in VM, where client config. archive will be stored (in BASE64 encoding)
+    //private String clientConfArchiveDest;       // location in VM, where client config. archive will be extracted
+
+    private String serverCertFileAtServer;      // location of EMS server certificate in server (in config-files)
+    private String serverCertFileAtClient;      // location in VM, where EMS server certificate will be stored
+    private String copyFilesFromServerDir;      // location in EMS server whose contents will be copied to VM
+    private String copyFilesToClientDir;        // location in VM where server files will be copied into
+
+    private String clientTmpDir;                // location of temp. directory in VM (typically /tmp)
+    private String serverTmpDir;                // location of temp. directory in EMS server
+    private boolean keepTempFiles;              // keep temporary files in EMS server (during debug)
+
+    // ----------------------------------------------------
+
+    private boolean simulateConnection;
+    private boolean simulateExecution;
+
+    private int maxRetries = 5;
+    private long retryDelay = 1000L;
+    private double retryBackoffFactor = 1.0;
+
+    private long connectTimeout = 60000;
+    private long authenticateTimeout = 60000;
+    private long heartbeatInterval = 60000;
+    private long heartbeatReplyWait = heartbeatInterval;
+    private long commandExecutionTimeout = 60000;
+
+    private final Map<String, List<String>> instructions = new LinkedHashMap<>();
+    private final Map<String, String> parameters = new LinkedHashMap<>();
+
+    private boolean continueOnFail = false;
+    private String sessionRecordingDir = "logs";
+
+    // ----------------------------------------------------
+
+    private String clientInstallVarName = "__EMS_CLIENT_INSTALL__";
+    private Pattern clientInstallSuccessPattern = Pattern.compile("^INSTALLED($|[\\s:=])", Pattern.CASE_INSENSITIVE);
+    private Pattern clientInstallErrorPattern = Pattern.compile("^ERROR($|[\\s:=])", Pattern.CASE_INSENSITIVE);
+    private boolean clientInstallSuccessIfVarIsMissing = false;
+    private boolean clientInstallErrorIfVarIsMissing = true;
+
+    private String skipInstallVarName = "__EMS_CLIENT_INSTALL__";
+    private Pattern skipInstallPattern = Pattern.compile("^SKIPPED($|[\\s:=])", Pattern.CASE_INSENSITIVE);
+    private boolean skipInstallIfVarIsMissing = false;
+
+    private String ignoreNodeVarName = "__EMS_IGNORE_NODE__";
+    private Pattern ignoreNodePattern = Pattern.compile("^IGNORED($|[\\s:=])", Pattern.CASE_INSENSITIVE);
+    private boolean ignoreNodeIfVarIsMissing = false;
+
+    // ----------------------------------------------------
+
+    private List<Class<InstallationContextProcessorPlugin>> installationContextProcessorPlugins = Collections.emptyList();
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallationTask.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install;
+
+import gr.iccs.imu.ems.baguette.client.install.instruction.InstructionsSet;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.translate.TranslationContext;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * Client installation task
+ */
+@Data
+@Builder
+public class ClientInstallationTask {
+    private final String id;
+    private final String nodeId;
+    private final String name;
+    private final String os;
+    private final String address;
+    private final String type;
+    private final String provider;
+    private final SshConfig ssh;
+    private final NodeRegistryEntry nodeRegistryEntry;
+    private final List<InstructionsSet> instructionSets;
+    private final TranslationContext translationContext;
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstaller.java

@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install;
+
+import gr.iccs.imu.ems.baguette.server.BaguetteServer;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.common.plugin.PluginManager;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import javax.validation.constraints.NotNull;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Client installer
+ */
+@Slf4j
+@Service
+@NoArgsConstructor
+public class ClientInstaller implements InitializingBean {
+    private static ClientInstaller singleton;
+
+    @Autowired
+    private ClientInstallationProperties properties;
+    @Autowired
+    private BaguetteServer baguetteServer;
+    @Autowired
+    private PluginManager pluginManager;
+
+    private final AtomicLong taskCounter = new AtomicLong();
+    private ExecutorService executorService;
+
+    @Override
+    public void afterPropertiesSet() {
+        singleton = this;
+        executorService = Executors.newFixedThreadPool(properties.getWorkers());
+        properties.getInstallationContextProcessorPlugins().forEach(pluginClass -> {
+            log.debug("ClientInstaller: Initializing plugin: {}", pluginClass);
+            pluginManager.initializePlugin(pluginClass);
+        });
+    }
+
+    public static ClientInstaller instance() { return singleton; }
+
+    public void addTask(@NotNull ClientInstallationTask task) {
+        executorService.submit(() -> {
+            long taskCnt = taskCounter.getAndIncrement();
+            try {
+                log.info("ClientInstaller: Executing Client installation Task #{}: task-id={}, node-id={}, name={}, type={}, address={}",
+                        taskCnt, task.getId(), task.getNodeId(), task.getName(), task.getType(), task.getAddress());
+                long startTm = System.currentTimeMillis();
+                boolean result = executeTask(task, taskCnt);
+                long endTm = System.currentTimeMillis();
+                log.info("ClientInstaller: Client installation Task #{}: result={}, duration={}ms",
+                        taskCnt, result ? "SUCCESS" : "FAILED", endTm - startTm);
+            } catch (Throwable t) {
+                log.info("ClientInstaller: Exception caught in Client installation Task #{}: Exception: ", taskCnt, t);
+            }
+        });
+    }
+
+    private boolean executeTask(ClientInstallationTask task, long taskCounter) {
+        if (baguetteServer.getNodeRegistry().getCoordinator()==null)
+            throw new IllegalStateException("Baguette Server Coordinator has not yet been initialized");
+
+        if ("VM".equalsIgnoreCase(task.getType()) || "baremetal".equalsIgnoreCase(task.getType())) {
+            NodeRegistryEntry entry = baguetteServer.getNodeRegistry().getNodeByAddress(task.getAddress());
+            if (entry==null)
+                throw new IllegalStateException("Node entry has been removed from Node Registry before installation: Node IP address: "+task.getAddress());
+                //baguetteServer.handleNodeSituation(task.getAddress(), INTERNAL_ERROR);
+            entry.nodeInstalling(task);
+
+            // Call InstallationContextPlugin's before installation
+            log.debug("ClientInstaller: PRE-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
+            pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
+                    .forEach(plugin->((InstallationContextProcessorPlugin)plugin).processBeforeInstallation(task, taskCounter));
+
+            log.debug("ClientInstaller: INSTALLATION: Executing installation task: task-counter={}, task={}", taskCounter, task);
+            boolean success = executeVmTask(task, taskCounter);
+            log.debug("ClientInstaller: NODE_REGISTRY_ENTRY after installation execution: \n{}", task.getNodeRegistryEntry());
+
+            if (entry.getState()==NodeRegistryEntry.STATE.INSTALLING) {
+                log.warn("ClientInstaller: NODE_REGISTRY_ENTRY status is still INSTALLING after executing client installation. Changing to INSTALL_ERROR");
+                entry.nodeInstallationError(null);
+            }
+
+            // Call InstallationContextPlugin's after installation
+            log.debug("ClientInstaller: POST-INSTALLATION: Calling installation context processors: {}", properties.getInstallationContextProcessorPlugins());
+            pluginManager.getActivePlugins(InstallationContextProcessorPlugin.class)
+                    .forEach(plugin->((InstallationContextProcessorPlugin)plugin).processAfterInstallation(task, taskCounter, success));
+
+            // Pre-register Node to baguette Server Coordinator
+            log.debug("ClientInstaller: POST-INSTALLATION: Node is being pre-registered: {}", entry);
+            baguetteServer.getNodeRegistry().getCoordinator().preregister(entry);
+
+            log.debug("ClientInstaller: Installation outcome: {}", success ? "Success" : "Error");
+            return success;
+        } else {
+            log.error("ClientInstaller: UNSUPPORTED TASK TYPE: {}", task.getType());
+        }
+        return false;
+    }
+
+    private boolean executeVmTask(ClientInstallationTask task, long taskCounter) {
+        // Select the appropriate client installer plugin to run installation task.
+        // Currently, two installer plugins are available: SshClientInstaller, and SshJsClientInstaller
+        boolean result;
+        if (properties.getInstallerType()==ClientInstallationProperties.INSTALLER_TYPE.JS_INSTALLER) {
+            log.info("ClientInstaller: Using SshJsClientInstaller for task #{}", taskCounter);
+            result = SshJsClientInstaller.jsBuilder()
+                    .task(task)
+                    .taskCounter(taskCounter)
+                    .properties(properties)
+                    .build()
+                    .execute();
+        } else {
+            log.info("ClientInstaller: Using SshClientInstaller (default) for task #{}", taskCounter);
+            result = SshClientInstaller.builder()
+                    .task(task)
+                    .taskCounter(taskCounter)
+                    /*.maxRetries(properties.getMaxRetries())
+                    .authenticationTimeout(properties.getAuthenticateTimeout())
+                    .connectTimeout(properties.getConnectTimeout())
+                    .heartbeatInterval(properties.getHeartbeatInterval())
+                    .simulateConnection(properties.isSimulateConnection())
+                    .simulateExecution(properties.isSimulateExecution())
+                    .commandExecutionTimeout(properties.getCommandExecutionTimeout())*/
+                    .properties(properties)
+                    .build()
+                    .execute();
+        }
+        log.info("ClientInstaller: Task execution result #{}: success={}", taskCounter, result);
+        return result;
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/ClientInstallerPlugin.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install;
+
+public interface ClientInstallerPlugin {
+    default boolean execute() {
+        preProcessTask();
+        boolean result = executeTask();
+        result = result && postProcessTask();
+        return result;
+    }
+
+    void preProcessTask();  // Throw exception to block task execution
+    boolean executeTask();
+    boolean postProcessTask();
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/InstallationContextProcessorPlugin.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install;
+
+import gr.iccs.imu.ems.util.Plugin;
+
+public interface InstallationContextProcessorPlugin extends Plugin {
+    void processBeforeInstallation(ClientInstallationTask task, long taskCounter);
+    void processAfterInstallation(ClientInstallationTask task, long taskCounter, boolean success);
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/SshClientInstaller.java

@@ -0,0 +1,963 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install;
+
+import gr.iccs.imu.ems.baguette.client.install.instruction.INSTRUCTION_RESULT;
+import gr.iccs.imu.ems.baguette.client.install.instruction.Instruction;
+import gr.iccs.imu.ems.baguette.client.install.instruction.InstructionsService;
+import gr.iccs.imu.ems.baguette.client.install.instruction.InstructionsSet;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringSubstitutor;
+import org.apache.sshd.client.SshClient;
+import org.apache.sshd.client.channel.ChannelExec;
+import org.apache.sshd.client.channel.ChannelSession;
+import org.apache.sshd.client.channel.ClientChannelEvent;
+import org.apache.sshd.client.config.hosts.HostConfigEntryResolver;
+import org.apache.sshd.client.keyverifier.AcceptAllServerKeyVerifier;
+import org.apache.sshd.client.session.ClientSession;
+import org.apache.sshd.common.PropertyResolverUtils;
+import org.apache.sshd.core.CoreModuleProperties;
+import org.apache.sshd.mina.MinaServiceFactoryFactory;
+import org.apache.sshd.scp.client.DefaultScpClientCreator;
+import org.apache.sshd.scp.client.ScpClient;
+import org.apache.sshd.scp.client.ScpClientCreator;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemReader;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.*;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * SSH client installer
+ */
+@Slf4j
+@Getter
+public class SshClientInstaller implements ClientInstallerPlugin {
+    private final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss.SSS");
+
+    private final ClientInstallationTask task;
+    private final long taskCounter;
+
+    private final int maxRetries;
+    private final long retryDelay;
+    private final double retryBackoffFactor;
+    private final long connectTimeout;
+    private final long authenticationTimeout;
+    private final long heartbeatInterval;
+    private final long heartbeatReplyWait;
+    private final boolean simulateConnection;
+    private final boolean simulateExecution;
+    private final long commandExecutionTimeout;
+    private final boolean continueOnFail;
+
+    private final ClientInstallationProperties properties;
+
+    private SshClient sshClient;
+    //private SimpleClient simpleClient;
+    private ClientSession session;
+    //private ChannelShell shellChannel;
+    private StreamLogger streamLogger;
+
+    @Builder
+    public SshClientInstaller(ClientInstallationTask task, long taskCounter, ClientInstallationProperties properties) {
+        this.task = task;
+        this.taskCounter = taskCounter;
+
+        this.maxRetries = properties.getMaxRetries()>=0 ? properties.getMaxRetries() : 5;
+        this.retryDelay = properties.getRetryDelay()>0 ? properties.getRetryDelay() : 1000L;
+        this.retryBackoffFactor = properties.getRetryBackoffFactor()>0 ? properties.getRetryBackoffFactor() : 1.0;
+
+        this.connectTimeout = properties.getConnectTimeout()>0 ? properties.getConnectTimeout() : 60000;
+        this.authenticationTimeout = properties.getAuthenticateTimeout()>0 ? properties.getAuthenticateTimeout() : 60000;
+        this.heartbeatInterval = properties.getHeartbeatInterval()>0 ? properties.getHeartbeatInterval() : 10000;
+        this.heartbeatReplyWait = properties.getHeartbeatReplyWait()>0 ? properties.getHeartbeatReplyWait() : 10 * heartbeatInterval;
+        this.simulateConnection = properties.isSimulateConnection();
+        this.simulateExecution = properties.isSimulateExecution();
+        this.commandExecutionTimeout = properties.getCommandExecutionTimeout()>0 ? properties.getCommandExecutionTimeout() : 120000;
+        this.continueOnFail = properties.isContinueOnFail();
+
+        this.properties = properties;
+    }
+
+    @Override
+    public boolean executeTask(/*int retries*/) {
+        if (! openSshConnection())
+            return false;
+
+        boolean success;
+        try {
+            INSTRUCTION_RESULT exitResult = executeInstructionSets();
+            success = exitResult != INSTRUCTION_RESULT.FAIL;
+        } catch (Exception ex) {
+            log.error("SshClientInstaller: Failed executing installation instructions for task #{}, Exception: ", taskCounter, ex);
+            success = false;
+        }
+
+        if (success) log.info("SshClientInstaller: Task completed successfully #{}", taskCounter);
+        else log.info("SshClientInstaller: Error occurred while executing task #{}", taskCounter);
+        return closeSshConnection(success);
+    }
+
+    protected boolean openSshConnection() {
+        task.getNodeRegistryEntry().nodeInstalling(task.getNodeRegistryEntry().getPreregistration());
+        boolean success = false;
+        int retries = 0;
+        while (!success && retries<=maxRetries) {
+            if (retries>0) log.warn("SshClientInstaller: Retry {}/{} executing task #{}", retries, maxRetries, taskCounter);
+            try {
+                sshConnect();
+                //sshOpenShell();
+                success = true;
+            } catch (Exception ex) {
+                log.error("SshClientInstaller: Failed executing task #{}, Exception: ", taskCounter, ex);
+                retries++;
+                if (retries<=maxRetries)
+                    waitToRetry(retries);
+            }
+        }
+        if (!success) {
+            log.error("SshClientInstaller: Giving up executing task #{} after {} retries", taskCounter, maxRetries);
+            return false;
+        }
+        return true;
+    }
+
+    protected boolean closeSshConnection(boolean success) {
+        try {
+            //sshCloseShell();
+            sshDisconnect();
+            return success;
+        } catch (Exception ex) {
+            log.error("SshClientInstaller: Exception while disconnecting. Task #{}, Exception: ", taskCounter, ex);
+            return false;
+        }
+    }
+
+    private void waitToRetry(int retries) {
+        long delay = Math.max(1, (long)(retryDelay * Math.pow(retryBackoffFactor, retries-1)));
+        try {
+            log.debug("SshClientInstaller: waitToRetry: Waiting for {}ms to retry", delay);
+            Thread.sleep(delay);
+        } catch (InterruptedException e) {
+            log.warn("SshClientInstaller: waitToRetry: Waiting to retry interrupted: ", e);
+        }
+    }
+
+    private boolean sshConnect() throws Exception {
+        SshConfig config = task.getSsh();
+        String host = config.getHost();
+        int port = config.getPort();
+
+        if (simulateConnection) {
+            log.info("SshClientInstaller: Simulate connection to remote host: task #{}: host: {}:{}", taskCounter, host, port);
+            return true;
+        }
+
+        // Get connection information
+        String privateKey = config.getPrivateKey();
+        String fingerprint = config.getFingerprint();
+        String username = config.getUsername();
+        String password = config.getPassword();
+
+        // Create and configure SSH client
+        this.sshClient = SshClient.setUpDefaultClient();
+        sshClient.setHostConfigEntryResolver(HostConfigEntryResolver.EMPTY);
+        sshClient.setServerKeyVerifier(AcceptAllServerKeyVerifier.INSTANCE);
+
+        //this.simpleClient = SshClient.wrapAsSimpleClient(sshClient);
+        //simpleClient.setConnectTimeout(connectTimeout);
+        //simpleClient.setAuthenticationTimeout(authenticationTimeout);
+
+        // Set a huge idle timeout, keep-alive to true and heartbeat to configured value
+        PropertyResolverUtils.updateProperty(sshClient, CoreModuleProperties.HEARTBEAT_INTERVAL.getName(), heartbeatInterval);      // Prevents server-side connection closing
+        PropertyResolverUtils.updateProperty(sshClient, CoreModuleProperties.HEARTBEAT_REPLY_WAIT.getName(), heartbeatReplyWait);   // Prevents client-side connection closing
+        PropertyResolverUtils.updateProperty(sshClient, CoreModuleProperties.IDLE_TIMEOUT.getName(), Integer.MAX_VALUE);
+        PropertyResolverUtils.updateProperty(sshClient, CoreModuleProperties.SOCKET_KEEPALIVE.getName(), true);               // Socket keep-alive at OS-level
+        log.debug("SshClientInstaller: Set IDLE_TIMEOUT to MAX, SOCKET-KEEP-ALIVE to true, and HEARTBEAT to {}", heartbeatInterval);
+
+        // Explicitly set IO service factory factory to prevent conflict between MINA and Netty options
+        sshClient.setIoServiceFactoryFactory(new MinaServiceFactoryFactory());
+
+        // Start client and connect to SSH server
+        try {
+            sshClient.start();
+            this.session = sshClient.connect(username, host, port)
+                    .verify(connectTimeout)
+                    .getSession();
+            if (StringUtils.isNotBlank(privateKey)) {
+                PrivateKey privKey = getPrivateKey(privateKey);
+                //PublicKey pubKey = getPublicKey(publicKeyStr);
+                PublicKey pubKey = getPublicKey(privKey);
+                KeyPair keyPair = new KeyPair(pubKey, privKey);
+                session.addPublicKeyIdentity(keyPair);
+            }
+            if (StringUtils.isNotBlank(password)) {
+                session.addPasswordIdentity(password);
+            }
+            session.auth()
+                    .verify(authenticationTimeout);
+
+            // Initialize standard streams' logger
+            initStreamLogger();
+
+            log.info("SshClientInstaller: Connected to remote host: task #{}: host: {}:{}", taskCounter, host, port);
+            return true;
+
+        } catch (Exception ex) {
+            log.error("SshClientInstaller: Error while connecting to remote host: task #{}: ", taskCounter, ex);
+            throw ex;
+        }
+    }
+
+    private PrivateKey getPrivateKey(String pemStr) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+        KeyFactory factory = KeyFactory.getInstance("RSA");
+        try (StringReader keyReader = new StringReader(pemStr); PemReader pemReader = new PemReader(keyReader)) {
+            PemObject pemObject = pemReader.readPemObject();
+            byte[] content = pemObject.getContent();
+            PKCS8EncodedKeySpec keySpecPKCS8 = new PKCS8EncodedKeySpec(content);
+            PrivateKey privKey = factory.generatePrivate(keySpecPKCS8);
+            return privKey;
+        }
+        //PKCS8EncodedKeySpec keySpecPKCS8 = new PKCS8EncodedKeySpec(Base64.decode(privateKeyContent.replaceAll("\\n", "").replace("-----BEGIN PRIVATE KEY-----", "").replace("-----END PRIVATE KEY-----", "")));
+        //PrivateKey privKey = kf.generatePrivate(keySpecPKCS8);
+    }
+
+    private PublicKey getPublicKey(PrivateKey privateKey) throws NoSuchAlgorithmException, InvalidKeySpecException {
+        KeyFactory factory = KeyFactory.getInstance(privateKey.getAlgorithm());
+        PKCS8EncodedKeySpec pubKeySpec = new PKCS8EncodedKeySpec(privateKey.getEncoded());
+        PublicKey publicKey = factory.generatePublic(pubKeySpec);
+        return publicKey;
+    }
+
+    /*private PublicKey getPublicKey(String pemStr) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+        KeyFactory factory = KeyFactory.getInstance("RSA");
+        try (StringReader keyReader = new StringReader(pemStr); PemReader pemReader = new PemReader(keyReader)) {
+            PemObject pemObject = pemReader.readPemObject();
+            byte[] content = pemObject.getContent();
+            X509EncodedKeySpec pubKeySpec = new X509EncodedKeySpec(content);
+            PublicKey publicKey = factory.generatePublic(pubKeySpec);
+            return publicKey;
+        }
+        //X509EncodedKeySpec keySpecX509 = new X509EncodedKeySpec(
+        //        Base64.decode(
+        //                pemStr.replaceAll("\\n", "").replace("-----BEGIN PUBLIC KEY-----", "").replace("-----END PUBLIC KEY-----", "")
+        //                        .getBytes()));
+        //RSAPublicKey pubKey = (RSAPublicKey) factory.generatePublic(keySpecX509);
+    }
+
+    private PublicKey getPublicKey(RSAPublicKeySpec rsaPrivateKey) throws NoSuchAlgorithmException, InvalidKeySpecException {
+        KeyFactory factory = KeyFactory.getInstance("RSA");
+        PublicKey publicKey = factory.generatePublic(new RSAPublicKeySpec(rsaPrivateKey.getModulus(), rsaPrivateKey.getPublicExponent()));
+        return publicKey;
+    }
+
+    private PublicKey getPublicKey(BCRSAPrivateCrtKey rsaPrivateKey) throws NoSuchAlgorithmException, InvalidKeySpecException {
+        KeyFactory factory = KeyFactory.getInstance("RSA");
+        PublicKey publicKey = factory.generatePublic(new RSAPublicKeySpec(rsaPrivateKey.getModulus(), rsaPrivateKey.getPublicExponent()));
+        return publicKey;
+    }*/
+
+    private boolean sshDisconnect() throws Exception {
+        SshConfig config = task.getSsh();
+        String host = config.getHost();
+        int port = config.getPort();
+        if (simulateConnection) {
+            log.info("SshClientInstaller: Simulate disconnect from remote host: task #{}: host: {}:{}", taskCounter, host, port);
+            return true;
+        }
+
+        try {
+            if (streamLogger!=null)
+                streamLogger.close();
+
+            //channel.close(false).await();
+            session.close(false);
+            //simpleClient.close();
+            sshClient.stop();
+
+            log.info("SshClientInstaller: Disconnected from remote host: task #{}: host: {}:{}", taskCounter, host, port);
+            return true;
+        } catch (Exception ex) {
+            log.error("SshClientInstaller: Error while disconnecting from remote host: task #{}: ", taskCounter, ex);
+            throw ex;
+        } finally {
+            session = null;
+            //simpleClient = null;
+            sshClient = null;
+        }
+    }
+
+    private void initStreamLogger() throws IOException {
+        if (streamLogger!=null) return;
+
+        String address = session.getConnectAddress().toString().replace("/","").replace(":", "-");
+        //log.trace("SshClientInstaller: address: {}", address);
+        String logFile = StringUtils.isNotBlank(properties.getSessionRecordingDir())
+                ? properties.getSessionRecordingDir()+"/"+address+"-"+ simpleDateFormat.format(new Date())+"-"+taskCounter+".txt"
+                : null;
+        log.info("SshClientInstaller: Task #{}: Session will be recorded in file: {}", taskCounter, logFile);
+        this.streamLogger = new StreamLogger(logFile, "  Task #"+taskCounter);
+    }
+
+    private void setChannelStreams(ChannelSession channel) throws IOException {
+        initStreamLogger();
+        channel.setIn( streamLogger.getIn() );
+        channel.setOut( streamLogger.getOut() );
+        channel.setErr( streamLogger.getErr() );
+    }
+
+    /*public boolean sshOpenShell() throws IOException {
+        if (simulateConnection) {
+            log.info("SshClientInstaller: Simulate open shell channel: task #{}", taskCounter);
+            return true;
+        }
+
+        shellChannel = session.createShellChannel();
+        setChannelStreams(shellChannel);
+        shellChannel.open().verify(connectTimeout);
+        //shellPipedIn = shellChannel.getInvertedIn();
+        log.info("SshClientInstaller: Opened shell channel: task #{}", taskCounter);
+
+        shellChannel.waitFor(
+                EnumSet.of(ClientChannelEvent.CLOSED),
+                authenticationTimeout);
+                //TimeUnit.SECONDS.toMillis(5));
+        log.info("SshClientInstaller: Shell channel ready: task #{}", taskCounter);
+
+        return true;
+    }
+
+    public boolean sshCloseShell() throws IOException {
+        if (simulateConnection) {
+            log.info("SshClientInstaller: Simulate close shell channel: task #{}", taskCounter);
+            return true;
+        }
+
+        shellChannel.close();
+        shellChannel = null;
+        //shellPipedIn = null;
+        streamLogger.close();
+        streamLogger = null;
+        log.info("SshClientInstaller: Closed shell channel: task #{}", taskCounter);
+        return true;
+    }
+
+    public boolean sshShellExec(@NotNull String command, long executionTimeout) throws IOException {
+        if (simulateConnection || simulateExecution) {
+            log.info("SshClientInstaller: Simulate command execution: task #{}: command: {}", taskCounter, command);
+            return true;
+        }
+
+        // Send command to remote side
+        if (!command.endsWith("\n"))
+            command += "\n";
+        log.info("SshClientInstaller: Sending command: {}", command);
+        streamLogger.getInvertedIn().write(command.getBytes());
+        streamLogger.getInvertedIn().flush();
+
+        // Search remote side output for expected patterns
+        // Not implemented
+
+        shellChannel.waitFor(
+                EnumSet.of(ClientChannelEvent.CLOSED),
+                executionTimeout>0 ? executionTimeout : commandExecutionTimeout);
+                //TimeUnit.SECONDS.toMillis(5));
+        return true;
+    }*/
+
+    public Integer sshExecCmd(String command) throws IOException {
+        return sshExecCmd(command, commandExecutionTimeout);
+    }
+
+    public Integer sshExecCmd(String command, long executionTimeout) throws IOException {
+        if (simulateConnection || simulateExecution) {
+            log.info("SshClientInstaller: Simulate shell command execution: task #{}: command: {}", taskCounter, command);
+            return null;
+        }
+
+        // Using EXEC channel
+        Integer exitStatus = null;
+        ChannelExec channel = session.createExecChannel(command);
+        setChannelStreams(channel);
+        log.debug("SshClientInstaller: task #{}: EXEC: New channel id: {}", taskCounter, channel.getChannelId());
+        //streamLogger.getInvertedIn().write(command.getBytes());
+        streamLogger.logMessage(String.format("EXEC: %s\n", command));
+        try {
+            // Sending command to remote side
+            log.debug("SshClientInstaller: task #{}: EXEC: Sending command for execution: {}   (connect timeout: {}ms)", taskCounter, command, connectTimeout);
+            session.resetIdleTimeout();
+            channel.open().verify(connectTimeout);
+            log.trace("SshClientInstaller: task #{}: EXEC: Sending command verified: {}", taskCounter, command);
+            log.debug("SshClientInstaller: task #{}: EXEC: Opened channel id: {}", taskCounter, channel.getChannelId());
+
+            //XXX: TODO: Search remote side output for expected patterns
+
+            // Wait until channel closes from server side (i.e. command completed) or timeout occurs
+            log.trace("SshClientInstaller: task #{}: EXEC: instruction execution-timeout: {}", taskCounter, executionTimeout);
+            log.trace("SshClientInstaller: task #{}: EXEC: default command-execution-timeout: {}", taskCounter, commandExecutionTimeout);
+            long execTimeout = executionTimeout != 0 ? executionTimeout : commandExecutionTimeout;
+            log.debug("SshClientInstaller: task #{}: EXEC: effective instruction execution-timeout: {}", taskCounter, execTimeout);
+            Set<ClientChannelEvent> eventSet = channel.waitFor(
+                    EnumSet.of(ClientChannelEvent.CLOSED),
+                    execTimeout);
+                    //TimeUnit.SECONDS.toMillis(50));
+            log.debug("SshClientInstaller: task #{}: EXEC: Exit event set: {}", taskCounter, eventSet);
+            exitStatus = channel.getExitStatus();
+            log.debug("SshClientInstaller: task #{}: EXEC: Exit status: {}", taskCounter, exitStatus);
+        } finally {
+            channel.close();
+        }
+
+        return exitStatus;
+    }
+
+    public boolean sshFileDownload(String remoteFilePath, String localFilePath) throws IOException {
+        if (simulateConnection || simulateExecution) {
+            log.info("SshClientInstaller: Simulate file download: task #{}: remote: {} -> local: {}", taskCounter, remoteFilePath, localFilePath);
+            return true;
+        }
+
+        streamLogger.logMessage(String.format("DOWNLOAD: SCP: %s -> %s\n", remoteFilePath, localFilePath));
+        try {
+            log.info("SshClientInstaller: Downloading file: task #{}: remote: {} -> local: {}", taskCounter, remoteFilePath, localFilePath);
+            ScpClientCreator creator = new DefaultScpClientCreator();
+            ScpClient scpClient = creator.createScpClient(session);
+            scpClient.download(remoteFilePath, localFilePath, ScpClient.Option.PreserveAttributes);
+            log.info("SshClientInstaller: File download completed: task #{}: remote: {} -> local: {}", taskCounter, remoteFilePath, localFilePath);
+        } catch (Exception ex) {
+            log.error("SshClientInstaller: File download failed: task #{}: remote: {} -> local: {} Exception: ", taskCounter, remoteFilePath, localFilePath, ex);
+            throw ex;
+        }
+
+        return true;
+    }
+
+    public boolean sshFileUpload(String localFilePath, String remoteFilePath) throws IOException {
+        if (simulateConnection || simulateExecution) {
+            log.info("SshClientInstaller: Simulate file upload: task #{}: local: {} -> remote: {}", taskCounter, localFilePath, remoteFilePath);
+            return true;
+        }
+
+        streamLogger.logMessage(String.format("UPLOAD: SCP: %s -> %s\n", localFilePath, remoteFilePath));
+        try {
+            long startTm = System.currentTimeMillis();
+            log.info("SshClientInstaller: Uploading file: task #{}: local: {} -> remote: {}", taskCounter, localFilePath, remoteFilePath);
+            ScpClientCreator creator = new DefaultScpClientCreator();
+            ScpClient scpClient = creator.createScpClient(session);
+            scpClient.upload(localFilePath, remoteFilePath, ScpClient.Option.PreserveAttributes);
+            long endTm = System.currentTimeMillis();
+            log.info("SshClientInstaller: File upload completed in {}ms: task #{}: local: {} -> remote: {}", endTm-startTm, taskCounter, localFilePath, remoteFilePath);
+        } catch (Exception ex) {
+            log.error("SshClientInstaller: File upload failed: task #{}: local: {} -> remote: {} Exception: ", taskCounter, localFilePath, remoteFilePath, ex);
+            throw ex;
+        }
+
+        return true;
+    }
+
+    public boolean sshFileWrite(String content, String remoteFilePath, boolean isExecutable) throws IOException {
+        if (simulateConnection || simulateExecution) {
+            log.info("SshClientInstaller: Simulate file upload: task #{}: remote: {}, content-length={}", taskCounter, remoteFilePath, content.length());
+            return true;
+        }
+
+        streamLogger.logMessage(String.format("WRITE FILE: SCP: %s, content-length=%d \n", remoteFilePath, content.length()));
+        try {
+            long timestamp = System.currentTimeMillis();
+            /*Collection<PosixFilePermission> permissions = isExecutable
+                    ? Arrays.asList(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)
+                    : Arrays.asList(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE);
+            log.info("SshClientInstaller: Uploading file: task #{}: remote: {}, perm={}, content-length={}", taskCounter, remoteFilePath, permissions, content.length());
+            log.trace("SshClientInstaller: Uploading file: task #{}: remote: {}, perm={}, content:\n{}", taskCounter, remoteFilePath, permissions, content);
+            ScpClient scpClient = session.createScpClient();
+            scpClient.upload(content.getBytes(), remoteFilePath, permissions, new ScpTimestamp(timestamp, timestamp));
+            */
+
+             /*
+             The alternative approach next is much faster than the original approach above (commented out)
+             Old approach: write bytes directly to remote file
+             New approach: write contents to a local temp. file and then upload it to remote side
+             */
+
+            // Write contents to a temporary local file
+            File tmpDir = Paths.get(properties.getServerTmpDir()).toFile();
+            tmpDir.mkdirs();
+            File tmp = File.createTempFile("bci_upload_", ".tmp", tmpDir);
+            log.debug("SshClientInstaller: Write to temp. file: task #{}: temp-file: {}, remote: {}, content-length: {}", taskCounter, tmp, remoteFilePath, content.length());
+            log.trace("SshClientInstaller: Write to temp. file: task #{}: temp-file: {}, remote: {}, content:\n{}", taskCounter, tmp, remoteFilePath, content);
+            try (FileWriter fw = new FileWriter(tmp.getAbsoluteFile())) { fw.write(content); }
+
+            // Upload temporary local file to remote side
+            log.trace("SshClientInstaller: Call 'sshFileUpload': task #{}: temp-file={}, remote={}", taskCounter, tmp, remoteFilePath);
+            sshFileUpload(tmp.getAbsolutePath(), remoteFilePath);
+
+            // Delete temporary file
+            if (!properties.isKeepTempFiles()) {
+                log.trace("SshClientInstaller: Remove temp. file: task #{}: temp-file={}", taskCounter, tmp);
+                tmp.delete();
+            }
+
+            long endTm = System.currentTimeMillis();
+            log.info("SshClientInstaller: File upload completed in {}ms: task #{}: remote: {}, content-length={}", endTm-timestamp, taskCounter, remoteFilePath, content.length());
+            log.trace("SshClientInstaller: File upload completed in {}ms: task #{}: remote: {}, content:\n{}", endTm-timestamp, taskCounter, remoteFilePath, content);
+        } catch (Exception ex) {
+            log.error("SshClientInstaller: File upload failed: task #{}: remote: {}, Exception: ", taskCounter, remoteFilePath, ex);
+            throw ex;
+        }
+
+        return true;
+    }
+
+    private INSTRUCTION_RESULT executeInstructionSets() throws IOException {
+        List<InstructionsSet> instructionsSetList = task.getInstructionSets();
+        INSTRUCTION_RESULT exitResult = INSTRUCTION_RESULT.SUCCESS;
+        int cntSuccess = 0;
+        int cntFail = 0;
+        for (InstructionsSet instructionsSet : instructionsSetList) {
+            log.info("\n  ----------------------------------------------------------------------\n  Task #{} :  Instruction Set: {}", taskCounter, instructionsSet.getDescription());
+
+            // Check installation instructions condition
+            try {
+                if (! InstructionsService.getInstance().checkCondition(instructionsSet, task.getNodeRegistryEntry().getPreregistration())) {
+                    log.info("SshClientInstaller: Task #{}: Installation Instructions set is skipped due to failed condition: {}", taskCounter, instructionsSet.getDescription());
+                    if (instructionsSet.isStopOnConditionFail()) {
+                        log.info("SshClientInstaller: Task #{}: No further installation instructions sets will be executed due to stopOnConditionFail: {}", taskCounter, instructionsSet.getDescription());
+                        exitResult = INSTRUCTION_RESULT.FAIL;
+                        break;
+                    }
+                    continue;
+                }
+                log.debug("SshClientInstaller: Task #{}: Condition evaluation for Installation Instructions Set succeeded: {}", taskCounter, instructionsSet.getDescription());
+            } catch (Exception e) {
+                log.error("sshClientInstaller: Task #{}: Installation Instructions Set Condition evaluation error. Will not process remaining installation instructions sets: {}\n", taskCounter, instructionsSet.getDescription(), e);
+                exitResult = INSTRUCTION_RESULT.FAIL;
+                break;
+            }
+
+            // Execute installation instructions
+            log.info("SshClientInstaller: Task #{}: Executing installation instructions set: {}", taskCounter, instructionsSet.getDescription());
+            streamLogger.logMessage(
+                    String.format("\n  ----------------------------------------------------------------------\n  Task #%d :  Executing instruction set: %s\n",
+                    taskCounter, instructionsSet.getDescription()));
+            INSTRUCTION_RESULT result = executeInstructions(instructionsSet);
+            if (result==INSTRUCTION_RESULT.FAIL) {
+                log.error("SshClientInstaller: Task #{}: Installation Instructions set failed: {}", taskCounter, instructionsSet.getDescription());
+                cntFail++;
+                if (!continueOnFail) {
+                    exitResult = INSTRUCTION_RESULT.FAIL;
+                    break;
+                }
+            } else
+            if (result==INSTRUCTION_RESULT.EXIT) {
+                log.info("SshClientInstaller: Task #{}: Instruction set processing exits", taskCounter);
+                cntSuccess++;
+                exitResult = INSTRUCTION_RESULT.EXIT;
+                break;
+            } else {
+                log.info("SshClientInstaller: Task #{}: Installation Instructions set succeeded: {}", taskCounter, instructionsSet.getDescription());
+                cntSuccess++;
+            }
+        }
+        log.info("\n  -------------------------------------------------------------------------\n  Task #{} :  Instruction sets processed: successful={}, failed={}, exit-result={}", taskCounter, cntSuccess, cntFail, exitResult);
+        return exitResult;
+    }
+
+    private INSTRUCTION_RESULT executeInstructions(InstructionsSet instructionsSet) throws IOException {
+        Map<String, String> valueMap = task.getNodeRegistryEntry().getPreregistration();
+        int numOfInstructions = instructionsSet.getInstructions().size();
+        int cnt = 0;
+        int insCount = instructionsSet.getInstructions().size();
+        for (Instruction ins : instructionsSet.getInstructions()) {
+            if (ins==null) continue;
+            cnt++;
+
+            // Check instruction condition
+            try {
+                if (! InstructionsService.getInstance().checkCondition(ins, valueMap)) {
+                    log.info("SshClientInstaller: Task #{}: Instruction is skipped due to failed condition {}/{}: {}", taskCounter, cnt, numOfInstructions, ins.description());
+                    if (ins.isStopOnConditionFail()) {
+                        log.info("SshClientInstaller: Task #{}: No further instructions will be executed due to stopOnConditionFail: {}/{}: {}", taskCounter, cnt, numOfInstructions, ins.description());
+                        return INSTRUCTION_RESULT.FAIL;
+                    }
+                    continue;
+                }
+                log.debug("SshClientInstaller: Task #{}: Condition evaluation for instruction succeeded: {}/{}: {}", taskCounter, cnt, numOfInstructions, ins.description());
+            } catch (Exception e) {
+                log.error("sshClientInstaller: Task #{}: Instruction Condition evaluation error. Will not process remaining instructions: {}/{}: {}\n", taskCounter, cnt, numOfInstructions, ins.description(), e);
+                return INSTRUCTION_RESULT.FAIL;
+            }
+
+            // Execute instruction
+            ins = InstructionsService
+                    .getInstance()
+                    .resolvePlaceholders(ins, valueMap);
+            log.trace("SshClientInstaller: Task #{}: Executing instruction {}/{}: {}", taskCounter, cnt, numOfInstructions, ins);
+            log.info("SshClientInstaller: Task #{}: Executing instruction {}/{}: {}", taskCounter, cnt, numOfInstructions, ins.description());
+            Integer exitStatus;
+            boolean result = true;
+            switch (ins.taskType()) {
+                case LOG:
+                    log.info("SshClientInstaller: Task #{}: LOG: {}", taskCounter, ins.message());
+                    break;
+                case CMD:
+                    log.info("SshClientInstaller: Task #{}: EXEC: {}", taskCounter, ins.command());
+                    int retries = 0;
+                    int maxRetries = ins.retries();
+                    while (true) {
+                        try {
+                            exitStatus = sshExecCmd(ins.command(), ins.executionTimeout());
+                            result = (exitStatus!=null);
+                            //result = (exitStatus==0);
+                            log.info("SshClientInstaller: Task #{}: EXEC: exit-status={}", taskCounter, exitStatus);
+                            if (result) break;
+                        } catch (Exception ex) {
+                            if (retries+1>=maxRetries)
+                                throw ex;
+                            else
+                                log.error("SshClientInstaller: Task #{}: EXEC: Last command raised exception: ", taskCounter, ex);
+                        }
+
+                        retries++;
+                        if (retries<=maxRetries) {
+                            log.info("SshClientInstaller: Task #{}: Retry {}/{} for instruction {}/{}: {}",
+                                    taskCounter, retries, maxRetries, cnt, numOfInstructions, ins.description());
+                        } else {
+                            if (maxRetries>0)
+                                log.error("sshClientInstaller: Task #{}: Last instruction failed {} times. Giving up", taskCounter, maxRetries);
+                            result = false;
+                            break;
+                        }
+                    }
+                    break;
+                /*case SHELL:
+                    log.info("SshClientInstaller: Task #{}: SHELL: {}", taskCounter, ins.getCommand());
+                    retries = 0;
+                    maxRetries = ins.getRetries();
+                    while (true) {
+                        try {
+                            result = sshShellExec(ins.getCommand(), ins.getExecutionTimeout());
+                            log.info("SshClientInstaller: Task #{}: SHELL: exit-status={}", taskCounter, result);
+                            if (result) break;
+                        } catch (Exception ex) {
+                            if (retries+1>=maxRetries)
+                                throw ex;
+                            else
+                                log.error("SshClientInstaller: Task #{}: SHELL: Last command raised exception: ", taskCounter, ex);
+                        }
+
+                        retries++;
+                        if (retries<=maxRetries) {
+                            log.info("SshClientInstaller: Task #{}: Retry {}/{} for instruction {}/{}: {}",
+                                    taskCounter, retries, maxRetries, cnt, numOfInstructions, ins.getDescription());
+                        } else {
+                            if (maxRetries>0)
+                                log.error("sshClientInstaller: Task #{}: Last instruction failed {} times. Giving up", taskCounter, maxRetries);
+                            result = false;
+                            break;
+                        }
+                    }
+                    break;*/
+                case FILE:
+                    //log.info("SshClientInstaller: Task #{}: FILE: {}, content-length={}", taskCounter, ins.getFileName(), ins.getContents().length());
+                    if (Paths.get(ins.localFileName()).toFile().isDirectory()) {
+                        log.info("SshClientInstaller: Task #{}: FILE: COPY-PROCESS DIR: {} -> {}", taskCounter, ins.localFileName(), ins.fileName());
+                        result = copyDir(ins.localFileName(), ins.fileName(), valueMap);
+                    } else
+                    if (Paths.get(ins.localFileName()).toFile().isFile()) {
+                        log.info("SshClientInstaller: Task #{}: FILE: COPY-PROCESS FILE: {} -> {}", taskCounter, ins.localFileName(), ins.fileName());
+                        Path sourceFile = Paths.get(ins.localFileName());
+                        Path sourceBaseDir = Paths.get(ins.localFileName()).getParent();
+                        result = copyFile(sourceFile, sourceBaseDir, ins.fileName(), valueMap, ins.executable());
+                    } else {
+                        log.error("SshClientInstaller: Task #{}: FILE: ERROR: Local file is not directory or normal file: {}", taskCounter, ins.localFileName());
+                        result = false;
+                    }
+                    break;
+                case COPY:
+                case UPLOAD:
+                    log.info("SshClientInstaller: Task #{}: UPLOAD: {} -> {}", taskCounter, ins.localFileName(), ins.fileName());
+                    result = sshFileUpload(ins.localFileName(), ins.fileName());
+                    break;
+                case DOWNLOAD:
+                    log.info("SshClientInstaller: Task #{}: DOWNLOAD: {} -> {}", taskCounter, ins.fileName(), ins.localFileName());
+                    result = sshFileDownload(ins.fileName(), ins.localFileName());
+                    if (result)
+                        result = processPatterns(ins, valueMap);
+                    break;
+                case CHECK:
+                    log.info("SshClientInstaller: Task #{}: CHECK: {}", taskCounter, ins.command());
+                    exitStatus = sshExecCmd(ins.command());
+                    log.info("SshClientInstaller: Task #{}: CHECK: exit-status={}", taskCounter, exitStatus);
+                    log.debug("SshClientInstaller: Task #{}: CHECK: Result: match={}, match-status={}, exec-status={}",
+                            taskCounter, ins.match(), ins.exitCode(), exitStatus);
+                    if (ins.match() && exitStatus==ins.exitCode()
+                        || !ins.match() && exitStatus!=ins.exitCode())
+                    {
+                        log.info("SshClientInstaller: Task #{}: CHECK: MATCH: {}", taskCounter, ins.message());
+                        log.info("SshClientInstaller: Task #{}: CHECK: MATCH: Will not process more instructions", taskCounter);
+                        return INSTRUCTION_RESULT.SUCCESS;
+                    }
+                    break;
+
+                case SET_VARS:
+                    log.info("SshClientInstaller: Task #{}: SET_VARS:", taskCounter);
+                    if (ins.variables()!=null && ins.variables().size()>0) {
+                        ins.variables().forEach((varName, varExpression) -> {
+                            try {
+                                String varValue = InstructionsService.getInstance().processPlaceholders(varExpression, valueMap);
+                                log.info("SshClientInstaller: Task #{}:   Setting VAR: {} = {}", taskCounter, varName, varValue);
+                                valueMap.put(varName, varValue);
+                            } catch (Exception e) {
+                                log.error("SshClientInstaller: Task #{}:   ERROR while Setting VAR: {}: {}\n", taskCounter, varName, varExpression, e);
+                            }
+                        });
+                    } else
+                        log.warn("SshClientInstaller: Task #{}: SET_VARS:  No variables specified", taskCounter);
+                    break;
+                case UNSET_VARS:
+                    log.info("SshClientInstaller: Task #{}: UNSET_VARS:", taskCounter);
+                    if (ins.variables()!=null && ins.variables().size()>0) {
+                        Set<String> vars = ins.variables().keySet();
+                        log.info("SshClientInstaller: Task #{}:   Unsetting VAR: {}", taskCounter, vars);
+                        valueMap.keySet().removeAll(vars);
+                    } else
+                        log.warn("SshClientInstaller: Task #{}: UNSET_VARS:  No variables specified", taskCounter);
+                    break;
+                case PRINT_VARS:
+                    //log.info("SshClientInstaller: Task #{}: PRINT_VARS:", taskCounter);
+                    String output = valueMap.entrySet().stream()
+                            .map(e -> "    VAR: "+e.getKey()+" = "+e.getValue())
+                            .collect(Collectors.joining("\n"));
+                    log.info("SshClientInstaller: Task #{}: PRINT_VARS:\n{}", taskCounter, output);
+                    break;
+                case EXIT_SET:
+                    log.info("SshClientInstaller: Task #{}: EXIT_SET: Stop this instruction set processing", taskCounter);
+                    try {
+                        if (StringUtils.isNotBlank(ins.command())) {
+                            String exitResult = ins.command().trim().toUpperCase();
+                            log.info("SshClientInstaller: Task #{}: EXIT_SET: Result={}", taskCounter, exitResult);
+                            return INSTRUCTION_RESULT.valueOf(exitResult);
+                        }
+                    } catch (Exception e) {
+                        log.error("SshClientInstaller: Task #{}: EXIT_SET: Invalid EXIT_SET result: {}. Will return FAIL", taskCounter, ins.command());
+                        return INSTRUCTION_RESULT.FAIL;
+                    }
+                    log.info("SshClientInstaller: Task #{}: EXIT_SET: Result={}", taskCounter, INSTRUCTION_RESULT.SUCCESS);
+                    return INSTRUCTION_RESULT.SUCCESS;
+                case EXIT:
+                    log.info("SshClientInstaller: Task #{}: EXIT: Stop any further instruction processing", taskCounter);
+                    return INSTRUCTION_RESULT.EXIT;
+                default:
+                    log.error("sshClientInstaller: Unknown instruction type. Ignoring it: {}", ins);
+            }
+            if (!result) {
+                log.error("sshClientInstaller: Last instruction failed. Will not process remaining instructions");
+                return INSTRUCTION_RESULT.FAIL;
+            }
+
+            if (cnt<insCount)
+                log.trace("sshClientInstaller: Continuing with next command...");
+            else
+                log.trace("sshClientInstaller: No more instructions");
+        }
+        return INSTRUCTION_RESULT.SUCCESS;
+    }
+
+    public boolean copyDir(String sourceDir, String targetDir, Map<String,String> valueMap) throws IOException {
+        // Copy files from EMS server to Baguette Client
+        if (StringUtils.isNotEmpty(sourceDir) && StringUtils.isNotEmpty(targetDir)) {
+            Path baseDir = Paths.get(sourceDir).toAbsolutePath();
+            try (Stream<Path> stream = Files.walk(baseDir, Integer.MAX_VALUE)) {
+                List<Path> paths = stream
+                        .filter(Files::isRegularFile)
+                        .map(Path::toAbsolutePath)
+                        .sorted()
+                        .collect(Collectors.toList());
+                for (Path p : paths) {
+                    if (!copyFile(p, baseDir, targetDir, valueMap, false))
+                        return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    public boolean copyFile(Path sourcePath, Path sourceBaseDir, String targetDir, Map<String,String> valueMap, boolean isExecutable) throws IOException {
+        String targetFile = StringUtils.substringAfter(sourcePath.toUri().toString(), sourceBaseDir.toUri().toString());
+        if (!targetFile.startsWith("/")) targetFile = "/"+targetFile;
+        targetFile = targetDir + targetFile;
+
+        String contents = new String(Files.readAllBytes(sourcePath));
+        log.info("SshClientInstaller: Task #{}: FILE: {}, content-length={}", taskCounter, targetFile, contents.length());
+        contents = StringSubstitutor.replace(contents, valueMap);
+        log.trace("SshClientInstaller: Task #{}: FILE: {}, final-content:\n{}", taskCounter, targetFile, contents);
+
+        String description = String.format("Copy file from server to temp to client: %s -> %s", sourcePath.toString(), targetFile);
+
+        return sshFileWrite(contents, targetFile, isExecutable);
+    }
+
+    private boolean processPatterns(Instruction ins, Map<String,String> valueMap) {
+        Map<String, Pattern> patterns = ins.patterns();
+        if (patterns==null || patterns.size()==0) {
+            log.info("SshClientInstaller: processPatterns: No patterns to process");
+            return true;
+        }
+
+        // Read local file
+        String[] linesArr;
+        try (Stream<String> lines = Files.lines(Paths.get(ins.localFileName()))) {
+            linesArr = lines.toArray(String[]::new);
+        } catch (IOException e) {
+            log.error("SshClientInstaller: processPatterns: Error while reading local file: {} -- Exception: ", ins.localFileName(), e);
+            return false;
+        }
+
+        // Process file lines against instruction patterns
+        patterns.forEach((varName,pattern) -> {
+            log.trace("SshClientInstaller: processPatterns: For-Each: var-name={}, pattern={}, pattern-flags={}", varName, pattern, pattern.flags());
+            Matcher matcher = null;
+            for (String line : linesArr) {
+                Matcher m = pattern.matcher(line);
+                if (m.matches()) {
+                    matcher = m;
+                    //break;    // Uncomment to return the first match. Comment to return the last match.
+                }
+            }
+            if (matcher!=null && matcher.matches()) {
+                String varValue = matcher.group( matcher.groupCount()>0 ? 1 : 0 );
+                log.info("SshClientInstaller: processPatterns: Setting variable '{}' to: {}", varName, varValue);
+                valueMap.put(varName, varValue);
+            } else {
+                log.info("SshClientInstaller: processPatterns: No match for variable '{}' with pattern: {}", varName, pattern);
+            }
+        });
+
+        return true;
+    }
+
+    @Override
+    public void preProcessTask() {
+        // Throw exception to prevent task exception, if task data have problem
+    }
+
+    @Override
+    public boolean postProcessTask() {
+        log.trace("SshClientInstaller: postProcessTask: BEGIN:\n{}", task.getNodeRegistryEntry().getPreregistration());
+
+        // Check if Baguette client has been installed (or failed to install)
+        log.trace("SshClientInstaller: postProcessTask: CLIENT INSTALLATION....");
+        boolean result = postProcessVariable(
+                properties.getClientInstallVarName(),
+                properties.getClientInstallSuccessPattern(),
+                value -> { task.getNodeRegistryEntry().nodeInstallationComplete(value); return true; },
+                null, null);
+        log.trace("SshClientInstaller: postProcessTask: CLIENT INSTALLATION.... result: {}", result);
+        if (result) return true;
+
+        // Check if Baguette client installation has failed
+        log.trace("SshClientInstaller: postProcessTask: CLIENT INSTALLATION FAILED....");
+        result = postProcessVariable(
+                properties.getClientInstallVarName(),
+                properties.getClientInstallErrorPattern(),
+                value -> { task.getNodeRegistryEntry().nodeInstallationError(value); return true; },
+                null, null);
+        log.trace("SshClientInstaller: postProcessTask: CLIENT INSTALLATION.... result: {}", result);
+        if (result) return true;
+
+        // Check if Baguette client installation has been skipped (not attempted at all)
+        log.trace("SshClientInstaller: postProcessTask: CLIENT INSTALLATION SKIP....");
+        result = postProcessVariable(
+                properties.getSkipInstallVarName(),
+                properties.getSkipInstallPattern(),
+                value -> { task.getNodeRegistryEntry().nodeNotInstalled(value); return true; },
+                null, null);
+        log.trace("SshClientInstaller: postProcessTask: CLIENT INSTALLATION SKIP.... result: {}", result);
+        if (result) return true;
+
+        // Check if the Node must be ignored by EMS
+        log.trace("SshClientInstaller: postProcessTask: NODE IGNORE....");
+        result = postProcessVariable(
+                properties.getIgnoreNodeVarName(),
+                properties.getIgnoreNodePattern(),
+                value -> { task.getNodeRegistryEntry().nodeIgnore(value); return true; },
+                null, null);
+        log.trace("SshClientInstaller: postProcessTask: NODE IGNORE.... result: {}", result);
+        if (result) return true;
+
+        // Process defaults, if variables are missing or inconclusive
+        log.trace("SshClientInstaller: postProcessTask: DEFAULTS....");
+        if (properties.isIgnoreNodeIfVarIsMissing()) {
+            log.trace("SshClientInstaller: postProcessTask: DEFAULTS.... NODE IGNORED");
+            task.getNodeRegistryEntry().nodeIgnore(null);
+        } else
+        if (properties.isSkipInstallIfVarIsMissing()) {
+            log.trace("SshClientInstaller: postProcessTask: DEFAULTS.... CLIENT INSTALLATION SKIPPED");
+            task.getNodeRegistryEntry().nodeNotInstalled(null);
+        } else
+        if (properties.isClientInstallSuccessIfVarIsMissing()) {
+            log.trace("SshClientInstaller: postProcessTask: DEFAULTS.... CLIENT INSTALLED");
+            task.getNodeRegistryEntry().nodeInstallationComplete(null);
+        } else
+        if (properties.isClientInstallErrorIfVarIsMissing()) {
+            log.trace("SshClientInstaller: postProcessTask: DEFAULTS.... CLIENT INSTALLATION ERROR");
+            task.getNodeRegistryEntry().nodeInstallationError(null);
+        } else
+            log.trace("SshClientInstaller: postProcessTask: DEFAULTS.... NO DEFAULT");
+        log.trace("SshClientInstaller: postProcessTask: END");
+        return true;
+    }
+
+    private boolean postProcessVariable(String varName, Pattern pattern, @NonNull Function<String,Boolean> match, Function<String,Boolean> notMatch, Supplier<Boolean> missing) {
+        log.trace("SshClientInstaller: postProcessVariable: var={}, pattern={}", varName, pattern);
+        if (StringUtils.isNotBlank(varName) && pattern!=null) {
+            String value = task.getNodeRegistryEntry().getPreregistration().get(varName);
+            log.trace("SshClientInstaller: postProcessVariable: var={}, value={}", varName, value);
+            if (value!=null) {
+                if (pattern.matcher(value).matches()) {
+                    log.trace("SshClientInstaller: postProcessVariable: MATCH-END: var={}, value={}, pattern={}", varName, value, pattern);
+                    return match.apply(value);
+                } else {
+                    log.trace("SshClientInstaller: postProcessVariable: NO MATCH: var={}, value={}, pattern={}", varName, value, pattern);
+                    if (notMatch!=null) {
+                        log.trace("SshClientInstaller: postProcessVariable: NO MATCH-END: var={}, value={}, pattern={}", varName, value, pattern);
+                        return notMatch.apply(value);
+                    }
+                }
+            }
+        }
+        if (missing!=null) {
+            log.trace("SshClientInstaller: postProcessVariable: DEFAULT-END: var={}", varName);
+            return missing.get();
+        }
+        log.trace("SshClientInstaller: postProcessVariable: False-END: var={}", varName);
+        return false;
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/SshConfig.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install;
+
+import lombok.Builder;
+import lombok.Data;
+import lombok.ToString;
+
+/**
+ * SSH connection information
+ */
+@Data
+@Builder
+@ToString(exclude = {"password", "privateKey"})
+public class SshConfig {
+    private String host;
+    @Builder.Default
+    private int port = 22;
+    private String username;
+    private String password;
+    private String privateKey;
+    private String fingerprint;
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/SshJsClientInstaller.java

@@ -0,0 +1,187 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install;
+
+import gr.iccs.imu.ems.baguette.client.install.instruction.INSTRUCTION_RESULT;
+import gr.iccs.imu.ems.baguette.client.install.instruction.InstructionsSet;
+import lombok.Builder;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.util.ResourceUtils;
+
+import javax.script.ScriptContext;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+/**
+ * SSH-Javascript client installer
+ */
+@Slf4j
+public class SshJsClientInstaller extends SshClientInstaller {
+
+    @Builder(builderMethodName = "jsBuilder")
+    public SshJsClientInstaller(ClientInstallationTask task, long taskCounter, ClientInstallationProperties properties) {
+        super(task, taskCounter, properties);
+    }
+
+    @Override
+    public boolean executeTask() {
+        log.info("SshJsClientInstaller: Task #{}: Opening SSH connection...", getTaskCounter());
+        if (!openSshConnection()) {
+            return false;
+        }
+
+        boolean success;
+        try {
+            log.info("SshJsClientInstaller: Task #{}: Executing JS installation scripts...", getTaskCounter());
+            INSTRUCTION_RESULT exitResult = executeJsScripts();
+            success = exitResult != INSTRUCTION_RESULT.FAIL;
+        } catch (Exception ex) {
+            log.error("SshJsClientInstaller: Task #{}: Exception while executing JS installation scripts: ", getTaskCounter(), ex);
+            success = false;
+        }
+
+        log.info("SshJsClientInstaller: Task #{}: Closing SSH connection...", getTaskCounter());
+        return closeSshConnection(success);
+    }
+
+    private INSTRUCTION_RESULT executeJsScripts() throws IOException {
+        List<String> jsScriptList = Optional.ofNullable(getTask().getInstructionSets())
+                .orElseThrow(() -> new IllegalArgumentException("No SSH-Javascript installer scripts configured"))
+                .stream()
+                .map(InstructionsSet::getFileName)
+                .filter(StringUtils::isNotBlank)
+                .map(String::trim)
+                .collect(Collectors.toList());
+        log.debug("SshJsClientInstaller: Task #{}: Configured installation scripts: {}", getTaskCounter(), jsScriptList);
+        if (jsScriptList.isEmpty())
+            throw new IllegalArgumentException("SSH-Javascript installation scripts are blank");
+
+        INSTRUCTION_RESULT exitResult = null;
+        int cntSuccess = 0;
+        int cntFail = 0;
+        for (String jsScript : jsScriptList) {
+            log.info("\n  ----------------------------------------------------------------------\n  Task #{} :  JS installation script: {}", getTaskCounter(), jsScript);
+
+            // Execute JS installation script
+            getStreamLogger().logMessage(
+                    String.format("\n  ----------------------------------------------------------------------\n  Task #%d :  JS installation script: %s\n",
+                    getTaskCounter(), jsScript));
+
+            INSTRUCTION_RESULT result = executeJsScript(jsScript);
+
+            if (result==INSTRUCTION_RESULT.FAIL) {
+                log.error("SshJsClientInstaller: Task #{}: JS installation script failed: {}", getTaskCounter(), jsScript);
+                getStreamLogger().logMessage(
+                        String.format("\n  Task #%d :  JS installation script failed: %s\n", getTaskCounter(), jsScript));
+                cntFail++;
+                exitResult = INSTRUCTION_RESULT.FAIL;
+                if (!isContinueOnFail()) {
+                    break;
+                }
+            } else
+            if (result==INSTRUCTION_RESULT.EXIT) {
+                log.info("SshJsClientInstaller: Task #{}: JS installation script processing exits", getTaskCounter());
+                getStreamLogger().logMessage(
+                        String.format("\n  Task #%d :  JS installation script processing exits\n", getTaskCounter()));
+                cntSuccess++;
+                exitResult = INSTRUCTION_RESULT.EXIT;
+                break;
+            } else {
+                log.info("SshJsClientInstaller: Task #{}: JS installation script succeeded: {}", getTaskCounter(), jsScript);
+                getStreamLogger().logMessage(
+                        String.format("\n  Task #%d :  JS installation script succeeded: %s\n", getTaskCounter(), jsScript));
+                cntSuccess++;
+                exitResult = INSTRUCTION_RESULT.SUCCESS;
+            }
+        }
+        log.info("\n  -------------------------------------------------------------------------\n  Task #{} :  JS installation scripts processed: successful={}, failed={}, exit-result={}", getTaskCounter(), cntSuccess, cntFail, exitResult);
+        getStreamLogger().logMessage(
+                String.format("\n  ----------------------------------------------------------------------\n  Task #%d :  JS installation scripts processed: successful=%d, failed=%d, exit-result=%s\n", getTaskCounter(), cntSuccess, cntFail, exitResult));
+        return exitResult;
+    }
+
+    public void printAndLog(Object args) {
+        try {
+            String message;
+            if (args==null) {
+                message = "null";
+            } else
+            if (args.getClass().isArray()) {
+                message = Arrays.stream((Object[]) args)
+                        .map(o -> o == null ? "null" : o.toString())
+                        .collect(Collectors.joining(" "));
+            } else {
+                message = args.toString();
+            }
+            if (!message.endsWith("\n")) message += "\n";
+//            getStreamLogger().getOut().write(String.format(message).getBytes());
+            getStreamLogger().logMessage(message);
+        } catch (IOException e) {
+            log.error("SshJsClientInstaller: printAndLog: ", e);
+        }
+    }
+
+    private INSTRUCTION_RESULT executeJsScript(String jsScript) {
+        try {
+            // Initializing JS engine
+            log.debug("SshJsClientInstaller: Task #{}: Initializing JS engine", getTaskCounter());
+            ScriptEngineManager scriptEngineManager = new ScriptEngineManager();
+            ScriptEngine engine = scriptEngineManager.getEngineByName("nashorn");
+            engine.getContext().getBindings(ScriptContext.GLOBAL_SCOPE).put("installer", this);
+            engine.getContext().getBindings(ScriptContext.GLOBAL_SCOPE).put("log", (Consumer<?>)this::printAndLog);
+
+            log.debug("SshJsClientInstaller: Task #{}: Executing JS script: {}", getTaskCounter(), jsScript);
+            File jsFile = ResourceUtils.getFile(jsScript);
+            log.trace("SshJsClientInstaller: Task #{}: JS script file: {}", getTaskCounter(), jsFile);
+            Object result = engine.eval(new FileReader(jsFile));
+
+            if (result==null) {
+                log.error("SshJsClientInstaller: Task #{}: JS installation script returned NULL: {}", getTaskCounter(), jsScript);
+                return INSTRUCTION_RESULT.FAIL;
+            }
+            if (result instanceof Integer) {
+                int code = (int)result;
+                log.info("SshJsClientInstaller: Task #{}: JS installation script returned: code={}, script: {}", getTaskCounter(), code, jsScript);
+                return code==0 ? INSTRUCTION_RESULT.SUCCESS : INSTRUCTION_RESULT.FAIL;
+            } else {
+                log.error("SshJsClientInstaller: Task #{}: JS installation script returned NON-INTEGER value: {}, script: {}", getTaskCounter(), result, jsScript);
+                return INSTRUCTION_RESULT.FAIL;
+            }
+        } catch (ScriptException | IOException e) {
+            log.error("SshJsClientInstaller: Task #{}: Exception while executing script: {}, Exception: ", getTaskCounter(), jsScript, e);
+            return INSTRUCTION_RESULT.FAIL;
+        }
+    }
+
+    public String getInstallationResult() {
+        return getTask().getNodeRegistryEntry().getPreregistration().get(getProperties().getClientInstallVarName());
+    }
+
+    public void setInstallationResult(boolean success) {
+        getTask().getNodeRegistryEntry().getPreregistration().put(
+                getProperties().getClientInstallVarName(),
+                success ? "INSTALLED" : "ERROR");
+    }
+
+    public void clearInstallationResult() {
+        getTask().getNodeRegistryEntry().getPreregistration().remove(
+                getProperties().getClientInstallVarName());
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/StreamLogger.java

@@ -0,0 +1,166 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.*;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * Logs and formats In/Out/Err streams
+ */
+@Slf4j
+public class StreamLogger {
+    private final FileOutputStream fos;
+    private final PipedOutputStream pos;
+    private final PipedInputStream pis;
+    private final MonitorOutputStream mos;
+
+    private final OutputStream ncInvertedIn;
+    private final InputStream ncIn;
+    private final OutputStream ncOut;
+    private final OutputStream ncErr;
+
+    private String lastLine;
+    private long lastLineTime;
+
+    public StreamLogger(String logFile) throws IOException {
+        this(logFile, "");
+    }
+
+    public StreamLogger(String logFile, String prefix) throws IOException {
+        this.fos = StringUtils.isNotBlank(logFile) ? new FileOutputStream(logFile) : null;
+        this.pos = new PipedOutputStream();
+        this.pis = new PipedInputStream(pos);
+        this.mos = new MonitorOutputStream(this);
+
+        this.ncIn = new LoggerInputStream(pis, prefix+"  IN", toArray(System.out, fos));
+        this.ncInvertedIn = pos;
+        this.ncOut = new LoggerOutputStream(prefix+" OUT", toArray(System.out, mos, fos));
+        this.ncErr = new LoggerOutputStream(prefix+" ERR", toArray(System.err, fos));
+    }
+
+    private OutputStream[] toArray(OutputStream...streams) {
+        return Arrays.stream(streams)
+                .filter(Objects::nonNull)
+                .toArray(OutputStream[]::new);
+    }
+
+    public InputStream getIn() { return ncIn; }
+
+    public OutputStream getInvertedIn() {
+        return ncInvertedIn;
+    }
+
+    public OutputStream getOut() {
+        return ncOut;
+    }
+
+    public OutputStream getErr() {
+        return ncErr;
+    }
+
+    public void close() throws IOException {
+        if (fos!=null) fos.close();
+        pos.close();
+    }
+
+    public void logMessage(String message) throws IOException {
+        if (fos!=null) fos.write(message.getBytes());
+    }
+
+    private void newLine(String line, long timestamp) {
+        lastLine = line;
+        lastLineTime = timestamp;
+    }
+
+    static class LoggerInputStream extends InputStream {
+        private final InputStream in;
+        private final OutputStream[] streams;
+        private final byte[] prefix;
+
+        public LoggerInputStream(InputStream in, String prefix, OutputStream...streams) {
+            this.in = in;
+            this.prefix = (prefix+"< ").getBytes();
+            this.streams = streams;
+        }
+
+        @Override
+        public int read() throws IOException {
+            int b = in.read();
+            writeToStreams(b);
+            return b;
+        }
+
+        private void writeToStreams(int b) throws IOException {
+            for (int i=0; i<streams.length; i++)
+                streams[i].write(b);
+        }
+    }
+
+    static class LoggerOutputStream extends OutputStream {
+        private final OutputStream[] streams;
+        private final byte[] prefix;
+        private boolean newline = true;
+
+        public LoggerOutputStream(String prefix, OutputStream...streams) {
+            this.prefix = (prefix+"> ").getBytes();
+            this.streams = streams;
+        }
+
+        @Override
+        public void write(int b) throws IOException {
+            if (newline) {
+                writeToStreams(prefix);
+                newline = false;
+            }
+            writeToStreams(b);
+            if (b=='\n') newline = true;
+        }
+
+        private void writeToStreams(int b) throws IOException {
+            for (int i=0; i<streams.length; i++)
+                streams[i].write(b);
+        }
+
+        private void writeToStreams(byte[] buff) throws IOException {
+            for (int i=0; i<streams.length; i++)
+                streams[i].write(buff);
+        }
+    }
+
+    @RequiredArgsConstructor
+    static class MonitorOutputStream extends OutputStream {
+        private final StreamLogger streamLogger;
+        private boolean newline = true;
+        private StringBuilder lastLine;
+
+        @Override
+        public void write(int b) throws IOException {
+            if (newline) {
+                this.lastLine = new StringBuilder();
+                newline = false;
+            }
+            lastLine.append(b);
+            if (b=='\n') {
+                newline = true;
+                signalLine();
+            }
+        }
+
+        private void signalLine() {
+            streamLogger.newLine(lastLine.toString(), System.currentTimeMillis());
+        }
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/AbstractInstallationHelper.java

@@ -0,0 +1,286 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.helper;
+
+import com.google.gson.Gson;
+import gr.iccs.imu.ems.baguette.client.install.ClientInstallationProperties;
+import gr.iccs.imu.ems.baguette.client.install.instruction.InstructionsSet;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.util.KeystoreUtil;
+import gr.iccs.imu.ems.util.PasswordUtil;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringSubstitutor;
+import org.apache.tomcat.util.net.SSLHostConfig;
+import org.apache.tomcat.util.net.SSLHostConfigCertificate;
+import org.rauschig.jarchivelib.Archiver;
+import org.rauschig.jarchivelib.ArchiverFactory;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.web.context.WebServerInitializedEvent;
+import org.springframework.boot.web.embedded.tomcat.TomcatWebServer;
+import org.springframework.context.ApplicationListener;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.stereotype.Service;
+
+import java.io.*;
+import java.nio.file.*;
+import java.security.cert.X509Certificate;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Baguette Client installation helper
+ */
+@Slf4j
+@Service
+public abstract class AbstractInstallationHelper implements InitializingBean, ApplicationListener<WebServerInitializedEvent>, InstallationHelper {
+    protected static AbstractInstallationHelper instance = null;
+    protected static List<String> LINUX_OS_FAMILIES;
+    protected static List<String> WINDOWS_OS_FAMILIES;
+
+    @Autowired
+    @Getter @Setter
+    protected ClientInstallationProperties properties;
+    @Autowired
+    protected PasswordUtil passwordUtil;
+
+    protected String archiveBase64;
+    protected boolean isServerSecure;
+    protected String serverCert;
+
+    public synchronized static AbstractInstallationHelper getInstance() {
+        return instance;
+    }
+
+    @Override
+    public void afterPropertiesSet() {
+        log.debug("AbstractInstallationHelper.afterPropertiesSet(): class={}: configuration: {}", getClass().getName(), properties);
+        AbstractInstallationHelper.instance = this;
+        LINUX_OS_FAMILIES = properties.getOsFamilies().get("LINUX");
+        WINDOWS_OS_FAMILIES = properties.getOsFamilies().get("WINDOWS");
+    }
+
+    @Override
+    public void onApplicationEvent(WebServerInitializedEvent event) {
+        log.debug("AbstractInstallationHelper.onApplicationEvent(): event={}", event);
+        TomcatWebServer tomcat = (TomcatWebServer) event.getSource();
+
+        try {
+            initServerCertificateFile(tomcat);
+            initBaguetteClientConfigArchive();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void initServerCertificateFile(TomcatWebServer tomcat) throws Exception {
+        this.isServerSecure = tomcat.getTomcat().getConnector().getSecure();
+        log.debug("AbstractInstallationHelper.initServerCertificate(): Embedded Tomcat is secure: {}", isServerSecure);
+
+        if (isServerSecure) {
+            // If HTTPS is enabled
+            SSLHostConfig[] sslHostConfigArr = tomcat.getTomcat().getConnector().findSslHostConfigs();
+            if (log.isDebugEnabled())
+                log.debug("AbstractInstallationHelper.initServerCertificate(): Tomcat SSL host config array: {}", Arrays.asList(sslHostConfigArr));
+            if (sslHostConfigArr.length!=1)
+                throw new RuntimeException("Embedded Tomcat has zero or more than one SSL host configurations: "+sslHostConfigArr.length);
+
+            // Get certificate entries (in key manager/store) for this SSL Hosting configuration
+            Set<SSLHostConfigCertificate> sslCertificatesSet = sslHostConfigArr[0].getCertificates();
+            log.debug("AbstractInstallationHelper.initServerCertificate(): SSL certificates set: {}", sslCertificatesSet);
+            int n = 0;
+            String serverCert = null;
+            for (SSLHostConfigCertificate sslCertificate : sslCertificatesSet) {
+                // Get entry alias
+                log.debug("AbstractInstallationHelper.initServerCertificate(): SSL certificate[{}]: {}", n, sslCertificate);
+                String keyAlias = sslCertificate.getCertificateKeyAlias();
+                log.debug("AbstractInstallationHelper.initServerCertificate(): SSL certificate[{}]: alias={}", n, keyAlias);
+
+                // Get certificate chain for entry with 'alias'
+                X509Certificate[] chain = sslCertificate.getSslContext().getCertificateChain(keyAlias);
+                StringBuilder sb = new StringBuilder();
+                int m = 0;
+                for (X509Certificate c : chain) {
+                    // Export certificate in PEM format (for each chain item)
+                    String certPem = KeystoreUtil.exportCertificateAsPEM(c);
+                    log.debug("AbstractInstallationHelper.initServerCertificate(): SSL certificate[{}]: {}: \n{}", n, m, certPem);
+                    // Append PEM certificate to 'sb'
+                    sb.append(certPem).append(System.getProperty("line.separator"));
+                    m++;
+                }
+                // The first entry is used as the server certificate
+                if (serverCert==null)
+                    serverCert = sb.toString();
+
+                n++;
+            }
+            this.serverCert = serverCert;
+            log.debug("AbstractInstallationHelper.initServerCertificate(): Server certificate:\n{}", serverCert);
+
+            // Write server certificate to PEM file (server.pem)
+            String certFileName = properties.getServerCertFileAtServer();
+            if (this.serverCert!=null && StringUtils.isNotEmpty(certFileName)) {
+                File certFile = new File(certFileName);
+                Files.writeString(certFile.toPath(), this.serverCert, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
+                if (! certFile.exists())
+                    throw new RuntimeException("Server PEM certificate file not found: "+certFile);
+                log.debug("AbstractInstallationHelper.initServerCertificate(): Server PEM certificate stored in file: {}", certFile);
+                log.info("Server PEM certificate stored in file: {}", certFile);
+            }
+
+        } else {
+            // If HTTPS is disabled
+            if (StringUtils.isNotEmpty(properties.getServerCertFileAtServer())) {
+                File certFile = new File(properties.getServerCertFileAtServer());
+                if (certFile.exists()) {
+                    log.debug("AbstractInstallationHelper.initServerCertificate(): Removing previous server certificate file");
+                    if (!certFile.delete())
+                        throw new RuntimeException("Could not remove previous server certificate file: " + certFile);
+                }
+                this.serverCert = null;
+            }
+        }
+    }
+
+    private void initBaguetteClientConfigArchive() throws IOException {
+        if (StringUtils.isEmpty(properties.getArchiveSourceDir()) || StringUtils.isEmpty(properties.getArchiveFile())) {
+            log.debug("AbstractInstallationHelper: No baguette client configuration archiving has been configured");
+            return;
+        }
+        log.info("AbstractInstallationHelper: Building baguette client configuration archive...");
+
+        // Get archiving settings
+        String configDirName = properties.getArchiveSourceDir();
+        File configDir = new File(configDirName);
+        log.debug("AbstractInstallationHelper: Baguette client configuration directory: {}", configDir);
+        if (!configDir.exists())
+            throw new FileNotFoundException("Baguette client configuration directory not found: " + configDirName);
+
+        String archiveName = properties.getArchiveFile();
+        String archiveDirName = properties.getArchiveDir();
+        File archiveDir = new File(archiveDirName);
+        log.debug("AbstractInstallationHelper: Baguette client configuration archive: {}/{}", archiveDirName, archiveName);
+        if (!archiveDir.exists())
+            throw new FileNotFoundException("Baguette client configuration archive directory not found: " + archiveDirName);
+
+        // Remove previous baguette client configuration archive
+        File archiveFile = new File(archiveDirName, archiveName);
+        if (archiveFile.exists()) {
+            log.debug("AbstractInstallationHelper: Removing previous archive...");
+            if (!archiveFile.delete())
+                throw new RuntimeException("AbstractInstallationHelper: Failed removing previous archive: " + archiveName);
+        }
+
+        // Create baguette client configuration archive
+        Archiver archiver = ArchiverFactory.createArchiver(archiveFile);
+        String tempFileName = "archive_" + System.currentTimeMillis();
+        log.debug("AbstractInstallationHelper: Temp. archive name: {}", tempFileName);
+        archiveFile = archiver.create(tempFileName, archiveDir, configDir);
+        log.debug("AbstractInstallationHelper: Archive generated: {}", archiveFile);
+        if (!archiveFile.getName().equals(archiveName)) {
+            log.debug("AbstractInstallationHelper: Renaming archive to: {}", archiveName);
+            if (!archiveFile.renameTo(archiveFile = new File(archiveDir, archiveName)))
+                throw new RuntimeException("AbstractInstallationHelper: Failed renaming generated archive to: " + archiveName);
+        }
+        log.info("AbstractInstallationHelper: Baguette client configuration archive: {}", archiveFile);
+
+        // Base64 encode archive and cache in memory
+        byte[] archiveBytes = Files.readAllBytes(archiveFile.toPath());
+        this.archiveBase64 = Base64.getEncoder().encodeToString(archiveBytes);
+        log.debug("AbstractInstallationHelper: Archive Base64 encoded: {}", archiveBase64);
+    }
+
+    private String getResourceAsString(String resourcePath) throws IOException {
+        InputStream resource = new FileSystemResource(resourcePath).getInputStream();
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource))) {
+            return reader.lines().collect(Collectors.joining("\n"));
+        }
+    }
+
+    public Optional<List<String>> getInstallationInstructionsForOs(NodeRegistryEntry entry) throws IOException {
+        if (! entry.getBaguetteServer().isServerRunning()) throw new RuntimeException("Baguette Server is not running");
+
+        List<InstructionsSet> instructionsSets = prepareInstallationInstructionsForOs(entry);
+        if (instructionsSets==null) {
+            String nodeOs = entry.getPreregistration().get("operatingSystem");
+            log.warn("AbstractInstallationHelper.getInstallationInstructionsForOs(): ERROR: Unknown node OS: {}: node-map={}", nodeOs, entry.getPreregistration());
+            return Optional.empty();
+        }
+
+        List<String> jsonSets = null;
+        if (!instructionsSets.isEmpty()) {
+            // Convert 'instructionsSet' into json string
+            Gson gson = new Gson();
+            jsonSets = instructionsSets.stream().map(instructionsSet -> gson.toJson(instructionsSet, InstructionsSet.class)).collect(Collectors.toList());
+        }
+        log.trace("AbstractInstallationHelper.getInstallationInstructionsForOs(): JSON instruction sets for node: node-map={}\n{}", entry.getPreregistration(), jsonSets);
+        return Optional.ofNullable(jsonSets);
+    }
+
+    public List<InstructionsSet> prepareInstallationInstructionsForOs(NodeRegistryEntry entry) throws IOException {
+        if (! entry.getBaguetteServer().isServerRunning()) throw new RuntimeException("Baguette Server is not running");
+        log.trace("AbstractInstallationHelper.prepareInstallationInstructionsForOs(): node-map={}", entry.getPreregistration());
+
+        String osFamily = entry.getPreregistration().get("operatingSystem");
+        List<InstructionsSet> instructionsSetList = null;
+        if (LINUX_OS_FAMILIES.contains(osFamily.toUpperCase()))
+            instructionsSetList = prepareInstallationInstructionsForLinux(entry);
+        else if (WINDOWS_OS_FAMILIES.contains(osFamily.toUpperCase()))
+            instructionsSetList = prepareInstallationInstructionsForWin(entry);
+        else
+            log.warn("AbstractInstallationHelper.prepareInstallationInstructionsForOs(): Unsupported OS family: {}", osFamily);
+        return instructionsSetList;
+    }
+
+    protected InstructionsSet _appendCopyInstructions(
+            InstructionsSet instructionsSet,
+            Path p,
+            Path startDir,
+            String copyToClientDir,
+            String clientTmpDir,
+            Map<String,String> valueMap
+    ) throws IOException
+    {
+        String targetFile = StringUtils.substringAfter(p.toUri().toString(), startDir.toUri().toString());
+        if (!targetFile.startsWith("/")) targetFile = "/"+targetFile;
+        targetFile = copyToClientDir + targetFile;
+        String contents = new String(Files.readAllBytes(p));
+        contents = StringSubstitutor.replace(contents, valueMap);
+        String tmpFile = clientTmpDir+"/installEMS_"+System.currentTimeMillis();
+        instructionsSet
+                .appendLog(String.format("Copy file from server to temp to client: %s -> %s -> %s", p.toString(), tmpFile, targetFile));
+        return _appendCopyInstructions(instructionsSet, targetFile, tmpFile, contents, clientTmpDir);
+    }
+
+    protected InstructionsSet _appendCopyInstructions(
+            InstructionsSet instructionsSet,
+            String targetFile,
+            String tmpFile,
+            String contents,
+            String clientTmpDir
+    ) throws IOException
+    {
+        if (StringUtils.isEmpty(tmpFile))
+            tmpFile = clientTmpDir+"/installEMS_"+System.currentTimeMillis();
+        instructionsSet
+                .appendWriteFile(tmpFile, contents, false)
+                .appendExec("sudo mv " + tmpFile + " " + targetFile)
+                .appendExec("sudo chmod u+rw,og-rwx " + targetFile);
+        return instructionsSet;
+    }
+
+    protected String _prepareUrl(String urlTemplate, String baseUrl) {
+        return urlTemplate
+                .replace("%{BASE_URL}%", Optional.ofNullable(baseUrl).orElse(""));
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/InstallationHelper.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.helper;
+
+import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask;
+import gr.iccs.imu.ems.baguette.client.install.instruction.InstructionsSet;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.translate.TranslationContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+
+public interface InstallationHelper {
+    Optional<List<String>> getInstallationInstructionsForOs(NodeRegistryEntry entry) throws IOException;
+
+    List<InstructionsSet> prepareInstallationInstructionsForOs(NodeRegistryEntry entry) throws IOException;
+    List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry);
+    List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException;
+
+    default ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry) throws Exception {
+        return createClientInstallationTask(entry, null);
+    }
+    ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws Exception;
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/InstallationHelperFactory.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.helper;
+
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
+import org.springframework.stereotype.Service;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+
+/**
+ * Installation helper factory
+ */
+@Slf4j
+@Service
+public class InstallationHelperFactory implements InitializingBean {
+    private static InstallationHelperFactory instance;
+
+    public synchronized static InstallationHelperFactory getInstance() { return instance; }
+
+    @Autowired
+    private ApplicationContext applicationContext;
+
+    @Override
+    public void afterPropertiesSet() {
+        InstallationHelperFactory.instance = this;
+    }
+
+    public InstallationHelper createInstallationHelper(NodeRegistryEntry entry) {
+        String nodeType = entry.getPreregistration().get("type");
+        if ("VM".equalsIgnoreCase(nodeType) || "baremetal".equalsIgnoreCase(nodeType)) {
+            return createVmInstallationHelper(entry);
+        }
+        throw new IllegalArgumentException("Unsupported or missing Node type: "+nodeType);
+    }
+
+    public InstallationHelper createInstallationHelperBean(String className, NodeRegistryEntry entry) throws ClassNotFoundException {
+        Class<?> clzz = Class.forName(className);
+        return (InstallationHelper) applicationContext.getBean(clzz);
+    }
+
+    public InstallationHelper createInstallationHelperInstance(String className, Map<String,Object> nodeMap)
+            throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, IllegalAccessException
+    {
+        Class<?> clzz = Class.forName(className);
+        return (InstallationHelper) clzz.getDeclaredMethod("getInstance").invoke(null);
+    }
+
+    private InstallationHelper createVmInstallationHelper(NodeRegistryEntry entry) {
+        return VmInstallationHelper.getInstance();
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/helper/VmInstallationHelper.java

@@ -0,0 +1,321 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.helper;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import gr.iccs.imu.ems.baguette.client.install.ClientInstallationProperties;
+import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask;
+import gr.iccs.imu.ems.baguette.client.install.SshConfig;
+import gr.iccs.imu.ems.baguette.client.install.instruction.Instruction;
+import gr.iccs.imu.ems.baguette.client.install.instruction.InstructionsService;
+import gr.iccs.imu.ems.baguette.client.install.instruction.InstructionsSet;
+import gr.iccs.imu.ems.baguette.server.BaguetteServer;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.translate.TranslationContext;
+import gr.iccs.imu.ems.util.CredentialsMap;
+import gr.iccs.imu.ems.util.NetUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringSubstitutor;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ResourceLoader;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Baguette Client installation helper
+ */
+@Slf4j
+@Service
+public class VmInstallationHelper extends AbstractInstallationHelper {
+    private final static SimpleDateFormat tsW3C = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
+    private final static SimpleDateFormat tsUTC = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");
+    private final static SimpleDateFormat tsFile = new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss.SSS");
+    static {
+        tsW3C.setTimeZone(TimeZone.getDefault());
+        tsUTC.setTimeZone(TimeZone.getTimeZone("UTC"));
+        tsFile.setTimeZone(TimeZone.getDefault());
+    }
+
+    @Autowired
+    private ResourceLoader resourceLoader;
+    @Autowired
+    private ClientInstallationProperties clientInstallationProperties;
+
+    @Override
+    public ClientInstallationTask createClientInstallationTask(NodeRegistryEntry entry, TranslationContext translationContext) throws IOException {
+        Map<String, String> nodeMap = entry.getPreregistration();
+
+        String baseUrl = nodeMap.get("BASE_URL");
+        String clientId = nodeMap.get("CLIENT_ID");
+        String ipSetting = nodeMap.get("IP_SETTING");
+
+        // Extract node identification and type information
+        String nodeId = nodeMap.get("id");
+        String nodeOs = nodeMap.get("operatingSystem");
+        String nodeAddress = nodeMap.get("address");
+        String nodeType = nodeMap.get("type");
+        String nodeName = nodeMap.get("name");
+        String nodeProvider = nodeMap.get("provider");
+
+        if (StringUtils.isBlank(nodeType)) nodeType = "VM";
+
+        if (StringUtils.isBlank(nodeOs)) throw new IllegalArgumentException("Missing OS information for Node");
+        if (StringUtils.isBlank(nodeAddress)) throw new IllegalArgumentException("Missing Address for Node");
+
+        // Extract node SSH information
+        int port = (int) Double.parseDouble(Objects.toString(nodeMap.get("ssh.port"), "22"));
+        if (port<1) port = 22;
+        String username = nodeMap.get("ssh.username");
+        String password = nodeMap.get("ssh.password");
+        String privateKey = nodeMap.get("ssh.key");
+        String fingerprint = nodeMap.get("ssh.fingerprint");
+
+        if (port>65535)
+            throw new IllegalArgumentException("Invalid SSH port for Node: " + port);
+        if (StringUtils.isBlank(username))
+            throw new IllegalArgumentException("Missing SSH username for Node");
+        if (StringUtils.isEmpty(password) && StringUtils.isBlank(privateKey))
+            throw new IllegalArgumentException("Missing SSH password or private key for Node");
+
+        // Get EMS client installation instructions for VM node
+        List<InstructionsSet> instructionsSetList =
+                prepareInstallationInstructionsForOs(entry);
+
+        // Create Installation Task for VM node
+        ClientInstallationTask installationTask = ClientInstallationTask.builder()
+                .id(clientId)
+                .nodeId(nodeId)
+                .name(nodeName)
+                .os(nodeOs)
+                .address(nodeAddress)
+                .ssh(SshConfig.builder()
+                        .host(nodeAddress)
+                        .port(port)
+                        .username(username)
+                        .password(password)
+                        .privateKey(privateKey)
+                        .fingerprint(fingerprint)
+                        .build())
+                .type(nodeType)
+                .provider(nodeProvider)
+                .instructionSets(instructionsSetList)
+                .nodeRegistryEntry(entry)
+                .translationContext(translationContext)
+                .build();
+        log.debug("VmInstallationHelper.createClientInstallationTask(): Created client installation task: {}", installationTask);
+
+        return installationTask;
+    }
+
+    @Override
+    public List<InstructionsSet> prepareInstallationInstructionsForWin(NodeRegistryEntry entry) {
+        log.warn("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
+        throw new IllegalArgumentException("VmInstallationHelper.prepareInstallationInstructionsForWin(): NOT YET IMPLEMENTED");
+    }
+
+    @Override
+    public List<InstructionsSet> prepareInstallationInstructionsForLinux(NodeRegistryEntry entry) throws IOException {
+        Map<String, String> nodeMap = entry.getPreregistration();
+        BaguetteServer baguette = entry.getBaguetteServer();
+
+        String baseUrl = StringUtils.removeEnd(nodeMap.get("BASE_URL"), "/");
+        String clientId = nodeMap.get("CLIENT_ID");
+        String ipSetting = nodeMap.get("IP_SETTING");
+        log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux(): Invoked: base-url={}", baseUrl);
+
+        // Get parameters
+        log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux(): properties: {}", properties);
+        String rootCmd = properties.getRootCmd();
+        String baseDir = properties.getBaseDir();
+        String checkInstallationFile = properties.getCheckInstalledFile();
+
+        String baseDownloadUrl = _prepareUrl(properties.getDownloadUrl(), baseUrl);
+        String apiKey = properties.getApiKey();
+        String installScriptUrl = _prepareUrl(properties.getInstallScriptUrl(), baseUrl);
+        String installScriptPath = properties.getInstallScriptFile();
+
+        String serverCertFile = properties.getServerCertFileAtClient();
+        String clientConfArchive = properties.getClientConfArchiveFile();
+
+        String copyFromServerDir = properties.getCopyFilesFromServerDir();
+        String copyToClientDir = properties.getCopyFilesToClientDir();
+
+        String clientTmpDir = StringUtils.firstNonBlank(properties.getClientTmpDir(), "/tmp");
+
+        // Create additional keys (with NODE_ prefix) for node map values (as aliases to the already existing keys)
+        /*
+        Map<String,String> additionalKeysMap = nodeMap.entrySet().stream()
+                .collect(Collectors.toMap(
+                        e -> e.getKey().startsWith("ssh.")
+                                ? "NODE_SSH_" + e.getKey().substring(4).toUpperCase()
+                                : "NODE_" + e.getKey().toUpperCase(),
+                        Map.Entry::getValue,
+                        (v1, v2) -> {
+                            log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
+                                    k, v1, v2);
+                            return v2;
+                        }
+                ));*/
+        final Map<String,String> additionalKeysMap = new HashMap<>();
+        nodeMap.forEach((k, v) -> {
+            try {
+                k = k.startsWith("ssh.")
+                        ? "NODE_SSH_" + k.substring(4).toUpperCase()
+                        : "NODE_" + k.toUpperCase();
+                if (additionalKeysMap.containsKey(k)) {
+                    log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux(): DUPLICATE KEY FOUND: key={}, old-value={}, new-value={}",
+                            k, additionalKeysMap.get(k), v);
+                }
+                additionalKeysMap.put(k, v);
+            } catch (Exception ex) {
+                log.error("VmInstallationHelper.prepareInstallationInstructionsForLinux(): EXCEPTION in additional keys copy loop: key={}, value={}, additionalKeysMap={}, Exception:\n",
+                        k, v, additionalKeysMap, ex);
+            }
+        });
+        nodeMap.putAll(additionalKeysMap);
+
+        // Load client config. template and prepare configuration
+        nodeMap.put("ROOT_CMD", rootCmd!=null ? rootCmd : "");
+        nodeMap.put("BAGUETTE_CLIENT_ID", clientId);
+        nodeMap.put("BAGUETTE_CLIENT_BASE_DIR", baseDir);
+        nodeMap.put("BAGUETTE_SERVER_ADDRESS", baguette.getConfiguration().getServerAddress());
+        nodeMap.put("BAGUETTE_SERVER_HOSTNAME", NetUtil.getHostname());
+        nodeMap.put("BAGUETTE_SERVER_PORT", ""+baguette.getConfiguration().getServerPort());
+        nodeMap.put("BAGUETTE_SERVER_PUBKEY", baguette.getServerPubkey());
+        nodeMap.put("BAGUETTE_SERVER_PUBKEY_FINGERPRINT", baguette.getServerPubkeyFingerprint());
+        nodeMap.put("BAGUETTE_SERVER_PUBKEY_ALGORITHM", baguette.getServerPubkeyAlgorithm());
+        nodeMap.put("BAGUETTE_SERVER_PUBKEY_FORMAT", baguette.getServerPubkeyFormat());
+        CredentialsMap.Entry<String,String> pair =
+                baguette.getConfiguration().getCredentials().hasPreferredPair()
+                        ? baguette.getConfiguration().getCredentials().getPreferredPair()
+                        : baguette.getConfiguration().getCredentials().entrySet().iterator().next();
+        nodeMap.put("BAGUETTE_SERVER_USERNAME", pair.getKey());
+        nodeMap.put("BAGUETTE_SERVER_PASSWORD", pair.getValue());
+
+        if (StringUtils.isEmpty(ipSetting)) throw new IllegalArgumentException("IP_SETTING must have a value");
+        nodeMap.put("IP_SETTING", ipSetting);
+
+        // Misc. installation property values
+        nodeMap.put("BASE_URL", baseUrl);
+        nodeMap.put("DOWNLOAD_URL", baseDownloadUrl);
+        nodeMap.put("API_KEY", apiKey);
+        nodeMap.put("SERVER_CERT_FILE", serverCertFile);
+        nodeMap.put("REMOTE_TMP_DIR", clientTmpDir);
+
+        Date ts = new Date();
+        nodeMap.put("TIMESTAMP", Long.toString(ts.getTime()));
+        nodeMap.put("TIMESTAMP-W3C", tsW3C.format(ts));
+        nodeMap.put("TIMESTAMP-UTC", tsUTC.format(ts));
+        nodeMap.put("TIMESTAMP-FILE", tsFile.format(ts));
+
+        nodeMap.putAll(clientInstallationProperties.getParameters());
+        nodeMap.put("EMS_PUBLIC_DIR", System.getProperty("PUBLIC_DIR", System.getenv("PUBLIC_DIR")));
+        log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux: value-map: {}", nodeMap);
+
+/*        // Clear EMS server certificate (PEM) file, if not secure
+        if (!isServerSecure) {
+            serverCertFile = "";
+        }
+
+        // Copy files from server to Baguette Client
+        if (StringUtils.isNotEmpty(copyFromServerDir) && StringUtils.isNotEmpty(copyToClientDir)) {
+            Path startDir = Paths.get(copyFromServerDir).toAbsolutePath();
+            try (Stream<Path> stream = Files.walk(startDir, Integer.MAX_VALUE)) {
+                List<Path> paths = stream
+                        .filter(Files::isRegularFile)
+                        .map(Path::toAbsolutePath)
+                        .sorted()
+                        .collect(Collectors.toList());
+                for (Path p : paths) {
+                    _appendCopyInstructions(instructionSets, p, startDir, copyToClientDir, clientTmpDir, valueMap);
+                }
+            }
+        }*/
+
+        List<InstructionsSet> instructionsSetList = new ArrayList<>();
+
+        try {
+            // Read installation instructions from JSON file
+            List<String> instructionSetFileList = null;
+            if (nodeMap.containsKey("instruction-files")) {
+                instructionSetFileList = Arrays.stream(nodeMap.getOrDefault("instruction-files", "").split(","))
+                        .filter(StringUtils::isNotBlank)
+                        .map(String::trim)
+                        .collect(Collectors.toList());
+                if (instructionSetFileList.isEmpty())
+                    log.warn("VmInstallationHelper.prepareInstallationInstructionsForLinux: Context map contains 'instruction-files' entry with no contents");
+            } else {
+                instructionSetFileList = properties.getInstructions().get("LINUX");
+            }
+            for (String instructionSetFile : instructionSetFileList) {
+                // Load instructions set from file
+                log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux: Installation instructions file for LINUX: {}", instructionSetFile);
+                InstructionsSet instructionsSet = InstructionsService.getInstance().loadInstructionsFile(instructionSetFile);
+                log.debug("VmInstallationHelper.prepareInstallationInstructionsForLinux: Instructions set loaded from file: {}\n{}", instructionSetFile, instructionsSet);
+
+                // Pretty print instructionsSet JSON
+                if (log.isTraceEnabled()) {
+                    Gson gson = new GsonBuilder().setPrettyPrinting().create();
+                    StringWriter stringWriter = new StringWriter();
+                    try (PrintWriter writer = new PrintWriter(stringWriter)) {
+                        gson.toJson(instructionsSet, writer);
+                    }
+                    log.trace("VmInstallationHelper.prepareInstallationInstructionsForLinux: Installation instructions for LINUX: json:\n{}", stringWriter);
+                }
+
+                instructionsSetList.add(instructionsSet);
+            }
+
+            return instructionsSetList;
+        } catch (Exception ex) {
+            log.error("VmInstallationHelper.prepareInstallationInstructionsForLinux: Exception while reading Installation instructions for LINUX: ", ex);
+            throw ex;
+        }
+    }
+
+    private InstructionsSet _appendCopyInstructions(
+            InstructionsSet instructionsSet,
+            Path path,
+            Path localBaseDir,
+            String remoteTargetDir,
+            Map<String,String> valueMap
+    ) throws IOException
+    {
+        String targetFile = StringUtils.substringAfter(path.toUri().toString(), localBaseDir.toUri().toString());
+        if (!targetFile.startsWith("/")) targetFile = "/"+targetFile;
+        targetFile = remoteTargetDir + targetFile;
+        String contents = new String(Files.readAllBytes(path));
+        contents = StringSubstitutor.replace(contents, valueMap);
+        String description = String.format("Copy file from server to temp to client: %s -> %s", path.toString(), targetFile);
+        return _appendCopyInstructions(instructionsSet, targetFile, description, contents);
+    }
+
+    private InstructionsSet _appendCopyInstructions(
+            InstructionsSet instructionsSet,
+            String targetFile,
+            String description,
+            String contents)
+    {
+        instructionsSet
+                .appendInstruction(Instruction.createWriteFile(targetFile, contents, false).description(description))
+                .appendExec("sudo chmod u+rw,og-rwx " + targetFile);
+        return instructionsSet;
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/instruction/AbstractInstructionsBase.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.instruction;
+
+import lombok.Data;
+
+@Data
+public abstract class AbstractInstructionsBase {
+    private String condition;
+    private boolean stopOnConditionFail;
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/instruction/INSTRUCTION_RESULT.java

@@ -0,0 +1,12 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.instruction;
+
+public enum INSTRUCTION_RESULT { SUCCESS, FAIL, EXIT }

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/instruction/INSTRUCTION_TYPE.java

@@ -0,0 +1,15 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.instruction;
+
+public enum INSTRUCTION_TYPE {
+    LOG, CHECK, CMD, SHELL, FILE, COPY, UPLOAD, DOWNLOAD,
+    SET_VARS, UNSET_VARS, PRINT_VARS, EXIT, EXIT_SET
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/instruction/Instruction.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.instruction;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.*;
+import lombok.experimental.Accessors;
+
+import javax.validation.constraints.NotNull;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+@Data
+@Accessors(chain = true, fluent = true)
+@Builder(toBuilder = true)
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode(callSuper = true)
+@Getter(onMethod = @__(@JsonProperty))
+public class Instruction extends AbstractInstructionsBase {
+    private INSTRUCTION_TYPE taskType;
+    private String description;
+    private String message;
+    private String command;
+    private String fileName;
+    private String localFileName;
+    private String contents;
+    private boolean executable;
+    private int exitCode;
+    private boolean match;
+    private long executionTimeout;
+    private int retries;
+
+    private Map<String, Pattern> patterns;
+    private Map<String, String> variables;
+
+    // Fluent API addition
+    public Instruction pattern(String varName, Pattern pattern) {
+        this.patterns.put(varName, pattern);
+        return this;
+    }
+
+    // Creators API
+    public static Instruction createLog(@NotNull String message) {
+        return Instruction.builder()
+                .taskType(INSTRUCTION_TYPE.LOG)
+                .command(message)
+                .build();
+    }
+
+    public static Instruction createShellCommand(@NotNull String command) {
+        return Instruction.builder()
+                .taskType(INSTRUCTION_TYPE.CMD)
+                .command(command)
+                .build();
+    }
+
+    public static Instruction createWriteFile(@NotNull String file, String contents, boolean executable) {
+        return Instruction.builder()
+                .taskType(INSTRUCTION_TYPE.FILE)
+                .fileName(file)
+                .contents(contents==null ? "" : contents)
+                .executable(executable)
+                .build();
+    }
+
+    public static Instruction createUploadFile(@NotNull String localFile, @NotNull String remoteFile) {
+        return Instruction.builder()
+                .taskType(INSTRUCTION_TYPE.COPY)
+                .fileName(remoteFile)
+                .localFileName(localFile)
+                .build();
+   }
+
+    public static Instruction createDownloadFile(@NotNull String remoteFile, @NotNull String localFile) {
+        return Instruction.builder()
+                .taskType(INSTRUCTION_TYPE.DOWNLOAD)
+                .fileName(remoteFile)
+                .localFileName(localFile)
+                .build();
+   }
+
+    public static Instruction createCheck(@NotNull String command, @NotNull int exitCode, boolean match, String message) {
+        return Instruction.builder()
+                .taskType(INSTRUCTION_TYPE.CHECK)
+                .command(command)
+                .exitCode(exitCode)
+                .match(match)
+                .contents(message)
+                .build();
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/instruction/InstructionsService.java

@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.instruction;
+
+import com.fasterxml.jackson.core.json.JsonReadFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringSubstitutor;
+import org.springframework.context.EnvironmentAware;
+import org.springframework.core.env.Environment;
+import org.springframework.core.io.ResourceLoader;
+import org.springframework.expression.ExpressionParser;
+import org.springframework.expression.spel.standard.SpelExpressionParser;
+import org.springframework.stereotype.Service;
+import org.springframework.util.FileCopyUtils;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class InstructionsService implements EnvironmentAware {
+    private Environment environment;
+    private final ResourceLoader resourceLoader;
+    private static InstructionsService INSTANCE;
+
+    public static InstructionsService getInstance() {
+        if (INSTANCE==null) throw new IllegalStateException("InstructionsService singleton instance has not yet been initialized");
+        return INSTANCE;
+    }
+
+    @Override
+    public void setEnvironment(Environment environment) {
+        this.environment = environment;
+        INSTANCE = this;
+    }
+
+    public boolean checkCondition(@NonNull AbstractInstructionsBase i, Map<String,String> valueMap) {
+        log.trace("InstructionsService: checkCondition: condition={}, value-map={}", i.getCondition(), valueMap);
+        String condition = i.getCondition();
+        if (StringUtils.isBlank(condition)) return true;
+        String conditionResolved = processPlaceholders(condition, valueMap);
+        log.trace("InstructionsService: checkCondition: Expression after placeholder resolution: {}", conditionResolved);
+        final ExpressionParser parser = new SpelExpressionParser();
+        Object result = parser.parseExpression(conditionResolved).getValue();
+        log.trace("InstructionsService: checkCondition: Expression result: {}", result);
+        if (result==null)
+            throw new IllegalArgumentException("Condition evaluation returned null: " + condition);
+        if (result instanceof Boolean)
+            return (Boolean)result;
+        throw new IllegalArgumentException("Condition evaluation returned a non-boolean value: " + result + ", condition:  " + condition+", resolved condition: "+ conditionResolved);
+    }
+
+    public Instruction resolvePlaceholders(Instruction instruction, Map<String,String> valueMap) {
+        return instruction.toBuilder()
+                .description(processPlaceholders(instruction.description(), valueMap))
+                .message(processPlaceholders(instruction.message(), valueMap))
+                .command(processPlaceholders(instruction.command(), valueMap))
+                .fileName(processPlaceholders(instruction.fileName(), valueMap))
+                .localFileName(processPlaceholders(instruction.localFileName(), valueMap))
+                .contents(processPlaceholders(instruction.contents(), valueMap))
+                .build();
+    }
+
+    public String processPlaceholders(String s, Map<String,String> valueMap) {
+        if (StringUtils.isBlank(s)) return s;
+        s = StringSubstitutor.replace(s, valueMap);
+        s = environment.resolvePlaceholders(s);
+        //s = environment.resolveRequiredPlaceholders(s);
+        s = s.replace('\\', '/');
+        return s;
+    }
+
+    public InstructionsSet loadInstructionsFile(@NonNull String fileName) throws IOException {
+        if (StringUtils.isBlank(fileName))
+            throw new IllegalArgumentException("File name is blank");
+        fileName = fileName.trim();
+
+        // Get file type from file extension
+        String ext = null;
+        int i = fileName.lastIndexOf('.');
+        if (i > 0) {
+            ext = fileName.substring(i+1);
+            if (ext.contains("/") || ext.contains("\\")) ext = null;
+        }
+        if (ext==null)
+            throw new IllegalArgumentException("Unknown file type: "+fileName);
+
+        // Process instructions file based on its type
+        try {
+            if ("json".equalsIgnoreCase(ext)) {
+                // Load instructions set from JSON file
+                return _loadFromJsonFile(fileName);
+            } else if ("yml".equalsIgnoreCase(ext) || "yaml".equalsIgnoreCase(ext)) {
+                // Load instructions set from YAML file
+                return _loadFromYamlFile(fileName);
+            } else if ("js".equalsIgnoreCase(ext)) {
+                // Just return an instruction set with the file name set
+                InstructionsSet is = new InstructionsSet();
+                is.setFileName(fileName);
+                return is;
+            }
+        } catch (IOException e) {
+            log.error("Exception thrown while processing instructions set file: {}", fileName);
+            throw new IOException(fileName+": "+e.getMessage(), e);
+        }
+        throw new IllegalArgumentException("Unsupported file type: "+fileName);
+    }
+
+    private InstructionsSet _loadFromJsonFile(String jsonFile) throws IOException {
+        log.debug("InstructionsService: Loading instructions from JSON file: {}", jsonFile);
+        byte[] bdata = FileCopyUtils.copyToByteArray(resourceLoader.getResource(jsonFile).getInputStream());
+        String jsonStr = new String(bdata, StandardCharsets.UTF_8);
+        log.trace("InstructionsService: JSON instructions file contents: \n{}", jsonStr);
+
+        // Create InstructionsSet object from JSON
+        ObjectMapper mapper = new ObjectMapper();
+        InstructionsSet instructionsSet = mapper.readerFor(InstructionsSet.class)
+                .with(JsonReadFeature.ALLOW_JAVA_COMMENTS)
+                .readValue(jsonStr);
+        instructionsSet.setFileName(jsonFile);
+        log.trace("InstructionsService: Installation instructions loaded from JSON file: {}\n{}", jsonFile, instructionsSet);
+
+        return instructionsSet;
+    }
+
+    private InstructionsSet _loadFromYamlFile(String yamlFile) throws IOException {
+        log.debug("InstructionsService: Loading instructions from YAML file: {}", yamlFile);
+        byte[] bdata = FileCopyUtils.copyToByteArray(resourceLoader.getResource(yamlFile).getInputStream());
+        String yamlStr = new String(bdata, StandardCharsets.UTF_8);
+        log.trace("InstructionsService: YAML instructions file contents: \n{}", yamlStr);
+
+        ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+        InstructionsSet instructionsSet =
+                mapper.readValue(yamlStr, InstructionsSet.class);
+        instructionsSet.setFileName(yamlFile);
+        log.trace("InstructionsService: Installation instructions loaded from YAML file: {}\n{}", yamlFile, instructionsSet);
+
+        return instructionsSet;
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/instruction/InstructionsSet.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.instruction;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
+import java.util.*;
+
+@Data
+@NoArgsConstructor
+@EqualsAndHashCode(callSuper = true)
+public class InstructionsSet extends AbstractInstructionsBase {
+    private String os;
+    private String description;
+    private String fileName;
+    private List<Instruction> instructions = new ArrayList<>();
+
+    public List<Instruction> getInstructions() {
+        return Collections.unmodifiableList(instructions);
+    }
+
+    public void setInstructions(List<Instruction> ni) {
+        instructions = new ArrayList<>(ni);
+    }
+
+    public InstructionsSet appendInstruction(Instruction i) {
+        instructions.add(i);
+        return this;
+    }
+
+    public InstructionsSet appendLog(String message) {
+        instructions.add(Instruction.createLog(message));
+        return this;
+    }
+
+    public InstructionsSet appendExec(String command) {
+        instructions.add(Instruction.createShellCommand(command));
+        return this;
+    }
+
+    public InstructionsSet appendWriteFile(String file, String contents, boolean executable) {
+        instructions.add(Instruction.createWriteFile(file, contents, executable));
+        return this;
+    }
+
+    public InstructionsSet appendUploadFile(String localFile, String remoteFile) {
+        instructions.add(Instruction.createUploadFile(localFile, remoteFile));
+        return this;
+    }
+
+    public InstructionsSet appendDownloadFile(String remoteFile, String localFile) {
+        instructions.add(Instruction.createDownloadFile(remoteFile, localFile));
+        return this;
+    }
+
+    public InstructionsSet appendCheck(String command, int exitCode, boolean match, String message) {
+        instructions.add(Instruction.createCheck(command, exitCode, match, message));
+        return this;
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/plugin/AllowedTopicsProcessorPlugin.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.plugin;
+
+import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask;
+import gr.iccs.imu.ems.baguette.client.install.InstallationContextProcessorPlugin;
+import gr.iccs.imu.ems.translate.model.Monitor;
+import gr.iccs.imu.ems.util.EmsConstant;
+import gr.iccs.imu.ems.util.StrUtil;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+
+/**
+ * Installation context processor plugin for generating 'allowed-topics' setting
+ * used in baguette-client[.yml/.properties] config. file.
+ * It set the 'COLLECTOR_ALLOWED_TOPICS' variable in pre-registration context.
+ */
+@Slf4j
+@Data
+@Service
+public class AllowedTopicsProcessorPlugin implements InstallationContextProcessorPlugin {
+    @Override
+    public void processBeforeInstallation(ClientInstallationTask task, long taskCounter) {
+        log.debug("AllowedTopicsProcessorPlugin: Task #{}: processBeforeInstallation: BEGIN", taskCounter);
+        log.trace("AllowedTopicsProcessorPlugin: Task #{}: processBeforeInstallation: BEGIN: task={}", taskCounter, task);
+
+        StringBuilder sbAllowedTopics = new StringBuilder();
+        Set<String> addedTopicsSet = new HashSet<>();
+
+        boolean first = true;
+        for (Monitor monitor : task.getTranslationContext().getMON()) {
+            try {
+                log.trace("AllowedTopicsProcessorPlugin: Task #{}: Processing monitor: {}", taskCounter, monitor);
+
+                String metricName = monitor.getMetric();
+                if (!addedTopicsSet.contains(metricName)) {
+                    if (first) first = false;
+                    else sbAllowedTopics.append(", ");
+
+                    sbAllowedTopics.append(metricName);
+                    addedTopicsSet.add(metricName);
+                }
+
+                // Get sensor configuration (as a list of KeyValuePair's)
+                Map<String,String> sensorConfig = null;
+                if (monitor.getSensor().isPullSensor()) {
+                    // Pull Sensor
+                    sensorConfig = monitor.getSensor().pullSensor().getConfiguration();
+                } else {
+                    // Push Sensor
+                    sensorConfig = monitor.getSensor().pushSensor().getAdditionalProperties();
+                }
+
+                // Process Destination aliases, if specified in configuration
+                if (sensorConfig!=null) {
+                    String k = sensorConfig.keySet().stream()
+                            .filter(key -> StrUtil.compareNormalized(key, EmsConstant.COLLECTOR_DESTINATION_ALIASES))
+                            .findAny().orElse(null);
+                    String aliases = (k!=null) ? sensorConfig.get(k) : null;
+
+                    if (StringUtils.isNotBlank(aliases)) {
+                        for (String alias : aliases.trim().split(EmsConstant.COLLECTOR_DESTINATION_ALIASES_DELIMITERS)) {
+                            if (!(alias=alias.trim()).isEmpty()) {
+                                if (!alias.equals(metricName)) {
+                                    sbAllowedTopics.append(", ");
+                                    sbAllowedTopics.append(alias).append(":").append(metricName);
+                                }
+                            }
+                        }
+                    }
+                }
+
+                log.trace("AllowedTopicsProcessorPlugin: Task #{}: MONITOR: metric={}, allowed-topics={}",
+                        taskCounter, metricName, sbAllowedTopics);
+
+            } catch (Exception e) {
+                log.error("AllowedTopicsProcessorPlugin: Task #{}: EXCEPTION while processing monitor. Skipping it: {}\n",
+                        taskCounter, monitor, e);
+            }
+        }
+
+        String allowedTopics = sbAllowedTopics.toString();
+        log.debug("AllowedTopicsProcessorPlugin: Task #{}: Allowed-Topics configuration for collectors: \n{}", taskCounter, allowedTopics);
+
+        task.getNodeRegistryEntry().getPreregistration().put(EmsConstant.COLLECTOR_ALLOWED_TOPICS_VAR, allowedTopics);
+        log.debug("AllowedTopicsProcessorPlugin: Task #{}: processBeforeInstallation: END", taskCounter);
+    }
+
+    @Override
+    public void processAfterInstallation(ClientInstallationTask task, long taskCounter, boolean success) {
+        log.debug("AllowedTopicsProcessorPlugin: Task #{}: processAfterInstallation: success={}", taskCounter, success);
+        log.trace("AllowedTopicsProcessorPlugin: Task #{}: processAfterInstallation: success={}, task={}", taskCounter, success, task);
+    }
+
+    @Override
+    public void start() {
+        log.debug("AllowedTopicsProcessorPlugin: start()");
+    }
+
+    @Override
+    public void stop() {
+        log.debug("AllowedTopicsProcessorPlugin: stop()");
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/install/plugin/PrometheusProcessorPlugin.java

@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.install.plugin;
+
+import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask;
+import gr.iccs.imu.ems.baguette.client.install.InstallationContextProcessorPlugin;
+import gr.iccs.imu.ems.translate.model.Interval;
+import gr.iccs.imu.ems.translate.model.Monitor;
+import gr.iccs.imu.ems.util.StrUtil;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+
+import java.util.Date;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Installation context processor plugin for generating Netdata configuration for collecting metrics from prometheus exporters
+ */
+@Slf4j
+@Data
+@Service
+public class PrometheusProcessorPlugin implements InstallationContextProcessorPlugin {
+    public final static String SENSOR_TYPE_KEY = "pull.sensor.type";
+    public final static String SENSOR_TYPE_VALUE = "prometheus";
+    public final static String NETDATA_PROMETHEUS_JOB_NAME = "pull.prometheus.job.name";
+    public final static String NETDATA_PROMETHEUS_ENDPOINT = "pull.prometheus.endpoint";
+    public final static String NETDATA_PROMETHEUS_AUTODETECTION = "pull.prometheus.autodetection";
+    public final static String NETDATA_PROMETHEUS_PRIORITY = "pull.prometheus.priority";
+    public final static String NETDATA_PROMETHEUS_CONFIGURATION_VAR = "NETDATA_PROMETHEUS_CONF";
+    public final static long DEFAULT_PRIORITY = 70000;
+
+    @Override
+    public void processBeforeInstallation(ClientInstallationTask task, long taskCounter) {
+        log.debug("PrometheusProcessorPlugin: Task #{}: processBeforeInstallation: BEGIN", taskCounter);
+        log.trace("PrometheusProcessorPlugin: Task #{}: processBeforeInstallation: BEGIN: task={}", taskCounter, task);
+
+        StringBuilder prometheusConf = new StringBuilder("# Generated on: ").append(new Date()).append("\n\n");
+        int headerLength = prometheusConf.length();
+
+        long minCollectionInterval = Long.MAX_VALUE;
+        long minAutodetectionInterval = Long.MAX_VALUE;
+        long minPriority = DEFAULT_PRIORITY;
+        boolean found = false;
+
+        prometheusConf.append("\njobs:\n");
+        for (Monitor monitor : task.getTranslationContext().getMON()) {
+            try {
+                log.trace("PrometheusProcessorPlugin: Task #{}: Processing monitor: {}", taskCounter, monitor);
+                String componentName = monitor.getComponent();
+                String metricName = monitor.getMetric();
+
+                log.trace("PrometheusProcessorPlugin: Task #{}: MONITOR: component={}, metric={}", taskCounter, componentName, metricName);
+                if (monitor.getSensor().isPullSensor()) {
+                    if (monitor.getSensor().pullSensor().getConfiguration()!=null) {
+                        Map<String, String> config = monitor.getSensor().pullSensor().getConfiguration();
+                        log.trace("PrometheusProcessorPlugin: Task #{}: MONITOR with PULL SENSOR: config: {}", taskCounter, config);
+
+                        // Get Prometheus related settings
+                        String sensorType = StrUtil.getWithNormalized(config, SENSOR_TYPE_KEY, SENSOR_TYPE_VALUE);
+                        String prometheusJobName = StrUtil.getWithNormalized(config, NETDATA_PROMETHEUS_JOB_NAME);
+                        String prometheusEndpoint = StrUtil.getWithNormalized(config, NETDATA_PROMETHEUS_ENDPOINT);
+                        log.trace("PrometheusProcessorPlugin: Task #{}: Prometheus Job settings: type={}, name={}, endpoint={}",
+                                taskCounter, sensorType, prometheusJobName, prometheusEndpoint);
+                        if (SENSOR_TYPE_VALUE.equals(sensorType)) {
+                            if (StringUtils.isNotBlank(prometheusJobName) && StringUtils.isNotBlank(prometheusEndpoint)) {
+                                prometheusConf.append("  - name: '").append(prometheusJobName).append("'\n");
+                                prometheusConf.append("    url: '").append(prometheusEndpoint).append("'\n");
+                                log.trace("PrometheusProcessorPlugin: Task #{}: Extracted Prometheus config: metricName={}, endpoint={}",
+                                        taskCounter, prometheusJobName, prometheusEndpoint);
+                                found = true;
+
+                                // Get monitor interval
+                                Interval interval = monitor.getSensor().pullSensor().getInterval();
+                                if (interval != null) {
+                                    int period = interval.getPeriod();
+                                    TimeUnit unit = TimeUnit.SECONDS;
+                                    if (interval.getUnit() != null) {
+                                        unit = TimeUnit.valueOf( interval.getUnit().name() );
+                                    }
+                                    long periodInSeconds = TimeUnit.SECONDS.convert(period, unit);
+                                    if (periodInSeconds > 0)
+                                        minCollectionInterval = Math.min(minCollectionInterval, periodInSeconds);
+                                }
+
+                                // Get autodetection interval
+                                String autodetectionStr = StrUtil.getWithNormalized(config, NETDATA_PROMETHEUS_AUTODETECTION);
+                                int autodetectionInSeconds = StrUtil.strToInt(autodetectionStr, 0, i -> i >= 0, false, null);
+                                if (autodetectionInSeconds > 0)
+                                    minAutodetectionInterval = Math.min(minAutodetectionInterval, autodetectionInSeconds);
+
+                                // Get priority
+                                String priorityStr = StrUtil.getWithNormalized(config, NETDATA_PROMETHEUS_PRIORITY);
+                                int priority = StrUtil.strToInt(priorityStr, (int)DEFAULT_PRIORITY, i -> i >= 0, false, null);
+                                if (priority >= 0)
+                                    minPriority = Math.min(minPriority, priority);
+                            }
+                        } else {
+                            log.debug("PrometheusProcessorPlugin: Task #{}: Sensor type is not Prometheus: {}", taskCounter, sensorType);
+                        }
+                    }
+                }
+
+            } catch (Exception e) {
+                log.error("PrometheusProcessorPlugin: Task #{}: EXCEPTION while processing monitor. Skipping it: {}\n", taskCounter, monitor, e);
+            }
+        }
+        log.debug("PrometheusProcessorPlugin: Task #{}: Netdata Prometheus configuration: \n{}", taskCounter, prometheusConf);
+        log.debug("PrometheusProcessorPlugin: Task #{}: Netdata Prometheus: found={}, collection-interval={}, autodetection={}, priority={}",
+                taskCounter, found, minCollectionInterval, minAutodetectionInterval, minPriority);
+
+        if (!found) {
+            task.getNodeRegistryEntry().getPreregistration().put(NETDATA_PROMETHEUS_CONFIGURATION_VAR, "");
+            log.debug("PrometheusProcessorPlugin: Task #{}: processBeforeInstallation: END: no prometheus.conf update", taskCounter);
+        } else
+        {
+            if (minCollectionInterval < Long.MAX_VALUE)
+                prometheusConf.insert(headerLength, "update_every: " + minCollectionInterval + "\n");
+            if (minAutodetectionInterval < Long.MAX_VALUE)
+                prometheusConf.insert(headerLength, "autodetection_retry: " + minAutodetectionInterval + "\n");
+            if (minPriority != DEFAULT_PRIORITY)
+                prometheusConf.insert(headerLength, "priority: " + minPriority + "\n");
+
+            task.getNodeRegistryEntry().getPreregistration().put(NETDATA_PROMETHEUS_CONFIGURATION_VAR, prometheusConf.toString());
+            log.debug("PrometheusProcessorPlugin: Task #{}: processBeforeInstallation: END", taskCounter);
+        }
+    }
+
+    @Override
+    public void processAfterInstallation(ClientInstallationTask task, long taskCounter, boolean success) {
+        log.debug("PrometheusProcessorPlugin: Task #{}: processAfterInstallation: success={}", taskCounter, success);
+        log.trace("PrometheusProcessorPlugin: Task #{}: processAfterInstallation: success={}, task={}", taskCounter, success, task);
+    }
+
+    @Override
+    public void start() {
+        log.debug("PrometheusProcessorPlugin: start()");
+    }
+
+    @Override
+    public void stop() {
+        log.debug("PrometheusProcessorPlugin: stop()");
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/selfhealing/ClientRecoveryPlugin.java

@@ -0,0 +1,182 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.selfhealing;
+
+import gr.iccs.imu.ems.baguette.client.install.ClientInstallationProperties;
+import gr.iccs.imu.ems.baguette.client.install.ClientInstallationTask;
+import gr.iccs.imu.ems.baguette.client.install.SshClientInstaller;
+import gr.iccs.imu.ems.baguette.client.install.helper.InstallationHelperFactory;
+import gr.iccs.imu.ems.baguette.server.BaguetteServer;
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.NodeRegistry;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.common.selfhealing.SelfHealingManager;
+import gr.iccs.imu.ems.util.EmsConstant;
+import gr.iccs.imu.ems.util.EventBus;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.concurrent.ScheduledFuture;
+
+@Slf4j
+@Service
+@ConditionalOnProperty(name = "enabled", prefix = EmsConstant.EMS_PROPERTIES_PREFIX + "self.healing", havingValue = "true", matchIfMissing = true)
+@RequiredArgsConstructor
+public class ClientRecoveryPlugin implements InitializingBean, EventBus.EventConsumer<String,Object,Object> {
+    private final EventBus<String,Object,Object> eventBus;
+    private final NodeRegistry nodeRegistry;
+    private final TaskScheduler taskScheduler;
+    private final ClientInstallationProperties clientInstallationProperties;
+    private final ServerSelfHealingProperties selfHealingProperties;
+    private final BaguetteServer baguetteServer;
+
+    private final HashMap<NodeRegistryEntry, ScheduledFuture<?>> pendingTasks = new HashMap<>();
+
+    private long clientRecoveryDelay;
+    private String recoveryInstructionsFile;
+
+    private final static String CLIENT_EXIT_TOPIC = "BAGUETTE_SERVER_CLIENT_EXITED";
+    private final static String CLIENT_REGISTERED_TOPIC = "BAGUETTE_SERVER_CLIENT_REGISTERED";
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        clientRecoveryDelay = selfHealingProperties.getRecovery().getDelay();
+        recoveryInstructionsFile = selfHealingProperties.getRecovery().getFile().getOrDefault("baguette", "");
+        log.debug("ClientRecoveryPlugin: recovery-delay={}, recovery-instructions-file (for baguette)={}", clientRecoveryDelay, recoveryInstructionsFile);
+
+        eventBus.subscribe(CLIENT_EXIT_TOPIC, this);
+        log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_EXITED events");
+        eventBus.subscribe(CLIENT_REGISTERED_TOPIC, this);
+        log.debug("ClientRecoveryPlugin: Subscribed for BAGUETTE_SERVER_CLIENT_REGISTERED events");
+
+        log.trace("ClientRecoveryPlugin: clientInstallationProperties: {}", clientInstallationProperties);
+        log.trace("ClientRecoveryPlugin: baguetteServer: {}", baguetteServer);
+
+        log.debug("ClientRecoveryPlugin: Recovery Delay: {}", clientRecoveryDelay);
+        log.debug("ClientRecoveryPlugin: Recovery Instructions File: {}", recoveryInstructionsFile);
+    }
+
+    @Override
+    public void onMessage(String topic, Object message, Object sender) {
+        log.debug("ClientRecoveryPlugin: onMessage(): BEGIN: topic={}, message={}, sender={}", topic, message, sender);
+
+        // Check if Self-Healing is enabled
+        if (! baguetteServer.getSelfHealingManager().isEnabled()) {
+            log.debug("ClientRecoveryPlugin: onMessage(): Self-Healing manager is disabled: message={}, sender={}", message, sender);
+            return;
+        }
+
+        // Only process messages of ClientShellCommand type are accepted (sent by CSC instances)
+        if (! (message instanceof ClientShellCommand)) {
+            log.warn("ClientRecoveryPlugin: onMessage(): Message is not a {} object. Will ignore it.", ClientShellCommand.class.getSimpleName());
+            return;
+        }
+
+        // Get NodeRegistryEntry from ClientShellCommand passed with event
+        ClientShellCommand csc = (ClientShellCommand)message;
+        String clientId = csc.getId();
+        String address = csc.getClientIpAddress();
+        log.debug("ClientRecoveryPlugin: onMessage(): client-id={}, client-address={}", clientId, address);
+
+        NodeRegistryEntry nodeInfo = csc.getNodeRegistryEntry();    //or = nodeRegistry.getNodeByAddress(address);
+        log.debug("ClientRecoveryPlugin: onMessage(): client-node-info={}", nodeInfo);
+        log.trace("ClientRecoveryPlugin: onMessage(): node-registry.node-addresses={}", nodeRegistry.getNodeAddresses());
+        log.trace("ClientRecoveryPlugin: onMessage(): node-registry.nodes={}", nodeRegistry.getNodes());
+
+        // Check if node is monitored by Self-Healing manager
+        if (! baguetteServer.getSelfHealingManager().isMonitored(nodeInfo)) {
+            log.warn("ClientRecoveryPlugin: processExitEvent(): Node is not monitored by Self-Healing manager: client-id={}, client-address={}", clientId, address);
+            return;
+        }
+
+        // Process event
+        if (CLIENT_EXIT_TOPIC.equals(topic)) {
+            log.debug("ClientRecoveryPlugin: onMessage(): CLIENT EXITED: message={}", message);
+            processExitEvent(nodeInfo);
+        }
+        if (CLIENT_REGISTERED_TOPIC.equals(topic)) {
+            log.debug("ClientRecoveryPlugin: onMessage(): CLIENT REGISTERED_TOPIC: message={}", message);
+            processRegisteredEvent(nodeInfo);
+        }
+    }
+
+    private void processExitEvent(NodeRegistryEntry nodeInfo) {
+        log.debug("ClientRecoveryPlugin: processExitEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
+
+        // Set node state to DOWN
+        baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.DOWN);
+
+        // Schedule a recovery task for node
+        ScheduledFuture<?> future = taskScheduler.schedule(() -> {
+            try {
+                // Set node state to RECOVERING
+                baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.RECOVERING);
+                // Run recovery task
+                runClientRecovery(nodeInfo);
+            } catch (Exception e) {
+                log.error("ClientRecoveryPlugin: processExitEvent(): EXCEPTION: while recovering node: node-info={} -- Exception: ", nodeInfo, e);
+            }
+        }, Instant.now().plusMillis(clientRecoveryDelay));
+
+        // Register the recovery task's future in pending list
+        ScheduledFuture<?> old = pendingTasks.put(nodeInfo, future);
+        log.info("ClientRecoveryPlugin: processExitEvent(): Added recovery task in the queue: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
+
+        // Cancel any previous recovery task (for the node) that is still pending
+        if (old!=null && ! old.isDone() && ! old.isCancelled()) {
+            log.warn("ClientRecoveryPlugin: processExitEvent(): Cancelled previous recovery task: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
+            old.cancel(false);
+        }
+    }
+
+    private void processRegisteredEvent(NodeRegistryEntry nodeInfo) {
+        log.debug("ClientRecoveryPlugin: processRegisteredEvent(): BEGIN: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
+
+        // Cancel any pending recovery task (for the node)
+        ScheduledFuture<?> future = pendingTasks.remove(nodeInfo);
+        if (future!=null && ! future.isDone() && ! future.isCancelled()) {
+            log.warn("ClientRecoveryPlugin: processRegisteredEvent(): Cancelled recovery task: client-id={}, client-address={}", nodeInfo.getClientId(), nodeInfo.getIpAddress());
+            future.cancel(false);
+        }
+
+        // Set node state to UP
+        baguetteServer.getSelfHealingManager().setNodeSelfHealingState(nodeInfo, SelfHealingManager.NODE_STATE.UP);
+    }
+
+    public void runClientRecovery(NodeRegistryEntry entry) throws Exception {
+        log.debug("ClientRecoveryPlugin: runClientRecovery(): node-info={}", entry);
+        if (entry==null) return;
+
+        log.trace("ClientRecoveryPlugin: runClientRecovery(): recoveryInstructionsFile={}", recoveryInstructionsFile);
+        entry.getPreregistration().put("instruction-files", recoveryInstructionsFile);
+
+        ClientInstallationTask task = InstallationHelperFactory.getInstance()
+                .createInstallationHelper(entry)
+                .createClientInstallationTask(entry);
+        log.debug("ClientRecoveryPlugin: runClientRecovery(): Client recovery task: {}", task);
+        SshClientInstaller installer = SshClientInstaller.builder()
+                .task(task)
+                .properties(clientInstallationProperties)
+                .build();
+
+        log.info("ClientRecoveryPlugin: runClientRecovery(): Starting client recovery: client-id={}, client-address={}", entry.getClientId(), entry.getIpAddress());
+        log.debug("ClientRecoveryPlugin: runClientRecovery(): Starting client recovery: node-info={}", entry);
+        boolean result = installer.execute();
+        pendingTasks.remove(entry);
+        log.info("ClientRecoveryPlugin: runClientRecovery(): Client recovery completed: result={}, client-id={}, client-address={}", result, entry.getClientId(), entry.getIpAddress());
+        log.debug("ClientRecoveryPlugin: runClientRecovery(): Client recovery completed: result={}, node-info={}", result, entry);
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/selfhealing/SelfHealingManagerImpl.java

@@ -0,0 +1,146 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.selfhealing;
+
+import gr.iccs.imu.ems.baguette.client.install.ClientInstallationProperties;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.common.recovery.RecoveryContext;
+import gr.iccs.imu.ems.common.selfhealing.SelfHealingManager;
+import lombok.Data;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Data
+@Service
+public class SelfHealingManagerImpl implements SelfHealingManager<NodeRegistryEntry>, InitializingBean {
+    private final ClientInstallationProperties clientInstallationProperties;
+    private final ServerSelfHealingProperties properties;
+    private final RecoveryContext recoveryContext;
+
+    private boolean enabled;
+    private MODE mode;
+    private Map<String, NodeRegistryEntry> nodes = new LinkedHashMap<>();
+    private Map<String, NODE_STATE> nodeStates = new LinkedHashMap<>();
+    private Map<String, String> nodeStateTexts = new LinkedHashMap<>();
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        log.info("Self-Healing Manager initialized");
+        setEnabled( properties.isEnabled() );
+        setMode( properties.getMode() );
+
+        // Initialize recovery context
+        recoveryContext.initialize(clientInstallationProperties, properties);
+        log.warn("Recovery context: {}", recoveryContext);
+    }
+
+    private void check() {
+        if (!enabled) throw new IllegalStateException("SelfHealingManager is not enabled");
+    }
+
+    @Override
+    public Collection<NodeRegistryEntry> getNodes() {
+        check();
+        return Collections.unmodifiableCollection(nodes.values());
+    }
+
+    @Override
+    public boolean containsNode(@NonNull NodeRegistryEntry node) {
+        check();
+        return nodes.containsKey(node.getIpAddress());
+    }
+
+    @Override
+    public boolean containsAny(@NonNull Collection<NodeRegistryEntry> nodes) {
+        check();
+        return Collections.disjoint(this.nodes.values(), nodes);
+    }
+
+    @Override
+    public boolean isMonitored(@NonNull NodeRegistryEntry node) {
+        check();
+        return mode==MODE.ALL ||
+                mode==MODE.INCLUDED && containsNode(node) ||
+                mode==MODE.EXCLUDED && ! containsNode(node);
+    }
+
+    @Override
+    public void addNode(@NonNull NodeRegistryEntry node) {
+        check();
+        nodes.put(node.getIpAddress(), node);
+    }
+
+    @Override
+    public void addAllNodes(@NonNull Collection<NodeRegistryEntry> nodes) {
+        check();
+        this.nodes.putAll(nodes.stream()
+                .filter(Objects::nonNull)
+                .collect(Collectors.toMap(NodeRegistryEntry::getIpAddress, Function.identity())));
+    }
+
+    @Override
+    public void removeNode(@NonNull NodeRegistryEntry node) {
+        check();
+        nodes.remove(node.getIpAddress());
+        nodeStates.remove(node.getIpAddress());
+        nodeStateTexts.remove(node.getIpAddress());
+    }
+
+    @Override
+    public void removeAllNodes(Collection<NodeRegistryEntry> nodes) {
+        check();
+        nodes.stream()
+                .filter(Objects::nonNull)
+                .forEach(this::removeNode);
+    }
+
+    @Override
+    public void clear() {
+        check();
+        nodes.clear();
+    }
+
+    @Override
+    public NODE_STATE getNodeSelfHealingState(@NonNull NodeRegistryEntry node) {
+        check();
+        if (mode!=MODE.EXCLUDED && ! nodes.containsKey(node.getIpAddress()))
+            return NODE_STATE.NOT_MONITORED;
+        if (mode==MODE.EXCLUDED && nodes.containsKey(node.getIpAddress()))
+            return NODE_STATE.NOT_MONITORED;
+        return nodeStates.get(node.getIpAddress());
+    }
+
+    @Override
+    public String getNodeSelfHealingStateText(@NonNull NodeRegistryEntry node) {
+        check();
+        if (mode!=MODE.EXCLUDED && ! nodes.containsKey(node.getIpAddress()))
+            return null;
+        if (mode==MODE.EXCLUDED && nodes.containsKey(node.getIpAddress()))
+            return null;
+        return nodeStateTexts.get(node.getIpAddress());
+    }
+
+    @Override
+    public void setNodeSelfHealingState(@NonNull NodeRegistryEntry node, @NonNull NODE_STATE state, String text) {
+        check();
+        if (!isMonitored(node)) return;
+        if (state==NODE_STATE.NOT_MONITORED)
+            throw new IllegalArgumentException("Node self-healing state cannot be set to NOT_MONITORED. Remove/Exclude node from self-healing instead");
+        nodeStates.put(node.getIpAddress(), state);
+        nodeStateTexts.put(node.getIpAddress(), text);
+    }
+}

ems-core/baguette-client-install/src/main/java/gr/iccs/imu/ems/baguette/client/selfhealing/ServerSelfHealingProperties.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.selfhealing;
+
+import gr.iccs.imu.ems.common.recovery.SelfHealingProperties;
+import gr.iccs.imu.ems.common.selfhealing.SelfHealingManager;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.context.annotation.Configuration;
+
+@Slf4j
+@Data
+@ToString(callSuper=true)
+@EqualsAndHashCode(callSuper = true)
+@Configuration
+public class ServerSelfHealingProperties extends SelfHealingProperties implements InitializingBean {
+	private SelfHealingManager.MODE mode = SelfHealingManager.MODE.INCLUDED;
+
+	@Override
+	public void afterPropertiesSet() throws Exception {
+		log.debug("ServerSelfHealingProperties: {}", this);
+	}
+}

ems-core/baguette-client/LICENSE

@@ -0,0 +1,373 @@
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in 
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.

ems-core/baguette-client/bin/baguette-client

@@ -0,0 +1,46 @@
+#! /bin/sh
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+### BEGIN INIT INFO
+# Provides:          baguette-client
+# Required-Start:    $local_fs $network
+# Required-Stop:     $local_fs
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: baguette-client
+# Description:       Controls the Baguette Client service
+### END INIT INFO
+
+export JAVA_HOME="/usr/bin/java"
+SU_USER=root
+ 
+#startcmd='/opt/baguette-client/bin/run.sh &>>/opt/baguette-client/logs/output.txt &'
+#stopcmd='/opt/baguette-client/bin/kill.sh &>>/opt/baguette-client/logs/output.txt'
+startcmd='/opt/baguette-client/bin/run.sh'
+stopcmd='/opt/baguette-client/bin/kill.sh'
+
+case "$1" in
+start)
+   echo "Starting Baguette Client..."
+   su -c "${startcmd}" $SU_USER
+;;
+restart)
+   echo "Re-starting Baguette Client..."
+   su -c "${stopcmd}" $SU_USER
+   su -c "${startcmd}" $SU_USER
+;;
+stop)
+   echo "Stopping Baguette Client..."
+   su -c "${stopcmd}" $SU_USER
+;;
+*)
+   echo "Usage: $0 {start|stop|restart}"
+exit 1
+esac

ems-core/baguette-client/bin/client.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless 
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at 
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+BASEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )
+JAVA_HOME=$( cd ${BASEDIR}/jre* && pwd )
+EMS_CONFIG_DIR=.
+
+#JAVA_OPTS=-Djavax.net.ssl.trustStore=./broker-truststore.p12\ -Djavax.net.ssl.trustStorePassword=melodic\ -Djavax.net.ssl.trustStoreType=pkcs12
+# -Djavax.net.debug=all
+# -Djavax.net.debug=ssl,handshake,record
+
+${JAVA_HOME}/bin/java $JAVA_OPTS -jar ${BASEDIR}/jars/broker-client/broker-client-jar-with-dependencies.jar $*

ems-core/baguette-client/bin/install.sh

@@ -0,0 +1,210 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+INSTALL_LOG=/opt/baguette-install.log
+echo "START: `date -Iseconds`" >> $INSTALL_LOG
+
+# Command line arguments: <server cert. file> <server url> <server api-key>
+SERVER_CERT=$1
+BASE_URL=$2
+APIKEY=$3
+
+if [ -z "$SERVER_CERT" ]; then
+  SERVER_CERT=""
+elif [ "$SERVER_CERT" = "-" ]; then
+  SERVER_CERT="--no-check-certificate"
+else
+  SERVER_CERT="--ca-certificate=${SERVER_CERT}"
+fi
+
+# Create installation directories
+BIN_DIRECTORY=/opt/baguette-client/bin
+CONF_DIRECTORY=/opt/baguette-client/conf
+LOGS_DIRECTORY=/opt/baguette-client/logs
+
+mkdir -p $BIN_DIRECTORY/
+mkdir -p $CONF_DIRECTORY/
+mkdir -p $LOGS_DIRECTORY/
+
+echo ""
+echo "** EMS Baguette Client **"
+echo "** Copyright ICCS-NTUA (C) 2016-2019, http://imu.iccs.gr **"
+echo ""
+date -Iseconds
+
+# Common variables
+DOWNLOAD_URL=$BASE_URL/baguette-client.tgz
+DOWNLOAD_URL_MD5=$BASE_URL/baguette-client.tgz.md5
+INSTALL_PACKAGE=/opt/baguette-client/baguette-client.tgz
+INSTALL_PACKAGE_MD5=/opt/baguette-client/baguette-client.tgz.md5
+INSTALL_DIR=/opt/
+STARTUP_SCRIPT=$BIN_DIRECTORY/baguette-client
+SERVICE_NAME=baguette-client
+CLIENT_CONF_FILE=$CONF_DIRECTORY/baguette-client.properties
+CLIENT_ID_FILE=$CONF_DIRECTORY/id.txt
+
+# Check if already installed
+if [ -f /opt/baguette-client/conf/ok.txt ]; then
+  echo "Already installed. Exiting..."
+  date -Iseconds
+  echo "END: Already installed: `date -Iseconds`" >> $INSTALL_LOG
+  exit 0
+fi
+
+# Create installation directory
+echo ""
+echo "Create installation directory..."
+date -Iseconds
+mkdir -p $INSTALL_DIR/baguette-client
+if [ $? != 0 ]; then
+  echo "Failed to create installation directory ($?)"
+  echo "Aborting installation..."
+  date -Iseconds
+  echo "ABORT: mkdir: `date -Iseconds`" >> $INSTALL_LOG
+  exit 1
+fi
+
+# Download installation package
+echo ""
+echo "Download installation package..."
+date -Iseconds
+wget $SERVER_CERT $DOWNLOAD_URL -O $INSTALL_PACKAGE
+if [ $? != 0 ]; then
+  echo "Failed to download installation package ($?)"
+  echo "Aborting installation..."
+  date -Iseconds
+  echo "ABORT: download: `date -Iseconds`" >> $INSTALL_LOG
+  exit 1
+fi
+date -Iseconds
+echo "Download installation package...ok"
+
+# Download installation package MD5 checksum
+echo ""
+echo "Download installation package MD5 checksum..."
+date -Iseconds
+wget $SERVER_CERT $DOWNLOAD_URL_MD5 -O $INSTALL_PACKAGE_MD5
+if [ $? != 0 ]; then
+  echo "Failed to download installation package ($?)"
+  echo "Aborting installation..."
+  date -Iseconds
+  echo "ABORT: download MD5: `date -Iseconds`" >> $INSTALL_LOG
+  exit 1
+fi
+date -Iseconds
+echo "Download installation package MD5 checksum...ok"
+
+# Check MD5 checksum
+PACKAGE_MD5=`cat $INSTALL_PACKAGE_MD5`
+PACKAGE_CHECKSUM=`md5sum $INSTALL_PACKAGE |cut -d " " -f 1`
+echo ""
+echo "Checksum MD5:  $PACKAGE_MD5"
+echo "Checksum calc: $PACKAGE_CHECKSUM"
+if [ $PACKAGE_CHECKSUM == $PACKAGE_MD5 ]; then
+  echo "Checksum: ok"
+else
+  echo "Checksum: wrong"
+  echo "Aborting installation..."
+  date -Iseconds
+  echo "ABORT: wrong MD5: `date -Iseconds`" >> $INSTALL_LOG
+  exit 1
+fi
+
+# Extract installation package
+echo ""
+echo "Extracting installation package..."
+date -Iseconds
+#unzip -o $INSTALL_PACKAGE -d $INSTALL_DIR
+tar -xvzf $INSTALL_PACKAGE -C $INSTALL_DIR
+if [ $? != 0 ]; then
+  echo "Failed to extract installation package contents ($?)"
+  echo "Aborting installation..."
+  date -Iseconds
+  echo "ABORT: extract: `date -Iseconds`" >> $INSTALL_LOG
+  exit 1
+fi
+date -Iseconds
+
+# Make scripts executable
+echo ""
+echo "Make scripts executable..."
+date -Iseconds
+chmod u=rx,og-rwx $INSTALL_DIR/baguette-client/bin/*
+if [ $? != 0 ]; then
+  echo "Failed to copy service script to /etc/init.d/ directory ($?)"
+  echo "Aborting installation..."
+  date -Iseconds
+  echo "ABORT: chmod: `date -Iseconds`" >> $INSTALL_LOG
+  exit 1
+fi
+
+# Register as a service
+echo ""
+echo "Register as a service..."
+date -Iseconds
+cp -f $STARTUP_SCRIPT /etc/init.d/
+if [ $? != 0 ]; then
+  echo "Failed to copy service script to /etc/init.d/ directory ($?)"
+  echo "Aborting installation..."
+  date -Iseconds
+  echo "ABORT: cp init.d: `date -Iseconds`" >> $INSTALL_LOG
+  exit 1
+fi
+
+update-rc.d $SERVICE_NAME defaults
+if [ $? != 0 ]; then
+  echo "Failed to register service script to /etc/init.d/ directory ($?)"
+  echo "Aborting installation..."
+  date -Iseconds
+  echo "ABORT: update-rc.d: `date -Iseconds`" >> $INSTALL_LOG
+  exit 1
+fi
+
+# Add Id, Credentials and Client configuration files
+echo "Add Id, Credentials and Client configuration files"
+date -Iseconds
+touch $CLIENT_ID_FILE $CLIENT_CONF_FILE
+if [ $? != 0 ]; then
+  echo "Failed to 'touch' configuration files ($?)"
+  echo "Aborting installation..."
+  date -Iseconds
+  echo "ABORT: touch: `date -Iseconds`" >> $INSTALL_LOG
+  exit 1
+fi
+
+chmod u=rw,og-rwx $CLIENT_ID_FILE $CLIENT_CONF_FILE
+if [ $? != 0 ]; then
+  echo "Failed to change permissions of configuration files ($?)"
+  echo "Aborting installation..."
+  date -Iseconds
+  echo "ABORT: chmod 2: `date -Iseconds`" >> $INSTALL_LOG
+  exit 1
+fi
+
+# Write successful installation file
+echo "Write successful installation file"
+date -Iseconds
+sudo touch $CONF_DIRECTORY/ok.txt
+
+echo "END: OK: `date -Iseconds`" >> $INSTALL_LOG
+
+# Launch Baguette Client
+echo "Launch Baguette Client"
+date -Iseconds
+sudo service baguette-client start
+
+echo "RUN: `date -Iseconds`" >> $INSTALL_LOG
+
+# Success
+echo ""
+echo "Success - Baguette client successfully installed on system"
+date -Iseconds
+echo ""
+exit 0

ems-core/baguette-client/bin/kill.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+# Get Baguette client home directory
+BASEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )
+
+# Update path
+#PATH=$PATH:/path/to/jre/bin/
+
+# Kill Baguette client
+#PID=`jps | grep BaguetteClient | cut -d " " -f 1`
+PID=`ps -ef |grep java |grep BaguetteClient | cut -c 10-20`
+if [ "$PID" != "" ]
+then
+	echo "Killing baguette client (pid: $PID)"
+	kill -9 $PID
+else
+	echo "Baguette client is not running"
+fi

ems-core/baguette-client/bin/run.bat

@@ -0,0 +1,44 @@
+@echo off
+::
+:: Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+::
+:: This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+:: Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+:: If a copy of the MPL was not distributed with this file, you can obtain one at
+:: https://www.mozilla.org/en-US/MPL/2.0/
+::
+
+setlocal
+set PWD=%~dp0
+cd %PWD%..
+set BASEDIR=%cd%
+IF NOT DEFINED EMS_CONFIG_DIR set EMS_CONFIG_DIR=%BASEDIR%\conf
+IF NOT DEFINED PAASAGE_CONFIG_DIR set PAASAGE_CONFIG_DIR=%BASEDIR%\conf
+IF NOT DEFINED EMS_CONFIG_LOCATION set EMS_CONFIG_LOCATION=optional:file:%EMS_CONFIG_DIR%\ems-client.yml,optional:file:%EMS_CONFIG_DIR%\ems-client.properties,optional:file:%EMS_CONFIG_DIR%\baguette-client.yml,optional:file:%EMS_CONFIG_DIR%\baguette-client.properties
+IF NOT DEFINED JASYPT_PASSWORD set JASYPT_PASSWORD=password
+set JAVA_HOME=%BASEDIR%/jre
+
+:: Update path
+set PATH=%JAVA_HOME%\bin;%PATH%
+
+:: Copy dependencies if missing
+if exist pom.xml (
+    if not exist %BASEDIR%\target\dependency cmd /C "mvn dependency:copy-dependencies"
+)
+
+:: Run Baguette Client
+set JAVA_OPTS= -Djavax.net.ssl.trustStore=%EMS_CONFIG_DIR%\client-broker-truststore.p12 ^
+ -Djavax.net.ssl.trustStorePassword=melodic ^
+ -Djavax.net.ssl.trustStoreType=pkcs12 ^
+ -Djasypt.encryptor.password=%JASYPT_PASSWORD% ^
+ --add-opens=java.base/java.lang=ALL-UNNAMED  --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+::set JAVA_OPTS=-Djavax.net.debug=all %JAVA_OPTS%
+::set JAVA_OPTS=-Dlogging.level.gr.iccs.imu.ems=TRACE %JAVA_OPTS%
+
+echo EMS_CONFIG_DIR=%EMS_CONFIG_DIR%
+echo EMS_CONFIG_LOCATION=%EMS_CONFIG_LOCATION%
+echo Starting baguette client...
+java %JAVA_OPTS% -classpath "%EMS_CONFIG_DIR%;%BASEDIR%\jars\*;%BASEDIR%\target\classes;%BASEDIR%\target\dependency\*" gr.iccs.imu.ems.baguette.client.BaguetteClient "--spring.config.location=%EMS_CONFIG_LOCATION%" "--logging.config=file:%EMS_CONFIG_DIR%\logback-spring.xml"  %*
+
+cd %PWD%
+endlocal

ems-core/baguette-client/bin/run.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+# Change directory to Baguette client home
+PREVWORKDIR=`pwd`
+BASEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )
+cd ${BASEDIR}
+EMS_CONFIG_DIR=${BASEDIR}/conf
+PAASAGE_CONFIG_DIR=${BASEDIR}/conf
+EMS_CONFIG_LOCATION=optional:file:$EMS_CONFIG_DIR/ems-client.yml,optional:file:$EMS_CONFIG_DIR/ems-client.properties,optional:file:$EMS_CONFIG_DIR/baguette-client.yml,optional:file:$EMS_CONFIG_DIR/baguette-client.properties
+LOG_FILE=${BASEDIR}/logs/output.txt
+TEE_FILE=${BASEDIR}/logs/tee.txt
+JASYPT_PASSWORD=password
+JAVA_HOME=${BASEDIR}/jre
+export EMS_CONFIG_DIR PAASAGE_CONFIG_DIR LOG_FILE JASYPT_PASSWORD JAVA_HOME
+
+# Update path
+PATH=${JAVA_HOME}/bin:$PATH
+
+# Check if baguette client is already running
+#PID=`jps | grep BaguetteClient | cut -d " " -f 1`
+PID=`ps -ef |grep java |grep BaguetteClient | cut -c 10-14`
+if [ "$PID" != "" ]
+then
+    echo "Baguette client is already running (pid: $PID)"
+    exit 0
+fi
+
+# Copy dependencies if missing
+if [ -f pom.xml ]; then
+	if [ ! -d ${BASEDIR}/target/dependency ]; then
+		mvn dependency:copy-dependencies
+	fi
+fi
+
+# Run Baguette client
+JAVA_OPTS=-Djavax.net.ssl.trustStore=${EMS_CONFIG_DIR}/client-broker-truststore.p12
+JAVA_OPTS="${JAVA_OPTS} -Djavax.net.ssl.trustStorePassword=melodic -Djavax.net.ssl.trustStoreType=pkcs12"
+JAVA_OPTS="${JAVA_OPTS} -Djasypt.encryptor.password=$JASYPT_PASSWORD"
+#JAVA_OPTS="-Djavax.net.debug=all ${JAVA_OPTS}"
+#JAVA_OPTS="-Dlogging.level.gr.iccs.imu.ems=TRACE ${JAVA_OPTS}"
+JAVA_OPTS="${JAVA_OPTS} --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
+
+echo "Starting baguette client..."
+echo "EMS_CONFIG_DIR=${EMS_CONFIG_DIR}"
+echo "EMS_CONFIG_LOCATION=${EMS_CONFIG_LOCATION}"
+echo "LOG_FILE=${LOG_FILE}"
+
+echo "Starting baguette client..." &>> ${LOG_FILE}
+echo "EMS_CONFIG_DIR=${EMS_CONFIG_DIR}" &>> ${LOG_FILE}
+echo "EMS_CONFIG_LOCATION=${EMS_CONFIG_LOCATION}" &>> ${LOG_FILE}
+echo "LOG_FILE=${LOG_FILE}" &>> ${LOG_FILE}
+
+if [ "$1" == "--i" ]; then
+  echo "Baguette client running in Interactive mode"
+  java ${JAVA_OPTS} -classpath "conf:jars/*:target/classes:target/dependency/*" gr.iccs.imu.ems.baguette.client.BaguetteClient "--spring.config.location=${EMS_CONFIG_LOCATION}" "--logging.config=file:${EMS_CONFIG_DIR}/logback-spring.xml" $* $* 2>&1 | tee ${TEE_FILE}
+else
+  java ${JAVA_OPTS} -classpath "conf:jars/*:target/classes:target/dependency/*" gr.iccs.imu.ems.baguette.client.BaguetteClient "--spring.config.location=${EMS_CONFIG_LOCATION}" "--logging.config=file:${EMS_CONFIG_DIR}/logback-spring.xml" $* &>> ${LOG_FILE} &
+  PID=`jps | grep BaguetteClient | cut -d " " -f 1`
+  PID=`ps -ef |grep java |grep BaguetteClient | cut -c 10-14`
+  echo "Baguette client PID: $PID"
+fi
+
+cd $PREVWORKDIR

ems-core/baguette-client/conf/baguette-client.properties.sample

@@ -0,0 +1,214 @@
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+################################################################################
+###                     EMS - Baguette Client properties                     ###
+################################################################################
+
+#password-encoder-class = password.gr.iccs.imu.ems.util.AsterisksPasswordEncoder
+#password-encoder-class = password.gr.iccs.imu.ems.util.IdentityPasswordEncoder
+#password-encoder-class = password.gr.iccs.imu.ems.util.PresentPasswordEncoder
+
+# Baguette Client configuration
+
+auth-timeout = 60000
+exec-timeout = 120000
+#retry-period = 60000
+exit-command-allowed = false
+#kill-delay = 10
+
+IP_SETTING=${IP_SETTING}
+EMS_CLIENT_ADDRESS=${${IP_SETTING}}
+
+node-properties=
+
+# -----------------------------------------------------------------------------
+# Client Id and Baguette Server credentials
+# -----------------------------------------------------------------------------
+
+client-id = ${BAGUETTE_CLIENT_ID}
+
+#server-address = ${BAGUETTE_SERVER_HOSTNAME}
+server-address = ${BAGUETTE_SERVER_ADDRESS}
+server-port = ${BAGUETTE_SERVER_PORT}
+server-pubkey = ${BAGUETTE_SERVER_PUBKEY}
+server-fingerprint = ${BAGUETTE_SERVER_PUBKEY_FINGERPRINT}
+
+server-username = ${BAGUETTE_SERVER_USERNAME}
+server-password = ${BAGUETTE_SERVER_PASSWORD}
+
+# -----------------------------------------------------------------------------
+# Client-side Self-healing settings
+# -----------------------------------------------------------------------------
+
+#self.healing.enabled=true
+#self.healing.recovery.file.baguette=conf/baguette.json
+#self.healing.recovery.file.netdata=conf/netdata.json
+#self.healing.recovery.delay=10000
+#self.healing.recovery.retry.wait=60000
+#self.healing.recovery.max.retries=3
+
+# -----------------------------------------------------------------------------
+# Collectors settings
+# -----------------------------------------------------------------------------
+
+#collector-classes = netdata.collector.gr.iccs.imu.ems.baguette.client.NetdataCollector
+
+collector.netdata.enable = true
+collector.netdata.delay = 10000
+collector.netdata.url = http://127.0.0.1:19999/api/v1/allmetrics?format=json
+collector.netdata.urlOfNodesWithoutClient = http://%s:19999/api/v1/allmetrics?format=json
+#collector.netdata.create-topic = true
+#collector.netdata.allowed-topics = netdata__system__cpu__user:an_alias
+collector.netdata.allowed-topics = ${COLLECTOR_ALLOWED_TOPICS}
+collector.netdata.error-limit = 3
+collector.netdata.pause-period = 60
+
+collector.prometheus.enable = true
+collector.prometheus.delay = 10000
+collector.prometheus.url = http://127.0.0.1:9090/metrics
+collector.prometheus.urlOfNodesWithoutClient = http://%s:9090/metrics
+#collector.prometheus.create-topic = true
+#collector.prometheus.allowed-topics = system__cpu__user:an_alias
+collector.prometheus.allowed-topics = ${COLLECTOR_ALLOWED_TOPICS}
+collector.prometheus.error-limit = 3
+collector.prometheus.pause-period = 60
+#
+#collector.prometheus.allowedTags =
+#collector.prometheus.allowTagsInDestinationName = true
+#collector.prometheus.destinationNameFormatter = ${metricName}_${method}
+#collector.prometheus.addTagsAsEventProperties = true
+#collector.prometheus.addTagsInEventPayload = true
+#collector.prometheus.throwExceptionWhenExcessiveCharsOccur = true
+
+# -----------------------------------------------------------------------------
+# Cluster settings
+# -----------------------------------------------------------------------------
+
+#cluster.cluster-id=cluster
+#cluster.local-node.id=local-node
+#cluster.local-node.address=localhost:1234
+#cluster.local-node.properties.name=value
+#cluster.member-addresses=[localhost:3456, localhost:5678]
+
+#cluster.useSwim=false
+#cluster.failureTimeout=5000
+cluster.testInterval=5000
+
+cluster.log-enabled=true
+cluster.out-enabled=true
+
+cluster.join-on-init=true
+cluster.election-on-join=false
+#cluster.usePBInMg=true
+#cluster.usePBInPg=true
+#cluster.mgName=system
+#cluster.pgName=data
+
+cluster.tls.enabled=true
+#cluster.tls.keystore=${EMS_CONFIG_DIR}/cluster.jks
+#cluster.tls.keystore-password=atomix
+#cluster.tls.truststore=${EMS_CONFIG_DIR}/cluster.jks
+#cluster.tls.truststore-password=atomix
+cluster.tls.keystore-dir=conf
+
+cluster.score.formula=20*cpu/32+80*ram/(256*1024)
+cluster.score.default-score=0
+cluster.score.default-args.cpu=1
+cluster.score.default-args.ram=128
+#cluster.score.throw-exception=false
+
+
+################################################################################
+###                       EMS - Broker-CEP properties                        ###
+################################################################################
+
+# Broker ports and protocol
+brokercep.broker-name = broker
+brokercep.broker-port = 61617
+#brokercep.management-connector-port = 1088
+brokercep.broker-protocol = ssl
+# Don't use in EMS server
+#brokercep.bypass-local-broker = true
+
+# Common Broker settings
+BROKER_URL_PROPERTIES = transport.daemon=true&transport.trace=false&transport.useKeepAlive=true&transport.useInactivityMonitor=false&transport.needClientAuth=${CLIENT_AUTH_REQUIRED}&transport.verifyHostName=true&transport.connectionTimeout=0&transport.keepAlive=true
+CLIENT_AUTH_REQUIRED = false
+brokercep.broker-url[0] = ${brokercep.broker-protocol}://0.0.0.0:${brokercep.broker-port}?${BROKER_URL_PROPERTIES}
+brokercep.broker-url[1] = tcp://127.0.0.1:61616?${BROKER_URL_PROPERTIES}
+brokercep.broker-url[2] =
+
+CLIENT_URL_PROPERTIES=daemon=true&trace=false&useInactivityMonitor=false&connectionTimeout=0&keepAlive=true
+brokercep.broker-url-for-consumer = tcp://127.0.0.1:61616?${CLIENT_URL_PROPERTIES}
+brokercep.broker-url-for-clients = ${brokercep.broker-protocol}://${EMS_CLIENT_ADDRESS}:${brokercep.broker-port}?${CLIENT_URL_PROPERTIES}
+# Must be a public IP address
+
+# Key store
+brokercep.ssl.keystore-file = ${EMS_CONFIG_DIR}/client-broker-keystore.p12
+brokercep.ssl.keystore-type = PKCS12
+#brokercep.ssl.keystore-password = melodic
+brokercep.ssl.keystore-password = ENC(ISMbn01HVPbtRPkqm2Lslg==)
+# Trust store
+brokercep.ssl.truststore-file = ${EMS_CONFIG_DIR}/client-broker-truststore.p12
+brokercep.ssl.truststore-type = PKCS12
+#brokercep.ssl.truststore-password = melodic
+brokercep.ssl.truststore-password = ENC(ISMbn01HVPbtRPkqm2Lslg==)
+# Certificate
+brokercep.ssl.certificate-file = ${EMS_CONFIG_DIR}/client-broker.crt
+# Key-and-Cert data
+brokercep.ssl.key-entry-generate = IF-IP-CHANGED
+brokercep.ssl.key-entry-name = ${EMS_CLIENT_ADDRESS}
+brokercep.ssl.key-entry-dname = CN=${EMS_CLIENT_ADDRESS},OU=Information Management Unit (IMU),O=Institute of Communication and Computer Systems (ICCS),L=Athens,ST=Attika,C=GR
+brokercep.ssl.key-entry-ext-san = dns:localhost,ip:127.0.0.1,ip:${DEFAULT_IP},ip:${PUBLIC_IP}
+
+# Authentication and Authorization settings
+brokercep.authentication-enabled = true
+#brokercep.additional-broker-credentials = aaa/111, bbb/222, morphemic/morphemic
+brokercep.additional-broker-credentials = ENC(axeJUxNHajYfBffUwvuT3kwTgLTpRliDMz/ZQ9hROZ3BNOv0Idw72NJsawzIZRuZ)
+brokercep.authorization-enabled = false
+
+# Broker instance settings
+brokercep.broker-persistence-enabled = false
+brokercep.broker-using-jmx = true
+brokercep.broker-advisory-support-enabled = true
+brokercep.broker-using-shutdown-hook = false
+
+#brokercep.broker-enable-statistics = true
+#brokercep.broker-populate-jmsx-user-id = true
+
+# Message interceptors
+brokercep.message-interceptors[0].destination = >
+brokercep.message-interceptors[0].className = interceptor.broker.gr.iccs.imu.ems.brokercep.SequentialCompositeInterceptor
+brokercep.message-interceptors[0].params[0] = #SourceAddressMessageUpdateInterceptor
+brokercep.message-interceptors[0].params[1] = #MessageForwarderInterceptor
+brokercep.message-interceptors[0].params[2] = #NodePropertiesMessageUpdateInterceptor
+
+brokercep.message-interceptors-specs.SourceAddressMessageUpdateInterceptor.className = interceptor.broker.gr.iccs.imu.ems.brokercep.SourceAddressMessageUpdateInterceptor
+brokercep.message-interceptors-specs.MessageForwarderInterceptor.className = interceptor.broker.gr.iccs.imu.ems.brokercep.MessageForwarderInterceptor
+brokercep.message-interceptors-specs.NodePropertiesMessageUpdateInterceptor.className = interceptor.broker.gr.iccs.imu.ems.brokercep.NodePropertiesMessageUpdateInterceptor
+
+# Message forward destinations (MessageForwarderInterceptor must be included in 'message-interceptors' property)
+#brokercep.message-forward-destinations[0].connection-string = tcp://localhost:51515
+#brokercep.message-forward-destinations[0].username = AAA
+#brokercep.message-forward-destinations[0].password = 111
+#brokercep.message-forward-destinations[1].connection-string = tcp://localhost:41414
+#brokercep.message-forward-destinations[1].username = AAA
+#brokercep.message-forward-destinations[1].password = 111
+
+# Advisory watcher
+brokercep.enable-advisory-watcher = true
+
+# Memory usage limit
+brokercep.usage.memory.jvm-heap-percentage = 20
+#brokercep.usage.memory.size = 134217728
+
+#brokercep.maxEventForwardRetries: -1
+#brokercep.maxEventForwardDuration: -1
+
+################################################################################

ems-core/baguette-client/conf/baguette-client.yml

@@ -0,0 +1,246 @@
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+################################################################################
+###                     EMS - Baguette Client properties                     ###
+################################################################################
+
+#password-encoder-class: password.gr.iccs.imu.ems.util.AsterisksPasswordEncoder
+#password-encoder-class: password.gr.iccs.imu.ems.util.IdentityPasswordEncoder
+#password-encoder-class: password.gr.iccs.imu.ems.util.PresentPasswordEncoder
+
+# Baguette Client configuration
+
+auth-timeout: 60000
+exec-timeout: 120000
+#retry-period: 60000
+exit-command-allowed: false
+#kill-delay: 10
+
+IP_SETTING: ${IP_SETTING}
+EMS_CLIENT_ADDRESS: ${${IP_SETTING}}
+
+node-properties:
+  node-id: ${NODE_CLIENT_ID}
+  public-ip: ${NODE_ADDRESS}
+  private-ip: ${NODE_ADDRESS}
+  instance: ${NODE_ADDRESS}
+  host: ${NODE_ADDRESS}
+  zone: ${zone-id}
+  region: ${zone-id}
+  cloud: ${provider}
+
+# -----------------------------------------------------------------------------
+# Client Id and Baguette Server credentials
+# -----------------------------------------------------------------------------
+
+client-id: ${BAGUETTE_CLIENT_ID}
+
+#server-address: ${BAGUETTE_SERVER_HOSTNAME}
+server-address: ${BAGUETTE_SERVER_ADDRESS}
+server-port: ${BAGUETTE_SERVER_PORT}
+server-pubkey: ${BAGUETTE_SERVER_PUBKEY}
+server-fingerprint: ${BAGUETTE_SERVER_PUBKEY_FINGERPRINT}
+
+server-username: ${BAGUETTE_SERVER_USERNAME}
+server-password: ${BAGUETTE_SERVER_PASSWORD}
+
+# -----------------------------------------------------------------------------
+# Client-side Self-healing settings
+# -----------------------------------------------------------------------------
+
+#self.healing:
+#  enabled: true
+#  recovery:
+#    file:
+#      baguette: conf/baguette.json
+#      netdata: conf/netdata.json
+#    delay: 10000
+#    retry-delay: 60000
+#    max-retries: 3
+
+# -----------------------------------------------------------------------------
+# Collectors settings
+# -----------------------------------------------------------------------------
+
+#collector-classes: netdata.collector.gr.iccs.imu.ems.baguette.client.NetdataCollector
+
+collector:
+  netdata:
+    enable: true
+    delay: 10000
+    url: http://127.0.0.1:19999/api/v1/allmetrics?format=json
+    urlOfNodesWithoutClient: http://%s:19999/api/v1/allmetrics?format=json
+    #create-topic: true
+    #allowed-topics: netdata__system__cpu__user:an_alias
+    allowed-topics: ${COLLECTOR_ALLOWED_TOPICS}
+    error-limit: 3
+    pause-period: 60
+  prometheus:
+    enable: true
+    delay: 10000
+    url: http://127.0.0.1:9090/metrics
+    urlOfNodesWithoutClient: http://%s:9090/metrics
+    #create-topic: true
+    #allowed-topics: system__cpu__user:an_alias
+    allowed-topics: ${COLLECTOR_ALLOWED_TOPICS}
+    error-limit: 3
+    pause-period: 60
+    #
+    #allowedTags: []
+    #allowTagsInDestinationName: true
+    #destinationNameFormatter: '${metricName}_${method}'
+    #addTagsAsEventProperties: true
+    #addTagsInEventPayload: true
+    #throwExceptionWhenExcessiveCharsOccur: true
+
+# -----------------------------------------------------------------------------
+# Cluster settings
+# -----------------------------------------------------------------------------
+
+cluster:
+  #cluster-id: cluster
+  #local-node.id: local-node
+  #local-node.address: localhost:1234
+  #local-node.properties:
+  #  name: value
+  #member-addresses: [localhost:3456, localhost:5678]
+
+  #useSwim: false
+  #failureTimeout: 5000
+  testInterval: 5000
+
+  log-enabled: true
+  out-enabled: true
+
+  join-on-init: true
+  election-on-join: false
+  #usePBInMg: true
+  #usePBInPg: true
+  #mgName: system
+  #pgName: data
+
+  tls:
+    enabled: true
+    #keystore: ${EMS_CONFIG_DIR}/cluster.jks
+    #keystore-password: atomix
+    #truststore: ${EMS_CONFIG_DIR}/cluster.jks
+    #truststore-password: atomix
+    keystore-dir: conf
+
+  score:
+    formula: 20*cpu/32+80*ram/(256*1024)
+    default-score: 0
+    default-args:
+      cpu: 1
+      ram: 128
+    #throw-exception: false
+
+
+################################################################################
+###                       EMS - Broker-CEP properties                        ###
+################################################################################
+
+BROKER_URL_PROPERTIES: transport.daemon=true&transport.trace=false&transport.useKeepAlive=true&transport.useInactivityMonitor=false&transport.needClientAuth=${CLIENT_AUTH_REQUIRED}&transport.verifyHostName=true&transport.connectionTimeout=0&transport.keepAlive=true
+CLIENT_AUTH_REQUIRED: false
+CLIENT_URL_PROPERTIES: daemon=true&trace=false&useInactivityMonitor=false&connectionTimeout=0&keepAlive=true
+
+brokercep:
+  # Broker ports and protocol
+  broker-name: broker
+  broker-port: 61617
+  broker-protocol: ssl
+  #management-connector-port: 1088
+  #bypass-local-broker: true   # Don't use in EMS server
+
+  # Broker connectors
+  broker-url:
+    - ${brokercep.broker-protocol}://0.0.0.0:${brokercep.broker-port}?${BROKER_URL_PROPERTIES}
+    - tcp://127.0.0.1:61616?${BROKER_URL_PROPERTIES}
+
+  # Broker URLs for (EMS) consumer and clients
+  broker-url-for-consumer: tcp://127.0.0.1:61616?${CLIENT_URL_PROPERTIES}
+  broker-url-for-clients: ${brokercep.broker-protocol}://${EMS_CLIENT_ADDRESS}:${brokercep.broker-port}?${CLIENT_URL_PROPERTIES}
+  # Must be a public IP address
+
+  ssl:
+    # Key store settings
+    keystore-file: ${EMS_CONFIG_DIR}/client-broker-keystore.p12
+    keystore-type: PKCS12
+    keystore-password: 'ENC(ISMbn01HVPbtRPkqm2Lslg==)'  # melodic
+
+    # Trust store settings
+    truststore-file: ${EMS_CONFIG_DIR}/client-broker-truststore.p12
+    truststore-type: PKCS12
+    truststore-password: 'ENC(ISMbn01HVPbtRPkqm2Lslg==)'  # melodic
+
+    # Certificate settings
+    certificate-file: ${EMS_CONFIG_DIR}/client-broker.crt
+
+    # key generation settings
+    key-entry-generate: IF-IP-CHANGED
+    key-entry-name: ${EMS_CLIENT_ADDRESS}
+    key-entry-dname: 'CN=${EMS_CLIENT_ADDRESS},OU=Information Management Unit (IMU),O=Institute of Communication and Computer Systems (ICCS),L=Athens,ST=Attika,C=GR'
+    key-entry-ext-san: 'dns:localhost,ip:127.0.0.1,ip:${DEFAULT_IP},ip:${PUBLIC_IP}'
+
+  # Authentication and Authorization settings
+  authentication-enabled: true
+  #additional-broker-credentials: aaa/111, bbb/222, morphemic/morphemic
+  additional-broker-credentials: 'ENC(axeJUxNHajYfBffUwvuT3kwTgLTpRliDMz/ZQ9hROZ3BNOv0Idw72NJsawzIZRuZ)'
+  authorization-enabled: false
+
+  # Broker instance settings
+  broker-persistence-enabled: false
+  broker-using-jmx: true
+  broker-advisory-support-enabled: true
+  broker-using-shutdown-hook: false
+
+  #broker-enable-statistics: true
+  #broker-populate-jmsx-user-id: true
+
+  # Message interceptors
+  message-interceptors:
+    - destination: '>'
+      className: 'interceptor.broker.gr.iccs.imu.ems.brokercep.SequentialCompositeInterceptor'
+      params:
+        - '#SourceAddressMessageUpdateInterceptor'
+        - '#MessageForwarderInterceptor'
+        - '#NodePropertiesMessageUpdateInterceptor'
+
+  message-interceptors-specs:
+    SourceAddressMessageUpdateInterceptor:
+      className: interceptor.broker.gr.iccs.imu.ems.brokercep.SourceAddressMessageUpdateInterceptor
+    MessageForwarderInterceptor:
+      className: interceptor.broker.gr.iccs.imu.ems.brokercep.MessageForwarderInterceptor
+    NodePropertiesMessageUpdateInterceptor:
+      className: interceptor.broker.gr.iccs.imu.ems.brokercep.NodePropertiesMessageUpdateInterceptor
+
+  # Message forward destinations (MessageForwarderInterceptor must be included in 'message-interceptors' property)
+  #message-forward-destinations:
+  #  - connection-string: tcp://localhost:51515
+  #    username: AAA
+  #    password: 111
+  #  - connection-string: tcp://localhost:41414
+  #    username: AAA
+  #    password: 111
+
+  # Advisory watcher
+  enable-advisory-watcher: true
+
+  # Memory usage limit
+  usage:
+    memory:
+      jvm-heap-percentage: 20
+      #size: 134217728
+
+  # Event forward settings
+  #maxEventForwardRetries: -1
+  #maxEventForwardDuration: -1
+
+################################################################################

ems-core/baguette-client/conf/baguette.json

@@ -0,0 +1,16 @@
+[{
+  "name": "Initial wait...",
+  "command": "pwd",
+  "waitBefore": 0,
+  "waitAfter": 5000
+}, {
+  "name": "Sending baguette client kill command...",
+  "command": "${BAGUETTE_CLIENT_BASE_DIR}/bin/kill.sh",
+  "waitBefore": 0,
+  "waitAfter": 2000
+}, {
+  "name": "Sending baguette client start command...",
+  "command": "${BAGUETTE_CLIENT_BASE_DIR}/bin/run.sh",
+  "waitBefore": 0,
+  "waitAfter": 10000
+}]

ems-core/baguette-client/conf/logback-spring.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+  ~
+  ~ This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+  ~ Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+  ~ If a copy of the MPL was not distributed with this file, you can obtain one at
+  ~ https://www.mozilla.org/en-US/MPL/2.0/
+  -->
+
+<configuration>
+    <include resource="org/springframework/boot/logging/logback/base.xml"/>
+
+    <!-- NOTE: Use this appender for simpler logging messages (only level and message) during development -->
+    <!-- Change ref="CONSOLE" to ref="STDOUT" in the logger entries -->
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <!--<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] %msg%n</pattern>-->
+            <pattern>BC> %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <logger name="root" level="ERROR">
+        <appender-ref ref="STDOUT"/>
+    </logger>
+    <logger name="org.springframework" level="INFO">
+        <appender-ref ref="STDOUT"/>
+    </logger>
+    <logger name="gr.iccs.imu.ems" level="INFO" additivity="false">
+        <appender-ref ref="STDOUT"/>
+    </logger>
+    <logger name="gr.iccs.imu.ems.baguette" level="INFO" additivity="false">
+        <appender-ref ref="STDOUT"/>
+    </logger>
+    <logger name="gr.iccs.imu.ems.brokercep" level="INFO" additivity="false">
+        <appender-ref ref="STDOUT"/>
+    </logger>
+</configuration>

ems-core/baguette-client/conf/netdata.json

@@ -0,0 +1,16 @@
+[{
+  "name": "Initial wait...",
+  "command": "pwd",
+  "waitBefore": 0,
+  "waitAfter": 5000
+}, {
+  "name": "Sending Netdata agent kill command...",
+  "command": "sudo sh -c  'ps -U netdata -o \"pid\" --no-headers | xargs kill -9' ",
+  "waitBefore": 0,
+  "waitAfter": 2000
+}, {
+  "name": "Sending Netdata agent start command...",
+  "command": "sudo netdata",
+  "waitBefore": 0,
+  "waitAfter": 10000
+}]

ems-core/baguette-client/pom.xml

@@ -0,0 +1,175 @@
+<!--
+  ~ Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+  ~
+  ~ This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+  ~ Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+  ~ If a copy of the MPL was not distributed with this file, you can obtain one at
+  ~ https://www.mozilla.org/en-US/MPL/2.0/
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>gr.iccs.imu.ems</groupId>
+        <artifactId>ems-core</artifactId>
+		<version>${revision}</version>
+	</parent>
+
+	<artifactId>baguette-client</artifactId>
+	<name>EMS - Baguette Client</name>
+
+	<properties>
+		<atomix.version>3.1.12</atomix.version>
+	</properties>
+
+	<dependencies>
+		<dependency>
+			<groupId>gr.iccs.imu.ems</groupId>
+			<artifactId>broker-cep</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>gr.iccs.imu.ems</groupId>
+			<artifactId>broker-client</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>gr.iccs.imu.ems</groupId>
+			<artifactId>common</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+
+		<!-- Spring-Boot dependencies -->
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring-web</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.github.ulisesbocchio</groupId>
+			<artifactId>jasypt-spring-boot-starter</artifactId>
+			<version>${jasypt.starter.version}</version>
+		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/org.apache.sshd/apache-sshd -->
+		<dependency>
+			<groupId>org.apache.sshd</groupId>
+			<artifactId>apache-sshd</artifactId>
+			<version>${apache-sshd.version}</version>
+			<type>pom</type>
+			<exclusions>
+				<exclusion> 
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-jdk14</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.bouncycastle</groupId>
+					<artifactId>*</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.springframework</groupId>
+					<artifactId>*</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.sshd</groupId>
+			<artifactId>sshd-scp</artifactId>
+			<version>${apache-sshd.version}</version>
+		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/org.projectlombok/lombok -->
+		<dependency>
+			<groupId>org.projectlombok</groupId>
+			<artifactId>lombok</artifactId>
+			<scope>provided</scope>
+		</dependency>
+
+		<!-- Atomix dependencies -->
+		<dependency>
+			<groupId>io.atomix</groupId>
+			<artifactId>atomix</artifactId>
+			<version>${atomix.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>com.google.guava</groupId>
+					<artifactId>guava</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>io.atomix</groupId>
+			<artifactId>atomix-raft</artifactId>
+			<version>${atomix.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>io.atomix</groupId>
+			<artifactId>atomix-primary-backup</artifactId>
+			<version>${atomix.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>io.atomix</groupId>
+			<artifactId>atomix-gossip</artifactId>
+			<version>${atomix.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>io.atomix</groupId>
+			<artifactId>atomix-storage</artifactId>
+			<version>${atomix.version}</version>
+		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava</artifactId>
+			<version>${guava.version}</version>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.codehaus.mojo</groupId>
+				<artifactId>exec-maven-plugin</artifactId>
+				<version>1.6.0</version>
+				<executions>
+					<execution>
+						<goals>
+							<goal>exec</goal>
+						</goals>
+					</execution>
+				</executions>
+				<configuration>
+					<mainClass>gr.iccs.imu.ems.baguette.client.BaguetteClient</mainClass>
+					<executable>maven</executable>
+				</configuration>
+			</plugin>
+
+			<!-- Assembly Maven plugin (https://maven.apache.org/plugin-developers/cookbook/generate-assembly.html) -->
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-assembly-plugin</artifactId>
+				<configuration>
+					<descriptors>
+						<descriptor>src/main/assembly/baguette-client-installation-package.xml</descriptor>
+					</descriptors>
+					<finalName>baguette-client</finalName>
+				</configuration>
+				<executions>
+					<execution>
+						<phase>package</phase>
+						<goals>
+							<goal>single</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+
+		</plugins>
+	</build>
+	
+</project>

ems-core/baguette-client/src/main/assembly/baguette-client-installation-package.xml

@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+  ~
+  ~ This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+  ~ Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+  ~ If a copy of the MPL was not distributed with this file, you can obtain one at
+  ~ https://www.mozilla.org/en-US/MPL/2.0/
+-->
+<assembly
+    xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="
+        http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
+        http://maven.apache.org/xsd/assembly-1.1.2.xsd"
+>
+    <id>installation-package</id>
+    <formats>
+        <format>tgz</format>
+    </formats>
+    <fileSets>
+        <fileSet>
+            <outputDirectory></outputDirectory>
+            <directory>${project.basedir}</directory>
+            <includes>
+                <include>README*</include>
+                <include>LICENSE*</include>
+                <include>INSTALLATION*</include>
+            </includes>
+            <lineEnding>unix</lineEnding>
+        </fileSet>
+        <fileSet>
+            <outputDirectory>bin</outputDirectory>
+            <directory>bin</directory>
+            <includes>
+                <include>*</include>
+            </includes>
+            <lineEnding>unix</lineEnding>
+            <fileMode>0755</fileMode>
+        </fileSet>
+        <!--<fileSet>
+            <outputDirectory>conf</outputDirectory>
+            <directory>conf</directory>
+            <includes>
+                <include>*</include>
+            </includes>
+            <lineEnding>unix</lineEnding>
+        </fileSet>-->
+        <fileSet>
+            <outputDirectory>logs</outputDirectory>
+            <directory>logs</directory>
+            <includes>
+                <include>*</include>
+            </includes>
+            <lineEnding>unix</lineEnding>
+        </fileSet>
+        <fileSet>
+            <outputDirectory>jars</outputDirectory>
+            <directory>${project.build.directory}</directory>
+            <includes>
+                <include>*.jar</include>
+            </includes>
+        </fileSet>
+        <fileSet>
+            <outputDirectory>jars/broker-client</outputDirectory>
+            <directory>${project.parent.basedir}/broker-client/target</directory>
+            <includes>
+                <include>broker-client-jar-with-dependencies.jar</include>
+            </includes>
+        </fileSet>
+        <!--<fileSet>
+            <outputDirectory>bin</outputDirectory>
+            <directory>${project.parent.basedir}/broker-client</directory>
+            <includes>
+                <include>client.*</include>
+            </includes>
+            <lineEnding>unix</lineEnding>
+            <fileMode>0755</fileMode>
+        </fileSet>-->
+        <fileSet>
+            <outputDirectory>bin</outputDirectory>
+            <directory>${project.parent.basedir}/bin</directory>
+            <includes>
+                <include>sysmon.*</include>
+            </includes>
+            <lineEnding>unix</lineEnding>
+            <fileMode>0755</fileMode>
+        </fileSet>
+    </fileSets>
+    <!-- use this section if you want to package dependencies -->
+    <dependencySets>
+        <dependencySet>
+            <outputDirectory>jars</outputDirectory>
+            <excludes>
+                <exclude>*:pom</exclude>
+            </excludes>
+            <useStrictFiltering>true</useStrictFiltering>
+            <useProjectArtifact>false</useProjectArtifact>
+            <scope>runtime</scope>
+        </dependencySet>
+    </dependencySets>
+</assembly>

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/BaguetteClient.java

@@ -0,0 +1,271 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client;
+
+import gr.iccs.imu.ems.baguette.client.cluster.ClusterManagerProperties;
+import gr.iccs.imu.ems.baguette.client.collector.netdata.NetdataCollector;
+//import prometheus.collector.gr.iccs.imu.ems.baguette.client.PrometheusCollector;
+import gr.iccs.imu.ems.baguette.client.plugin.recovery.SelfHealingPlugin;
+import gr.iccs.imu.ems.util.EventBus;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.NoSuchBeanDefinitionException;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Scope;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Baguette client
+ */
+@Slf4j
+@EnableScheduling
+@SpringBootApplication(scanBasePackages = {
+        "gr.iccs.imu.ems.baguette.client", "gr.iccs.imu.ems.brokercep", "gr.iccs.imu.ems.common",
+        "gr.iccs.imu.ems.brokerclient", "gr.iccs.imu.ems.util"})
+@RequiredArgsConstructor
+public class BaguetteClient implements ApplicationRunner {
+    @Getter
+    private final BaguetteClientProperties baguetteClientProperties;
+    private final ClusterManagerProperties clusterManagerProperties;
+    private final ConfigurableApplicationContext applicationContext;
+
+    private final List<Class<? extends Collector>> DEFAULT_COLLECTORS_LIST = List.of(
+        NetdataCollector.class//, PrometheusCollector.class
+    );
+
+    @Getter
+    private final List<Collector> collectorsList = new ArrayList<>();
+
+    private static int killDelay;
+
+    @Getter
+    private Sshc client;
+
+    public static void main(String[] args) {
+        SpringApplication.run(BaguetteClient.class, args);
+
+        forceExit();
+    }
+
+    @Bean
+    @Scope(value = ConfigurableBeanFactory.SCOPE_SINGLETON)
+    public EventBus<String,Object,Object> eventBus() {
+        return EventBus.<String,Object,Object>builder().build();
+    }
+
+    @Override
+    public void run(ApplicationArguments args) throws IOException {
+        log.debug("BaguetteClient: Starting");
+
+        // Process command line arguments
+        processCommandLineArgs(args);
+        killDelay = baguetteClientProperties.getKillDelay();
+        log.debug("BaguetteClient: configuration: {}", baguetteClientProperties);
+        log.debug("Cluster: configuration: {}", clusterManagerProperties);
+
+        boolean interactiveMode = args.containsOption("i");
+
+        // Start measurement collectors (but not in interactive mode)
+        if (!interactiveMode) {
+            startCollectors();
+            applicationContext.getBean(SelfHealingPlugin.class).start();
+        }
+
+        if (interactiveMode) {
+            // Run CLI
+            log.debug("BaguetteClient: Enters interactive mode");
+            runCli();
+        } else {
+            // Run SSH client
+            log.debug("BaguetteClient: Enters SSH mode");
+            runSshClient();
+        }
+        log.debug("BaguetteClient: Exiting");
+
+        // Stop measurement collectors
+        if (!interactiveMode) {
+            applicationContext.getBean(SelfHealingPlugin.class).stop();
+            stopCollectors();
+        }
+
+        // Stop Baguette Client services
+        applicationContext.close();
+
+        log.info("BaguetteClient: Bye");
+    }
+
+    private void processCommandLineArgs(ApplicationArguments args) {
+        // Get cluster node addresses and properties
+        List<String> addresses = args.getNonOptionArgs();
+        if (addresses!=null && addresses.size()>0) {
+            clusterManagerProperties.getLocalNode().setAddress(addresses.get(0));
+            if (addresses.size()>1) {
+                clusterManagerProperties.setMemberAddresses(addresses.subList(1, addresses.size()));
+            }
+        }
+
+        // Enable/Disable TLS
+        if (args.containsOption("tls"))
+            clusterManagerProperties.getTls().setEnabled(true);
+        if (args.containsOption("notls"))
+            clusterManagerProperties.getTls().setEnabled(false);
+    }
+
+    protected void startCollectors() {
+        if (!collectorsList.isEmpty())
+            throw new IllegalArgumentException("Collectors have already been started");
+
+        log.debug("BaguetteClient: Starting collectors...");
+        if (baguetteClientProperties.getCollectorClasses()==null)
+            baguetteClientProperties.setCollectorClasses(DEFAULT_COLLECTORS_LIST);
+        for (Class<? extends Collector> collectorClass : baguetteClientProperties.getCollectorClasses()) {
+            try {
+                log.debug("BaguetteClient: Starting collector: {}...", collectorClass.getName());
+                Collector collector = applicationContext.getBean(collectorClass);
+                collector.start();
+                collectorsList.add(collector);
+                log.debug("BaguetteClient: Starting collector: {}...ok", collectorClass.getName());
+            } catch (NoSuchBeanDefinitionException e) {
+                log.error("BaguetteClient: Exception while starting collector: {}: ", collectorClass.getName(), e);
+            }
+        }
+        log.debug("BaguetteClient: Starting collectors...ok");
+    }
+
+    protected void stopCollectors() {
+        log.debug("BaguetteClient: Stopping collectors...");
+        for (Collector collector : collectorsList) {
+            try {
+                log.debug("BaguetteClient: Stopping collector: {}...", collector.getClass().getName());
+                collector.stop();
+                log.debug("BaguetteClient: Stopping collector: {}...ok", collector.getClass().getName());
+            } catch (NoSuchBeanDefinitionException e) {
+                log.error("BaguetteClient: Exception while stopping collector: {}: ", collector.getClass().getName(), e);
+            }
+        }
+        collectorsList.clear();
+    }
+
+    protected void runSshClient() {
+        long retryDelay = baguetteClientProperties.getConnectionRetryDelay();
+        boolean retry = baguetteClientProperties.isConnectionRetryEnabled() && retryDelay>=0;
+        int retryLimit = baguetteClientProperties.getConnectionRetryLimit();
+        int retryCount = 0;
+        while (true) {
+            try {
+                // Connect to baguette server
+                startSshClient(retry);
+
+                // Exchange messages with Baguette server
+                log.trace("BaguetteClient: Calling SSHC run()");
+                client.run();
+                retryCount = 0;
+
+                // Disconnect from baguette server
+                stopSshClient();
+            } catch (Exception ex) {
+                log.error("BaguetteClient: EXCEPTION: ", ex);
+            }
+
+            // Check if retry is enabled
+            if (!retry) break;
+
+            // Check if retry limit has been reached
+            retryCount++;
+            if (retryLimit>=0 && retryCount > retryLimit) {
+                log.error("BaguetteClient: Giving up connection retries after {} failed attempts", retryCount-1);
+                break;
+            }
+
+            // Wait for a while before retrying to reconnect
+            try {
+                Thread.sleep(retryDelay);
+            } catch (InterruptedException e) {
+                log.warn("BaguetteClient: Cancelling connection retry");
+                break;
+            }
+            log.info("BaguetteClient: Retrying to connect (attempt #{})...", retryCount);
+        }
+    }
+
+    protected void runCli() throws IOException {
+        BaguetteClientCLI cli = applicationContext.getBean(BaguetteClientCLI.class);
+        cli.setConfiguration(baguetteClientProperties);
+        cli.run();
+    }
+
+    public synchronized void startSshClient(boolean retry) throws IOException {
+        log.trace("BaguetteClient: spring-boot application-context: {}", applicationContext);
+        client = applicationContext.getBean(Sshc.class);
+        client.setConfiguration(baguetteClientProperties);
+
+        log.trace("BaguetteClient: Sshc instance from application-context: {}", client);
+        log.trace("BaguetteClient: Calling SSHC start()");
+        client.start(retry);
+        client.greeting();
+    }
+
+    public synchronized void stopSshClient() throws IOException {
+        log.trace("BaguetteClient: Calling SSHC stop()");
+        Sshc tmp = client;
+        client = null;
+        tmp.stop();
+    }
+
+    /*protected static Properties loadConfig(String configFile) throws IOException {
+        Properties config = new Properties();
+        try {
+            try (InputStream in = new FileInputStream(new File(configFile))) {
+                config.load(in);
+            }
+        } catch (FileNotFoundException ex) {
+            try (InputStream in = BaguetteClient.class.getResourceAsStream(configFile)) {
+                if (in == null) throw ex;
+                config.load(in);
+            }
+        }
+        return config;
+    }*/
+
+    protected static void forceExit() {
+        // Print remaining threads
+        Thread.getAllStackTraces().keySet()
+                .forEach(s -> log.debug("---> {}.{}: {} alive={}, daemon={}, interrupted={}",
+                        s.getThreadGroup().getName(), s.getName(), s.getState(),
+                        s.isAlive(), s.isDaemon(), s.isInterrupted()));
+
+        // Start killer thread
+        if (killDelay>0) {
+            new Thread(() -> {
+                try { Thread.sleep(1000); } catch (InterruptedException ignored) { }
+                log.warn("Waiting JVM to exit for {} more seconds", killDelay);
+                try { Thread.sleep(killDelay * 1000); } catch (InterruptedException ignored) { }
+                log.warn("Forcing JVM to exit");
+                System.exit(0);
+            }) {{
+                setDaemon(true);
+                start();
+            }};
+        } else {
+            log.debug("Killer thread disabled");
+        }
+    }
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/BaguetteClientCLI.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client;
+
+import gr.iccs.imu.ems.baguette.client.cluster.ClusterManager;
+import gr.iccs.imu.ems.brokercep.BrokerCepService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.*;
+
+/**
+ * Baguette Client Command-Line Interface
+ */
+@Slf4j
+@Service
+public class BaguetteClientCLI {
+    private BaguetteClientProperties config;
+    private String clientId;
+    private String prompt = "CLI> ";
+
+    @Autowired
+    private CommandExecutor commandExecutor;
+    @Autowired
+    BrokerCepService brokerCepService;
+
+    public void setConfiguration(BaguetteClientProperties config) {
+        this.config = config;
+        this.clientId = config.getClientId();
+        if (StringUtils.isNotBlank(clientId))
+            prompt = "CLI-"+ ClusterManager.getLocalHostName()+" > ";
+        config.setExitCommandAllowed(true);
+        log.trace("Sshc: cmd-exec: {}", commandExecutor);
+        this.commandExecutor.setConfiguration(config);
+    }
+
+    public void run() throws IOException {
+        run(System.in, System.out, System.err);
+    }
+
+    public void run(InputStream in, PrintStream out, PrintStream err) throws IOException {
+        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+        out.print(prompt);
+        out.flush();
+        String line;
+        while ((line = reader.readLine()) != null) {
+            line = line.trim();
+            try {
+                boolean exit = commandExecutor.execCmd(line.split("[ \t]+"), in, out, err);
+                if (exit) break;
+            } catch (Exception ex) {
+                ex.printStackTrace(out);
+                out.flush();
+            }
+            out.print(prompt);
+            out.flush();
+        }
+    }
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/BaguetteClientProperties.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client;
+
+import gr.iccs.imu.ems.common.client.SshClientProperties;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.PropertySource;
+
+import java.util.List;
+
+@Data
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@Configuration
+@ConfigurationProperties
+@PropertySource(value = {
+		"file:${EMS_CONFIG_DIR}/ems-client.yml",
+		"file:${EMS_CONFIG_DIR}/ems-client.properties",
+		"file:${EMS_CONFIG_DIR}/baguette-client.yml",
+		"file:${EMS_CONFIG_DIR}/baguette-client.properties"
+}, ignoreResourceNotFound = true)
+public class BaguetteClientProperties extends SshClientProperties {
+	private String baseDir;
+
+	private boolean connectionRetryEnabled = true;
+	private long connectionRetryDelay = 10 * 1000L;
+	private int connectionRetryLimit = -1;
+
+	private boolean exitCommandAllowed = false;
+	private int killDelay = 5;
+
+	private List<Class<? extends Collector>> collectorClasses;
+
+	private String debugFakeIpAddress;
+
+	private long sendStatisticsDelay = 10000L;
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/Collector.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client;
+
+import gr.iccs.imu.ems.util.Plugin;
+
+public interface Collector extends Plugin {
+    void activeGroupingChanged(String oldGrouping, String newGrouping);
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/Sshc.java

@@ -0,0 +1,284 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client;
+
+import gr.iccs.imu.ems.brokercep.BrokerCepService;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringEscapeUtils;
+import org.apache.sshd.client.SshClient;
+import org.apache.sshd.client.channel.ClientChannel;
+import org.apache.sshd.client.config.hosts.HostConfigEntryResolver;
+import org.apache.sshd.client.keyverifier.ServerKeyVerifier;
+import org.apache.sshd.client.session.ClientSession;
+import org.apache.sshd.client.simple.SimpleClient;
+import org.apache.sshd.common.PropertyResolverUtils;
+import org.apache.sshd.common.config.keys.KeyUtils;
+import org.apache.sshd.core.CoreModuleProperties;
+import org.apache.sshd.mina.MinaServiceFactoryFactory;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.openssl.PEMParser;
+import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.*;
+import java.security.PublicKey;
+import java.util.Optional;
+
+
+/**
+ * Custom SSH client
+ */
+@Slf4j
+@Service
+public class Sshc implements gr.iccs.imu.ems.common.client.SshClient<BaguetteClientProperties> {
+    private BaguetteClientProperties config;
+    private SshClient client;
+    private SimpleClient simple;
+    private ClientSession session;
+    private ClientChannel channel;
+    private boolean started = false;
+    @Autowired
+    private CommandExecutor commandExecutor;
+    @Autowired
+    private BrokerCepService brokerCepService;
+
+    @Getter
+    private InputStream in;
+    @Getter
+    private PrintStream out;
+    @Getter
+    private PrintStream err;
+    @Getter
+    private String clientId;
+
+    @Getter @Setter
+    private boolean useServerKeyVerifier = true;
+
+    @Override
+    public void setConfiguration(BaguetteClientProperties config) {
+        log.trace("Sshc: New config: {}", config);
+        this.config = config;
+        this.clientId = config.getClientId();
+        log.trace("Sshc: cmd-exec: {}", commandExecutor);
+        if (this.commandExecutor!=null) this.commandExecutor.setConfiguration(config);
+    }
+
+    public synchronized void start(boolean retry) throws IOException {
+        if (retry) {
+            log.trace("Starting client in retry mode");
+            long retryPeriod = config.getRetryPeriod();
+            while (!started) {
+                log.debug("(Re-)trying to start client....");
+                try {
+                    start();
+                } catch (Exception ex) {
+                    log.warn("{}", ex.getMessage());
+                }
+                if (started) break;
+                log.trace("Failed to start. Sleeping for {}ms...", retryPeriod);
+                try {
+                    Thread.sleep(retryPeriod);
+                } catch (InterruptedException ex) {
+                    log.debug("Sleep: ", ex);
+                }
+            }
+        } else {
+            start();
+        }
+        if (started) log.trace("Client started");
+    }
+
+    @Override
+    public synchronized void start() throws IOException {
+        if (started) return;
+        log.info("Connecting to server...");
+
+        String host = config.getServerAddress();
+        int port = config.getServerPort();
+        String serverPubKey = StringEscapeUtils.unescapeJson(config.getServerPubkey());
+        String serverPubkeyFingerprint = config.getServerPubkeyFingerprint();
+        String serverPubKeyAlgorithm = config.getServerPubkeyAlgorithm();
+        String serverPubKeyFormat = config.getServerPubkeyFormat();
+        String username = config.getServerUsername();
+        String password = config.getServerPassword();
+        long connectTimeout = config.getConnectTimeout();
+        long authTimeout = config.getAuthTimeout();
+        long heartbeatInterval = config.getHeartbeatInterval();
+        long heartbeatReplyWait = config.getHeartbeatReplyWait();
+
+        // Starting client and connecting to server
+        this.client = SshClient.setUpDefaultClient();
+        client.setHostConfigEntryResolver(HostConfigEntryResolver.EMPTY);
+
+        if (useServerKeyVerifier) {
+            // Get configured server public key
+            PublicKey pubKey = getPublicKeyFromString(serverPubKeyAlgorithm, serverPubKeyFormat, serverPubKey);
+
+            // Provided server key verifiers
+            //client.setServerKeyVerifier(AcceptAllServerKeyVerifier.INSTANCE);
+            //client.setServerKeyVerifier(new RequiredServerKeyVerifier(pubKey));
+
+            // Custom server key verifier
+            client.setServerKeyVerifier( getCustomServerKeyVerifier(serverPubkeyFingerprint, pubKey) );
+        }
+
+        this.simple = SshClient.wrapAsSimpleClient(client);
+        //simple.setConnectTimeout(connectTimeout);
+        //simple.setAuthenticationTimeout(authTimeout);
+
+        // Set a huge idle timeout, keep-alive to true and heartbeat to configured value
+        PropertyResolverUtils.updateProperty(client, CoreModuleProperties.HEARTBEAT_INTERVAL.getName(), heartbeatInterval);      // Prevents server-side connection closing
+        PropertyResolverUtils.updateProperty(client, CoreModuleProperties.HEARTBEAT_REPLY_WAIT.getName(), heartbeatReplyWait);   // Prevents client-side connection closing
+        PropertyResolverUtils.updateProperty(client, CoreModuleProperties.IDLE_TIMEOUT.getName(), Integer.MAX_VALUE);
+        PropertyResolverUtils.updateProperty(client, CoreModuleProperties.SOCKET_KEEPALIVE.getName(), true);               // Socket keep-alive at OS-level
+        log.debug("Set IDLE_TIMEOUT to MAX, SOCKET-KEEP-ALIVE to true, and HEARTBEAT to {}", heartbeatInterval);
+
+        // Explicitly set IO service factory factory to prevent conflict between MINA and Netty options
+        client.setIoServiceFactoryFactory(new MinaServiceFactoryFactory());
+
+        // Start SSH client
+        client.start();
+
+        // Authenticate and start session
+        this.session = client.connect(username, host, port)
+                .verify(connectTimeout)
+                .getSession();
+        session.addPasswordIdentity(password);
+        session.auth()
+                .verify(authTimeout);
+
+        // Open command shell channel
+        this.channel = session.createChannel(ClientChannel.CHANNEL_SHELL);
+        PipedInputStream pIn = new PipedInputStream();
+        PipedOutputStream pOut = new PipedOutputStream();
+        //PipedOutputStream pErr = new PipedOutputStream();
+        this.in = new BufferedInputStream(pIn);
+        this.out = new PrintStream(pOut, true);
+        //this.err = new PrintStream(pErr, true);
+
+        channel.setIn(new PipedInputStream(pOut));
+        channel.setOut(new PipedOutputStream(pIn));
+        //channel.setErr(new PipedOutputStream(pErr));
+
+        channel.open();
+
+        log.info("SSH client is ready");
+        this.started = true;
+    }
+
+    private static ServerKeyVerifier getCustomServerKeyVerifier(String serverPubkeyFingerprint, PublicKey pubKey) {
+        return (clientSession, remoteAddress, publicKey) -> {
+            // boolean verifyServerKey(ClientSession clientSession, SocketAddress socketAddress, PublicKey publicKey)
+            log.info("verifyServerKey(): remoteAddress: {}", remoteAddress.toString());
+
+            // Check server public key fingerprint matches with the one in configuration
+            if (StringUtils.isNoneBlank(serverPubkeyFingerprint)) {
+                String fingerprint = KeyUtils.getFingerPrint(publicKey);
+                log.debug("verifyServerKey(): publicKey: fingerprint: {}", fingerprint);
+                if (fingerprint != null && KeyUtils.checkFingerPrint(serverPubkeyFingerprint, publicKey).getKey() != null)
+                    log.debug("verifyServerKey(): publicKey: fingerprint: MATCH");
+                else
+                    log.warn("verifyServerKey(): publicKey: fingerprint: NO MATCH");
+            }
+
+            // Check that server public key matches with the one in configuration
+            try {
+                // Compare session provided and configured public keys
+                log.debug("verifyServerKey(): configured server public key: {}", pubKey);
+                log.debug("verifyServerKey():   received server public key: {}", publicKey);
+                boolean match = KeyUtils.compareKeys(pubKey, publicKey);
+                log.debug("verifyServerKey(): Server keys match? {}", match);
+                return match;
+            } catch (Exception e) {
+                log.error("verifyServerKey(): publicKey: EXCEPTION: ", e);
+                return false;
+            }
+        };
+    }
+
+    private static PublicKey getPublicKeyFromString(String serverPubKeyAlgorithm, String serverPubKeyFormat, String serverPubKey) throws IOException {
+        log.debug("getPublicKeyFromString(): serverPubKeyAlgorithm: {}", serverPubKeyAlgorithm);
+        log.debug("getPublicKeyFromString():    serverPubKeyFormat: {}", serverPubKeyFormat);
+        log.debug("getPublicKeyFromString():          serverPubKey:\n{}", serverPubKey);
+
+        // Retrieve configured public key - First implementation
+        PEMParser pemParser = new PEMParser(new StringReader(serverPubKey));
+        PemObject pemObject = pemParser.readPemObject();
+        JcaPEMKeyConverter converter = new JcaPEMKeyConverter();
+        SubjectPublicKeyInfo publicKeyInfo = SubjectPublicKeyInfo.getInstance(pemObject.getContent());
+        PublicKey pubKey = converter.getPublicKey(publicKeyInfo);
+
+        // Retrieve configured public key - Alternative implementation
+        /*KeyFactory factory = KeyFactory.getInstance(serverPubKeyAlgorithm);
+        PublicKey pubKey;
+        try (StringReader keyReader = new StringReader(serverPubKey);
+             PemReader pemReader = new PemReader(keyReader))
+        {
+            PemObject pemObject = pemReader.readPemObject();
+            byte[] content = pemObject.getContent();
+            X509EncodedKeySpec pubKeySpec = new X509EncodedKeySpec(content);
+            //or PKCS8EncodedKeySpec pubKeySpec = new PKCS8EncodedKeySpec(content);
+            pubKey = factory.generatePublic(pubKeySpec);
+        }*/
+
+        log.debug("getPublicKeyFromString: Public key: {}", pubKey);
+        return pubKey;
+    }
+
+    @Override
+    public synchronized void stop() throws IOException {
+        if (!started) return;
+        this.started = false;
+        log.info("Stopping SSH client...");
+
+        channel.close(false).await();
+        session.close(false);
+        simple.close();
+        client.stop();
+
+        log.info("SSH client stopped");
+    }
+
+    public synchronized void greeting() {
+        if (!started) return;
+        String certOneLine = Optional
+                .ofNullable(brokerCepService.getBrokerCertificate())
+                .orElse("")
+                .replace(" ","~~")
+                .replace("\r\n","##")
+                .replace("\n","$$");
+        String clientAddress = config.getDebugFakeIpAddress();
+        int clientPort = -1;
+        out.printf("-HELLO FROM CLIENT: id=%s broker=%s address=%s port=%d username=%s password=%s cert=%s%n",
+                clientId.replace(" ", "~~"),
+                brokerCepService.getBrokerCepProperties().getBrokerUrlForClients(),
+                StringUtils.isNotBlank(clientAddress) ? clientAddress : "",
+                clientPort,
+                brokerCepService.getBrokerUsername(),
+                brokerCepService.getBrokerPassword(),
+                certOneLine);
+        out.flush();
+    }
+
+    public void run() throws IOException {
+        if (!started) return;
+
+        // Start communication protocol with Server
+        // Execution waits here until connection is closed
+        log.trace("run(): Calling communicateWithServer()...");
+        commandExecutor.communicateWithServer(in, out, out);
+        out.printf("-BYE FROM CLIENT: %s%n", clientId);
+    }
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/cluster/AbstractLogBase.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.cluster;
+
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.slf4j.helpers.MessageFormatter;
+
+import java.io.*;
+
+@Data
+@Slf4j
+public abstract class AbstractLogBase {
+    protected final static Object[] EMPTY_OBJECT_ARRAY = new Object[0];
+
+    @Getter(AccessLevel.NONE)
+    @Setter(AccessLevel.NONE)
+    private BufferedReader rIn = new BufferedReader(new InputStreamReader(System.in));
+    private InputStream in = System.in;
+    private PrintStream out = System.out;
+    private PrintStream err = System.err;
+    private boolean logEnabled = true;
+    private boolean outEnabled = true;
+
+    public void setIn(InputStream in) { this.in = in; this.rIn = new BufferedReader(new InputStreamReader(in)); }
+
+    protected String readLine(String prompt) throws IOException {
+        out.print(prompt);
+        out.flush();
+        return rIn.readLine();
+    }
+
+    protected void log_trace(String formatter, Object...args) {
+        if (log.isTraceEnabled()) {
+            if (logEnabled) log.trace(formatter, args);
+            if (outEnabled) out.println(MessageFormatter.arrayFormat(formatter, args).getMessage());
+        }
+    }
+
+    protected void log_debug(String formatter, Object...args) {
+        if (log.isDebugEnabled()) {
+            if (logEnabled) log.debug(formatter, args);
+            if (outEnabled) out.println(MessageFormatter.arrayFormat(formatter, args).getMessage());
+        }
+    }
+
+    protected void log_info(String formatter, Object...args) {
+        if (log.isInfoEnabled()) {
+            if (logEnabled) log.info(formatter, args);
+            if (outEnabled) out.println(MessageFormatter.arrayFormat(formatter, args).getMessage());
+        }
+    }
+
+    protected void log_warn(String formatter, Object...args) {
+        if (log.isWarnEnabled()) {
+            if (logEnabled) log.warn(formatter, args);
+            if (outEnabled) out.println(MessageFormatter.arrayFormat(formatter, args).getMessage());
+        }
+    }
+
+    protected void log_error(String formatter) {
+        if (log.isErrorEnabled()) {
+            if (logEnabled) log.error(formatter);
+            if (outEnabled) err.println(MessageFormatter.arrayFormat(
+                    formatter, EMPTY_OBJECT_ARRAY, null).getMessage());
+        }
+    }
+
+    protected void log_error(String formatter, Object...args) {
+        if (log.isErrorEnabled()) {
+            if (logEnabled) log.error(formatter, args);
+            if (outEnabled) err.println(MessageFormatter.arrayFormat(formatter, args).getMessage());
+        }
+    }
+
+    protected void log_error(String formatter, Exception ex) {
+        if (log.isErrorEnabled()) {
+            if (logEnabled) log.error(formatter, ex);
+            if (outEnabled) {
+                err.print(MessageFormatter.arrayFormat(
+                        formatter, EMPTY_OBJECT_ARRAY, ex).getMessage());
+                ex.printStackTrace(err);
+            }
+        }
+    }
+
+    protected void out_print(String formatter, Object...args) { stream_print(out, false, formatter, args); }
+    protected void out_println(String formatter, Object...args) { stream_print(out, true, formatter, args); }
+    protected void out_println() { stream_print(out, true, "", (Object)null); }
+    protected void err_print(String formatter, Object...args) { stream_print(err, false, formatter, args); }
+    protected void err_println(String formatter, Object...args) { stream_print(err, true, formatter, args); }
+    protected void err_println() { stream_print(err, true, "", (Object)null); }
+
+    protected void stream_print(PrintStream stream, boolean nl, String formatter, Object...args) {
+        if (outEnabled) {
+            String message = MessageFormatter.arrayFormat(formatter, args).getMessage();
+            if (nl)
+                stream.println(message);
+            else
+                stream.print(message);
+            stream.flush();
+        }
+    }
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/cluster/BrokerUtil.java

@@ -0,0 +1,437 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.cluster;
+
+import io.atomix.cluster.ClusterMembershipEvent;
+import io.atomix.cluster.Member;
+import io.atomix.core.Atomix;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import static gr.iccs.imu.ems.baguette.client.cluster.BrokerUtil.NODE_STATUS.*;
+
+@RequiredArgsConstructor
+public class BrokerUtil extends AbstractLogBase {
+    public enum NODE_STATUS { AGGREGATOR, CANDIDATE, NOT_CANDIDATE, INITIALIZING, STEPPING_DOWN, RETIRING, NOT_SET }
+
+    protected final static Collection<NODE_STATUS> BROKER_STATUSES = Arrays.asList(AGGREGATOR, RETIRING);
+    protected final static Collection<NODE_STATUS> CANDIDATE_STATUSES = Arrays.asList(CANDIDATE, AGGREGATOR, INITIALIZING);
+    protected final static Collection<NODE_STATUS> NON_CANDIDATE_STATUSES = Arrays.asList(NOT_CANDIDATE, STEPPING_DOWN, RETIRING, NOT_SET);
+
+    public final static String NODE_MESSAGE_TOPIC = "NODE-MESSAGE-TOPIC";
+    public final static String STATUS_PROPERTY = "node-status";
+
+    protected final static String MESSAGE_ELECTION = "election";
+    protected final static String MESSAGE_APPOINT = "appoint";
+    protected final static String MESSAGE_INITIALIZE = "initialize";
+    protected final static String MESSAGE_READY = "ready";
+    private static final String MARKER_NEW_CONFIGURATION = "New config: ";
+
+    private final Atomix atomix;
+    private final ClusterManager clusterManager;
+    private final AtomicBoolean backOff = new AtomicBoolean();
+
+    @Getter @Setter
+    private NodeCallback callback;
+
+    public BrokerUtil(ClusterManager clusterManager, NodeCallback callback) {
+        this.clusterManager = clusterManager;
+        this.atomix = clusterManager.getAtomix();
+        this.callback = callback;
+    }
+
+    void processBrokerMessage(Object m) {
+        if (m == null) return;
+        String message = m.toString();
+        log_info("BRU: **** Broker message received: {}", message);
+
+        String messageType = message.split(" ", 2)[0];
+        if (MESSAGE_ELECTION.equalsIgnoreCase(messageType)) {
+            // Get excluded nodes (if any)
+            List<String> excludes = Arrays.stream(message.split(" "))
+                    .filter(StringUtils::isNotBlank)
+                    .map(String::trim)
+                    .filter(s -> s.startsWith("-"))
+                    .map(s -> s.substring(1))
+                    .collect(Collectors.toList());
+            // Start election
+            log_info("BRU: **** BROKER: Starting Broker election: ");
+            election(excludes);
+        } else if (MESSAGE_APPOINT.equalsIgnoreCase(messageType)) {
+            String newBrokerId = message.split(" ", 2)[1];
+            appointment(newBrokerId);
+        } else if (MESSAGE_INITIALIZE.equalsIgnoreCase(messageType)) {
+            String newBrokerId = message.split(" ", 2)[1];
+            log_info("BRU: **** BROKER: New Broker initializes: {}", newBrokerId);
+            // Back off if i am also initializing but have a lower score or command order
+            backOff();
+        } else if (MESSAGE_READY.equalsIgnoreCase(messageType)) {
+            String[] part = message.split(" ", 3);
+            String brokerId = part[1];
+            String newConfig = part[2];
+            // Strip 'New config.' marker
+            if (newConfig.startsWith(MARKER_NEW_CONFIGURATION)) {
+                newConfig = newConfig.substring(MARKER_NEW_CONFIGURATION.length()).trim();
+            } else {
+                log_error("BRU: !!!!  BUG: New configuration not properly marked: {}  !!!!", newConfig);
+            }
+            log_info("BRU: **** BROKER: New Broker is ready: {}, New config: {}", brokerId, newConfig);
+
+            // If i am not the new Broker then reset broker status
+            Member local = getLocalMember();
+            NODE_STATUS localStatus = getLocalStatus();
+            log_debug("BRU: Nodes: local={}, broker={}", local.id().id(), brokerId);
+            if (BROKER_STATUSES.contains(localStatus))
+                if (!local.id().id().equals(brokerId)) {
+                    // Temporarily make node unavailable for being elected as Broker, until step down completes
+                    setLocalStatus(STEPPING_DOWN);
+
+                    // Step down
+                    log_info("BRU: Old broker steps down: {}", local.id().id());
+                    if (callback!=null)
+                        callback.stepDown();
+
+                    // After step down, and if node hasn't retired, node status changes to 'candidate'
+                    if (RETIRING!=localStatus)
+                        setLocalStatus(CANDIDATE);
+                    else
+                        setLocalStatus(NOT_CANDIDATE);
+                }
+
+            // Pass new configuration to callback
+            log_info("BRU: Node configuration updated: {}", newConfig);
+            if (callback!=null) {
+                callback.setConfiguration(newConfig);
+            }
+        } else
+            log_warn("BRU:    BROKER: Unknown message received: {}", message);
+    }
+
+    private void aggregatorStepDown() {
+        // Save previous status
+        NODE_STATUS oldStatus = getLocalStatus();
+
+        // Temporarily make node unavailable for being elected as Aggregator, until step down completes
+        setLocalStatus(STEPPING_DOWN);
+
+        switch (oldStatus) {
+            case CANDIDATE:
+                log_debug("BRU: Node is not Aggregator. Clearing back-off flag");
+                backOff.set(false); break;
+            case INITIALIZING:
+                log_debug("BRU: Node is initializing. Back-off flag set");
+                backOff.set(true); break;
+            case AGGREGATOR:
+                // Step down
+                log_info("BRU: Aggregator steps down: {}", getLocalMember().id().id());
+                if (callback!=null)
+                    callback.stepDown();
+                backOff.set(false);
+                log_info("BRU: Old aggregator stepped down");
+                break;
+            case STEPPING_DOWN:
+                log_debug("stepDown(): Node is already stepping down. Nothing to do");
+                backOff.set(false);
+                break;
+        }
+
+        // After step down, and if node hasn't retired, node status changes to 'candidate'
+        if (oldStatus!=RETIRING)
+            setLocalStatus(CANDIDATE);
+        else
+            setLocalStatus(NOT_CANDIDATE);
+    }
+
+    public void backOff() {
+        NODE_STATUS state = getLocalStatus();
+        if (state==INITIALIZING) {
+            log_debug("BRU: Set Back-off flag to step down after initialization");
+            backOff.set(true);
+        } else
+        if (state==AGGREGATOR) {
+            log_debug("BRU: Stepping down because Back-off flag has been set");
+            aggregatorStepDown();
+        }
+    }
+
+    public boolean isBackOffSet() {
+        return backOff.get();
+    }
+
+    public void startElection() {
+        log_info("BRU: Broker election requested: broadcasting election message...");
+        atomix.getCommunicationService().broadcastIncludeSelf(NODE_MESSAGE_TOPIC, MESSAGE_ELECTION);
+    }
+
+    public void election(List<String> excludeNodes) {
+        // Find the new Brokering node
+        if (excludeNodes == null) excludeNodes = Collections.emptyList();
+        final List<String> excludes = excludeNodes;
+        Member broker = atomix.getMembershipService().getMembers().stream()
+                .filter(m -> m.isActive() && m.isReachable())
+                .filter(m -> !excludes.contains(m.id().id()))
+                .filter(m -> CANDIDATE_STATUSES.contains(getNodeStatus(m)))
+                .map(m -> new MemberWithScore(m, clusterManager.getScoreFunction()))
+                .peek(ms -> log_info("BRU: Member-Score: {} => {}  {}", ms.getMember().id().id(), ms.getScore(),
+                        ms.getMember().properties().getProperty("uuid", null)))
+                .max(MemberWithScore::compareTo)
+                .orElse(MemberWithScore.NULL_MEMBER)
+                .getMember();
+        log_info("BRU: Broker: {}", broker != null ? broker.id().id() : null);
+
+        // If local node is the selected broker...
+        if (getLocalMember().equals(broker)) {
+            appointment(broker.id().id());
+        }
+    }
+
+    private void appointment(String appointedNodeId) {
+        // Check i am appointed
+        Member local = getLocalMember();
+        if (! local.id().id().equals(appointedNodeId)) {
+            log_debug("BRU: I am not appointed: me={} <> appointed={}", local.id().id(), appointedNodeId);
+            return;
+        }
+
+        // Check if i am already a broker
+        NODE_STATUS localStatus = getLocalStatus();
+        if (BROKER_STATUSES.contains(localStatus)) {
+            if (localStatus==RETIRING) {
+                log_error("BRU: !!!! BUG: RETIRING AGGREGATOR HAS BEEN ELECTED AGAIN !!!!");
+            } else {
+                log_info("BRU: Aggregator elected again");
+            }
+        } else {
+            // Start initializing as Broker...
+            aggregatorInitialize();
+        }
+
+        // Notify others that this node is ready to serve as Aggregator
+        String brokerId = local.id().id();
+        String newConf = MARKER_NEW_CONFIGURATION +
+                (callback!=null ? callback.getConfiguration(local) : "");
+        atomix.getCommunicationService().broadcastIncludeSelf(NODE_MESSAGE_TOPIC, MESSAGE_READY + " " + brokerId + " " + newConf);
+    }
+
+    private void aggregatorInitialize() {
+        if (backOff.getAndSet(false)) {
+            log_warn("BRU: Node cannot be initialized as Aggregator. Back off flag is set");
+            return;
+        }
+
+        // Notify others that this node starts initializing as Broker
+        log_info("BRU: Node will become Broker. Initializing...");
+        atomix.getCommunicationService().broadcast(NODE_MESSAGE_TOPIC, MESSAGE_INITIALIZE + " " + getLocalMember().id().id());
+        setLocalStatus(INITIALIZING);
+
+        // Start initializing as Aggregator...
+        if (callback!=null)
+            callback.initialize();
+
+        // Update node status to Broker
+        setLocalStatus(AGGREGATOR);
+        log_info("BRU: Node is ready to act as Aggregator. Ready");
+
+        if (backOff.getAndSet(false)) {
+            log_debug("initialize(): Back-off flag has been set. Stepping down immediately.");
+            aggregatorStepDown();
+        }
+    }
+
+    public void appoint(String brokerId) {
+        // Check if already a broker
+        if (getBrokers().stream().anyMatch(m -> m.id().id().equals(brokerId))) {
+            log_info("BRU: Node is already a broker: {}", brokerId);
+            if (getNodeStatus(brokerId)==RETIRING)
+                setNodeStatus(brokerId, AGGREGATOR);
+            return;
+        }
+
+        // Check if not a candidate
+        NODE_STATUS brokerStatus = getNodeStatus(brokerId);
+        log_debug("BRU: Node status: {}", brokerStatus);
+        if (NON_CANDIDATE_STATUSES.contains(brokerStatus)) {
+            log_info("BRU: Node is not a broker candidate: {}", brokerId);
+            return;
+        }
+
+        // Broadcast appointment message
+        atomix.getCommunicationService().broadcastIncludeSelf(NODE_MESSAGE_TOPIC, MESSAGE_APPOINT + " " + brokerId);
+        log_info("BRU: Broker appointment broadcast: {}", brokerId);
+    }
+
+    public void retire() {
+        NODE_STATUS localStatus = getLocalStatus();
+        if (BROKER_STATUSES.contains(localStatus)) {
+            if (localStatus==RETIRING) {
+                log_info("BRU: Already retiring");
+            } else {
+                setLocalStatus(RETIRING);
+                log_info("BRU: Broker retires: broadcasting election message...");
+                String localNodeId = getLocalMember().id().id();
+                atomix.getCommunicationService().broadcast(NODE_MESSAGE_TOPIC, MESSAGE_ELECTION + " -" + localNodeId);
+                //election(Collections.singletonList(localNodeId));
+            }
+        } else
+            log_info("BRU: Not an Aggregator");
+    }
+
+    public List<Member> getBrokers() {
+        return atomix.getMembershipService().getMembers().stream()
+                .filter(m -> m.isActive() && m.isReachable())
+                .filter(m -> BROKER_STATUSES.contains(getNodeStatus(m)))
+                .collect(Collectors.toList());
+    }
+
+    public Member getLocalMember() {
+        return atomix.getMembershipService().getLocalMember();
+    }
+
+    public NODE_STATUS getLocalStatus() {
+        return getNodeStatus(getLocalMember());
+    }
+
+    public void setLocalStatus(@NonNull NODE_STATUS status) {
+        setNodeStatus(getLocalMember(), status);
+    }
+
+    public NODE_STATUS getNodeStatus(@NonNull Member member) {
+        return NODE_STATUS.valueOf(member.properties().getProperty(STATUS_PROPERTY, NOT_SET.name()));
+    }
+
+    public void setNodeStatus(@NonNull Member member, @NonNull NODE_STATUS status) {
+        log_trace("BRU: setNodeStatus: Node properties BEFORE CHANGE: {}", member.properties());
+        String oldStatusName = (String) member.properties().setProperty(STATUS_PROPERTY, status.name());
+        log_trace("BRU: setNodeStatus: Node properties AFTER CHANGE:  {}", member.properties());
+        log_debug("BRU: setNodeStatus: Status changed: {} --> {}", oldStatusName, status);
+        NODE_STATUS oldStatus = StringUtils.isNotBlank(oldStatusName) ? NODE_STATUS.valueOf(oldStatusName) : null;
+        if (callback!=null & oldStatus!=status)
+            callback.statusChanged(oldStatus, status);
+    }
+
+    public NODE_STATUS getNodeStatus(@NonNull String memberId) {
+        Member member = getMemberById(memberId);
+        if (member != null)
+            return getNodeStatus(member);
+        return null;
+    }
+
+    public void setNodeStatus(@NonNull String memberId, @NonNull NODE_STATUS status) {
+        Member member = getMemberById(memberId);
+        if (member != null)
+            setNodeStatus(member, status);
+    }
+
+    private Member getMemberById(@NonNull String id) {
+        return atomix.getMembershipService().getMembers().stream()
+                .filter(m -> m.isActive() && m.isReachable())
+                .filter(m -> m.id().id().equals(id))
+                .findFirst()
+                .orElse(null);
+    }
+
+    public void setCandidate() {
+        NODE_STATUS localStatus = getLocalStatus();
+        if (localStatus==NOT_CANDIDATE || localStatus==NOT_SET) {
+            setLocalStatus(CANDIDATE);
+            log_info("BRU: Node becomes Aggregator candidate");
+        } else
+            log_info("BRU: Node is already Aggregator candidate");
+    }
+
+    public void clearCandidate() {
+        NODE_STATUS localStatus = getLocalStatus();
+        if (BROKER_STATUSES.contains(localStatus)) {
+            log_warn("BRU: Node is the Aggregator. Select 'retire' first");
+            return;
+        }
+        if (localStatus==INITIALIZING) {
+            log_warn("BRU: Node is initializing for Aggregator. Step down first");
+            return;
+        }
+        if (localStatus==STEPPING_DOWN) {
+            log_warn("BRU: Node is stepping down. Wait step down complete");
+            return;
+        }
+        if (localStatus==CANDIDATE) {
+            setLocalStatus(NOT_CANDIDATE);
+            log_info("BRU: Node removed from Broker candidates");
+        } else
+            log_info("BRU: Node is not Aggregator candidate");
+    }
+
+    public List<MemberWithScore> getCandidates() {
+        return atomix.getMembershipService().getMembers().stream()
+                .filter(m -> m.isActive() && m.isReachable())
+                .filter(m -> CANDIDATE_STATUSES.contains(getNodeStatus(m)))
+                .map(m -> new MemberWithScore(m, clusterManager.getScoreFunction()))
+                .collect(Collectors.toList());
+    }
+
+    public List<MemberWithScore> getActiveNodes() {
+        return atomix.getMembershipService().getMembers().stream()
+                .filter(m -> m.isActive() && m.isReachable())
+                .map(m -> new MemberWithScore(m, clusterManager.getScoreFunction()))
+                .collect(Collectors.toList());
+    }
+
+    public void checkBroker() {
+        List<Member> brokers = getBrokers();
+        log_info("BRU: Brokers after cluster change: {}", brokers);
+
+        // Check if any node is initializing as broker (then don't start election)
+        if (getActiveNodes().stream()
+                .map(MemberWithScore::getMember)
+                .map(this::getNodeStatus)
+                .noneMatch(s -> INITIALIZING==s || AGGREGATOR==s))
+        {
+            startElection();
+        }
+    }
+
+    public void checkBrokerNumber() {
+        List<Member> brokers = getBrokers();
+        log_debug("BRU: Check number of Brokers in cluster: {}", brokers);
+
+        // Check if there are more than one brokers in cluster
+        long numOfBrokers = getActiveNodes().stream()
+                .map(MemberWithScore::getMember)
+                .map(this::getNodeStatus)
+                .filter(s -> AGGREGATOR==s)
+                .count();
+        log_info("BRU: Number of Brokers in cluster: {}", numOfBrokers);
+        if (numOfBrokers>1) {
+            log_warn("BRU: {} brokers found in the cluster. Starting election...", numOfBrokers);
+            startElection();
+        }
+    }
+
+    public interface NodeCallback {
+        void joinedCluster();
+        void leftCluster();
+
+        void initialize();
+        void stepDown();
+        void statusChanged(NODE_STATUS oldStatus, NODE_STATUS newStatus);
+        void clusterChanged(ClusterMembershipEvent event);
+        String getConfiguration(Member local);
+        void setConfiguration(String newConfig);
+    }
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/cluster/ClusterCLI.java

@@ -0,0 +1,228 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.cluster;
+
+import io.atomix.cluster.Member;
+import io.atomix.cluster.MemberId;
+import io.atomix.cluster.messaging.ClusterCommunicationService;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.*;
+import java.util.concurrent.CompletableFuture;
+
+@RequiredArgsConstructor
+public class ClusterCLI extends AbstractLogBase {
+
+	private final ClusterManager clusterManager;
+
+	@Getter @Setter
+	private String prompt = " -> ";
+	@Getter @Setter
+	private boolean promptUpdate;
+
+	public void updatePrompt() {
+		if (promptUpdate) {
+			setPrompt((clusterManager != null && clusterManager.isRunning())
+					? "[" + clusterManager.getLocalMember().id().id() + "] => "
+					: " => ");
+		}
+	}
+
+	public void run() {
+		run(false, false, false, true);
+	}
+
+	public void run(boolean joinOnStart, boolean leaveOnExit, boolean autoElect, boolean allowExit) {
+		if (joinOnStart && !clusterManager.isInitialized()) {
+			clusterManager.initialize();
+		}
+		if (joinOnStart && !clusterManager.isRunning()) {
+			clusterManager.joinCluster(autoElect);
+		}
+		updatePrompt();
+
+		// Start doing work...
+		while (true) {
+			try {
+				String line = readLine(prompt);
+				if (StringUtils.isBlank(line)) continue;
+				String[] cmd = line.trim().split(" ");
+
+				if ("exit".equalsIgnoreCase(cmd[0])) {
+					if (allowExit)
+						break;
+				} else {
+					executeCommand(line, cmd);
+				}
+
+			} catch (Exception ex) {
+				log_error("CLI: Exception caught: ", ex);
+			}
+		}
+
+		if (leaveOnExit && clusterManager.isRunning())
+			clusterManager.leaveCluster();
+	}
+
+	public void executeCommand(String line, String[] cmd) {
+		if ("properties".equalsIgnoreCase(cmd[0])) {
+			Properties properties = clusterManager.getLocalMember().properties();
+			log_info("CLI: Local member properties:");
+			for (String propName : properties.stringPropertyNames()) {
+				log_info("CLI:    {} = {}", propName, properties.getProperty(propName));
+			}
+		} else if ("set".equalsIgnoreCase(cmd[0])) {
+			String setStr = line.trim().split(" ", 2)[1];
+			int p = setStr.indexOf("=");
+			String propName = setStr.substring(0, p).trim();
+			String propValue = setStr.substring(p + 1).trim();
+			log_info("CLI: SET PROPERTY: {} = {}", propName, propValue);
+			clusterManager.getLocalMember().properties().setProperty(propName, propValue);
+		} else if ("unset".equalsIgnoreCase(cmd[0])) {
+			String propName = cmd[1].trim();
+			log_info("CLI: UNSET PROPERTY: {}", propName);
+			clusterManager.getLocalMember().properties().setProperty(propName, "");
+		} else if ("score".equalsIgnoreCase(cmd[0])) {
+			if (cmd.length==1) {
+				log_info("CLI: Score function: {}", clusterManager.getScoreFunction());
+			} else {
+				String formula = clusterManager.getScoreFunction().getFormula();
+				Properties defs = new Properties();
+				defs.putAll(clusterManager.getScoreFunction().getArgumentDefaults());
+				double defScore = clusterManager.getScoreFunction().getDefaultScore();
+				boolean throwExceptions = clusterManager.getScoreFunction().isThrowExceptions();
+				if (!"-".equals(cmd[1]) && !"same".equalsIgnoreCase(cmd[1]))
+					formula = cmd[1];
+				for (int i = 2; i < cmd.length; i++) {
+					String[] part = cmd[i].split("=", 2);
+					if ("default".equalsIgnoreCase(part[0])) {
+						throwExceptions = false;
+						if ("-".equals(part[1]))
+							throwExceptions = true;
+						else
+							defScore = Double.parseDouble(part[1]);
+					} else if ("clear-defaults".equalsIgnoreCase(part[0]))
+						defs.clear();
+					else
+						defs.setProperty(part[0], String.valueOf(Double.parseDouble(part[1])));
+				}
+				clusterManager.setScoreFunction(MemberScoreFunction.builder()
+						.formula(formula)
+						.argumentDefaults(defs)
+						.defaultScore(defScore)
+						.throwExceptions(throwExceptions)
+						.build());
+			}
+
+		} else if ("members".equalsIgnoreCase(cmd[0])) {
+			// Get cluster members
+			log_info("CLI: Cluster members:");
+			for (Member member : clusterManager.getMembers()) {
+				String memId = member.id().id();
+				String memAddress = member.config().getAddress().toString();
+				Set<Map.Entry<Object, Object>> memProperties = member.properties().entrySet();
+				String active = (member.isActive() ? "active" : "inactive");
+				String reachable = (member.isReachable() ? "reachable" : "unreachable");
+				log_info("CLI:    {}/{}/{}-{}/{}", memId, memAddress, active, reachable, memProperties);
+			}
+		} else if ("join".equalsIgnoreCase(cmd[0])) {
+			if (cmd.length>1) {
+				ArrayList<String> tmp = new ArrayList<>(Arrays.asList(cmd));
+				tmp.remove(0);
+				clusterManager.getProperties().setMemberAddresses(tmp);
+			}
+
+			// Join/start cluster
+			clusterManager.initialize();
+			clusterManager.joinCluster();
+			updatePrompt();
+
+		} else if ("leave".equalsIgnoreCase(cmd[0])) {
+			clusterManager.leaveCluster();
+			updatePrompt();
+
+		} else if ("message".equalsIgnoreCase(cmd[0])) {
+			ClusterCommunicationService communicationService = clusterManager.getAtomix().getCommunicationService();
+			String op = cmd[1];
+			String topic = cmd[2];
+			if ("subscribe".equalsIgnoreCase(op)) {
+				communicationService.subscribe(topic, (m) -> {
+					log_info("CLI: **** Message: {} on Topic: {}", m, topic);
+					return CompletableFuture.completedFuture("Ok");
+				}).join();
+				log_info("CLI: Subscribed to topic: {}", topic);
+			} else
+			if ("unsubscribe".equalsIgnoreCase(op)) {
+				log_info("CLI: Unsubscribe from topic: {}", topic);
+				communicationService.unsubscribe(topic);
+			} else
+			if ("broadcast".equalsIgnoreCase(op)) {
+				log_info("CLI: Broadcast to topic: {}", topic);
+				String message = String.join(" ", Arrays.copyOfRange(cmd, 3, cmd.length));
+				communicationService.broadcast(topic, message);
+			} else
+			if ("send".equalsIgnoreCase(op)) {
+				MemberId mId = MemberId.from(cmd[3]);
+				log_info("CLI: Send to node: {}, on topic: {}", cmd[3], topic);
+				String message = String.join(" ", Arrays.copyOfRange(cmd, 4, cmd.length));
+				communicationService.send(topic, message, mId).join();
+			} else
+			if ("unicast".equalsIgnoreCase(op)) {
+				MemberId mId = MemberId.from(cmd[3]);
+				log_info("CLI: Send to node: {}, on topic: {}", cmd[3], topic);
+				String message = String.join(" ", Arrays.copyOfRange(cmd, 3, cmd.length));
+				communicationService.unicast(topic, message, mId).join();
+			} else
+				log_warn("CLI:    Invalid Message operation: {}", op);
+
+		} else if ("broker".equalsIgnoreCase(cmd[0]) || "bl".equalsIgnoreCase(cmd[0])) {
+			String op = cmd.length>1 ? cmd[1] : null;
+			if ("list".equalsIgnoreCase(op) || "bl".equalsIgnoreCase(cmd[0])) {
+				log_info("CLI: Node status and scores:");
+				final BrokerUtil brokerUtil1 = clusterManager.getBrokerUtil();
+				brokerUtil1.getActiveNodes().forEach(ms -> log_info("CLI:    {}  [{}, {}, {}]",
+						ms.getMember().id().id(), brokerUtil1.getNodeStatus(ms.getMember()),
+						ms.getScore(), ms.getMember().properties().getProperty("uuid", null)));
+			} else
+			if ("candidates".equalsIgnoreCase(op)) {
+				log_info("CLI: Broker candidates:");
+				final BrokerUtil brokerUtil1 = clusterManager.getBrokerUtil();
+				brokerUtil1.getCandidates().forEach(ms -> log_info("CLI:    {}  [{}, {}, {}]",
+						ms.getMember().id().id(), brokerUtil1.getNodeStatus(ms.getMember()),
+						ms.getScore(), ms.getMember().properties().getProperty("uuid", null)));
+			} else
+			if ("status".equalsIgnoreCase(op)) {
+				clusterManager.getBrokerUtil().getBrokers()
+						.forEach(m -> log_info("CLI: Current Broker: {}", m.id().id()));
+			} else
+			if ("elect".equalsIgnoreCase(op)) {
+				clusterManager.getBrokerUtil().startElection();
+			} else
+			if ("retire".equalsIgnoreCase(op)) {
+				clusterManager.getBrokerUtil().retire();
+			} else
+			if ("appoint".equalsIgnoreCase(op)) {
+				clusterManager.getBrokerUtil().appoint(cmd[2]);
+			} else
+			if ("on".equalsIgnoreCase(op)) {
+				clusterManager.getBrokerUtil().setCandidate();
+			} else
+			if ("off".equalsIgnoreCase(op)) {
+				clusterManager.getBrokerUtil().clearCandidate();
+			} else
+				log_warn("CLI:    Invalid Broker operation: {}", op);
+
+		} else
+			log_warn("CLI: Unknown command: {}", cmd[0]);
+	}
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/cluster/ClusterManager.java

@@ -0,0 +1,472 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.cluster;
+
+import io.atomix.cluster.ClusterMembershipEvent;
+import io.atomix.cluster.Member;
+import io.atomix.cluster.MemberId;
+import io.atomix.cluster.Node;
+import io.atomix.cluster.discovery.BootstrapDiscoveryProvider;
+import io.atomix.cluster.discovery.NodeDiscoveryProvider;
+import io.atomix.cluster.protocol.GroupMembershipProtocol;
+import io.atomix.cluster.protocol.HeartbeatMembershipProtocol;
+import io.atomix.cluster.protocol.SwimMembershipProtocol;
+import io.atomix.core.Atomix;
+import io.atomix.core.AtomixBuilder;
+import io.atomix.protocols.backup.partition.PrimaryBackupPartitionGroup;
+import io.atomix.protocols.raft.partition.RaftPartitionGroup;
+import io.atomix.utils.net.Address;
+import lombok.*;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.stereotype.Component;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.time.Duration;
+import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledFuture;
+import java.util.stream.Collectors;
+
+@Data
+@Component
+@EqualsAndHashCode(callSuper = true)
+public class ClusterManager extends AbstractLogBase {
+
+	private static final String NODE_NAME_PREFIX = "node_";
+
+	private ClusterManagerProperties properties;
+	private BrokerUtil.NodeCallback callback;
+	private ClusterCLI cli;
+
+	private MemberScoreFunction scoreFunction = new MemberScoreFunction("-1");
+
+	@Getter(AccessLevel.NONE) @Setter(AccessLevel.NONE)
+	private Address localAddress = null;
+	@Getter(AccessLevel.NONE) @Setter(AccessLevel.NONE)
+	private NodeDiscoveryProvider bootstrapDiscoveryProvider = null;
+	@Setter(AccessLevel.NONE)
+	private Atomix atomix = null;
+	@Setter(AccessLevel.NONE)
+	private BrokerUtil brokerUtil = null;
+
+	@Autowired
+	private TaskScheduler taskScheduler;
+	@Getter(AccessLevel.NONE) @Setter(AccessLevel.NONE)
+	private ScheduledFuture<?> checkerTask;
+
+	// ------------------------------------------------------------------------
+
+	public synchronized ClusterCLI getCli() {
+		if (cli==null) {
+			cli = new ClusterCLI(this);
+			cli.setLogEnabled(isLogEnabled());
+			cli.setOutEnabled(isOutEnabled());
+		}
+		return cli;
+	}
+
+	public Atomix getAtomix() {
+		if (atomix==null) throw new IllegalStateException("Not initialized");
+		return atomix;
+	}
+
+	public BrokerUtil getBrokerUtil() {
+		if (brokerUtil==null) throw new IllegalStateException("Not initialized");
+		return brokerUtil;
+	}
+
+	public Set<Member> getMembers() {
+		return getAtomix().getMembershipService().getMembers();
+	}
+
+	public Member getLocalMember() {
+		return getAtomix().getMembershipService().getLocalMember();
+	}
+
+	public Address getLocalAddress() {
+		return getLocalMember().address();
+	}
+
+	public Properties getLocalMemberProperties() {
+		return getAtomix().getMembershipService().getLocalMember().properties();
+	}
+
+	public void setCallback(BrokerUtil.NodeCallback callback) {
+		this.callback = callback;
+		if (brokerUtil!=null) brokerUtil.setCallback(callback);
+	}
+
+	// ------------------------------------------------------------------------
+
+	public boolean isInitialized() {
+		return atomix!=null;
+	}
+
+	public boolean isRunning() {
+		return (atomix!=null && atomix.isRunning());
+	}
+
+	public void initialize() {
+		initialize(properties, callback);
+	}
+
+	public void initialize(ClusterManagerProperties p) {
+		initialize(p, this.callback);
+	}
+
+	public void initialize(ClusterManagerProperties p, BrokerUtil.NodeCallback callback) {
+		// Store properties and callback
+		if (p!=null) this.properties = p;
+		if (callback!=null) this.callback = callback;
+
+		// Set logging and output flags
+		setLogEnabled(properties.isLogEnabled());
+		setOutEnabled(properties.isOutEnabled());
+
+		// Initialize member scoring function
+		this.scoreFunction = properties.getScore()!=null
+				? MemberScoreFunction.builder()
+						.formula(properties.getScore().getFormula())
+						.defaultScore(properties.getScore().getDefaultScore())
+						.argumentDefaults(properties.getScore().getDefaultArgs())
+						.throwExceptions(properties.getScore().isThrowException())
+						.build()
+				: this.scoreFunction;
+
+		// Get local address and port
+		localAddress = properties.getLocalNode().getAddress();
+		log_debug("CLM: Provided local-address: {}", localAddress);
+		if (localAddress==null) {
+			//localAddress = Address.from(getLocalHostName() + ":1234");
+			localAddress = Address.from(getLocalHostAddress() + ":1234");
+			log_debug("CLM: Resolving local-address: {}", localAddress);
+		}
+		log_info("CLM: Local address used for building Atomix: {}", localAddress);
+
+		// Initialize Membership provider
+		bootstrapDiscoveryProvider = buildNodeDiscoveryProvider(properties.getMemberAddresses());
+
+		// Create Atomix and Join/start cluster
+		atomix = buildAtomix(properties, localAddress, bootstrapDiscoveryProvider);
+		brokerUtil = new BrokerUtil(this, callback);
+		brokerUtil.setLogEnabled(isLogEnabled());
+		brokerUtil.setOutEnabled(isOutEnabled());
+	}
+
+	public void joinCluster() {
+		joinCluster(getProperties().isElectionOnJoin());
+	}
+
+	public void joinCluster(boolean startElection) {
+		// Initialize cluster if needed
+		if (atomix==null)
+			initialize();
+
+		// Start/Join cluster
+		log_info("CLM: Joining cluster...");
+		long startTm = System.currentTimeMillis();
+		atomix.start().join();
+		long endTm = System.currentTimeMillis();
+		log_debug("CLM: Joined cluster in {}ms", endTm-startTm);
+
+		// Populate default local member properties
+		Member localMember = atomix.getMembershipService().getLocalMember();
+		String addrStr = localMember.address().host() + ":" + localMember.address().port();
+		atomix.getMembershipService().getLocalMember().properties().setProperty("address", addrStr);
+		atomix.getMembershipService().getLocalMember().properties().setProperty("uuid", UUID.randomUUID().toString());
+		brokerUtil.setLocalStatus(BrokerUtil.NODE_STATUS.CANDIDATE);
+
+		// Add membership listener
+		atomix.getMembershipService().addListener(event -> {
+			log_debug("CLM: {}: node={}", event.type(), event.subject());
+			if (event.type()!=ClusterMembershipEvent.Type.REACHABILITY_CHANGED) {
+				if (event.type()!=ClusterMembershipEvent.Type.METADATA_CHANGED) {
+					log_info("CLM: {}: node={}", event.type(), event.subject().id().id());
+					brokerUtil.checkBroker();
+				}
+				if (callback!=null)
+					callback.clusterChanged(event);
+			}
+		});
+
+		// Add broker message listener
+		atomix.getCommunicationService().subscribe(BrokerUtil.NODE_MESSAGE_TOPIC, m -> {
+			brokerUtil.processBrokerMessage(m);
+			return CompletableFuture.completedFuture("ok");
+		});
+
+		// Start election if no broker exists
+		if (startElection) {
+			brokerUtil.checkBroker();
+		}
+
+		// Start cluster checker
+		if (properties.isClusterCheckerEnabled()) {
+			long delay = Math.max(properties.getClusterCheckerDelay(), 10000L);
+			log_info("CLM: Starting cluster checker (delay: {})...", delay);
+			checkerTask = taskScheduler.scheduleWithFixedDelay(() -> {
+				if (brokerUtil != null)
+					brokerUtil.checkBrokerNumber();
+				else
+					log_warn("CLM: Cluster checker: BrokerUtil is NULL  (is it a BUG?)");
+			}, Duration.ofMillis(delay));
+		} else {
+			log_warn("CLM: Cluster checker is DISABLED");
+		}
+	}
+
+	public void waitToJoin() {
+		while (true) {
+			if (isInitialized() && isRunning()) break;
+			try { Thread.sleep(500); } catch (InterruptedException e) { break; }
+		}
+		if (callback!=null)
+			callback.joinedCluster();
+	}
+
+	public void waitToJoin(long waitForMillis) {
+		long startTm = System.currentTimeMillis();
+		long endTm = startTm + waitForMillis;
+		while (true) {
+			if (isInitialized() && isRunning()) break;
+			long waitFor = Math.min(500, endTm-System.currentTimeMillis());
+			try { Thread.sleep(waitFor); } catch (InterruptedException e) { break; }
+		}
+		if (callback!=null)
+			callback.joinedCluster();
+	}
+
+	public void leaveCluster() {
+		// Stop cluster checker
+		if (checkerTask!=null && !checkerTask.isCancelled()) {
+			log_info("CLM: Stopping cluster checker...");
+			checkerTask.cancel(true);
+			checkerTask = null;
+		}
+
+		// Leave cluster
+		log_info("CLM: Leaving cluster...");
+		long startTm = System.currentTimeMillis();
+		if (atomix.isRunning())
+			atomix.stop().join();
+		long endTm = System.currentTimeMillis();
+		log_debug("CLM: Left cluster in {}ms", endTm-startTm);
+		atomix = null;
+		brokerUtil = null;
+		if (callback!=null)
+			callback.leftCluster();
+	}
+
+	// ------------------------------------------------------------------------
+
+	public static String getLocalHostName() {
+		String hostname = null;
+		try {
+			hostname = InetAddress.getLocalHost().getHostName();
+		} catch (UnknownHostException e) {
+			//log_error("Exception while getting Node hostname: ", e);
+		}
+		if (StringUtils.isBlank(hostname))
+			hostname = getLocalHostAddress();
+		return hostname;
+	}
+
+	public static String getLocalHostAddress() {
+		String address = null;
+		try {
+			address = InetAddress.getLocalHost().getHostAddress();
+		} catch (UnknownHostException e) {
+			//log_error("Exception while getting Node local address: ", e);
+		}
+		if (StringUtils.isBlank(address))
+			address = UUID.randomUUID().toString();
+		return address;
+	}
+
+	// ------------------------------------------------------------------------
+
+	private String createMemberName(int port) { return createMemberName(getLocalHostName()+":"+port); }
+	private String createMemberName(String address) {
+		return NODE_NAME_PREFIX+address.replace(":", "_");
+	}
+
+	private Node createNode(String address, String port) { return createNode(address, Integer.parseInt(port)); }
+	private Node createNode(String address, int port) { return createNode(address+":"+port); }
+	private Node createNode(String address) {
+		return Node.builder()
+				.withId(createMemberName(address))
+				.withAddress(Address.from(address))
+				.build();
+	}
+	private Node createNode(ClusterManagerProperties.NodeProperties nodeProperties) {
+		String nodeId = nodeProperties.getId();
+		if (StringUtils.isBlank(nodeId))
+			nodeId = createMemberName(nodeProperties.getAddress().port());
+		return Node.builder()
+				.withId(nodeId)
+				.withAddress(nodeProperties.getAddress())
+				.build();
+	}
+
+	public static Address getAddressFromString(String localAddressStr) {
+		Address localAddress;
+		localAddressStr = localAddressStr.trim();
+		if (StringUtils.isBlank(localAddressStr)) {
+			localAddress = Address.local();
+		} else
+		if (StringUtils.isNumeric(localAddressStr)) {
+			localAddress = Address.from(Integer.parseInt(localAddressStr));
+		} else {
+			localAddress = Address.from(localAddressStr);
+		}
+		return localAddress;
+	}
+
+	private NodeDiscoveryProvider buildNodeDiscoveryProvider(List<String> addresses) {
+		return buildNodeDiscoveryProviderFromProperties(
+				addresses!=null
+						? addresses.stream()
+								.map(ClusterManager::getAddressFromString)
+								.map(address -> new ClusterManagerProperties.NodeProperties(null, address, null))
+								.collect(Collectors.toList())
+						: null);
+	}
+
+	private NodeDiscoveryProvider buildNodeDiscoveryProviderFromProperties(List<ClusterManagerProperties.NodeProperties> nodePropertiesList) {
+		List<Node> nodes = new ArrayList<>();
+		if (nodePropertiesList!=null) {
+			nodes = nodePropertiesList.stream().map(this::createNode).collect(Collectors.toList());
+		}
+		log_info("CLM: Building Atomix: Other members: {}", nodes);
+		return BootstrapDiscoveryProvider.builder()
+				.withNodes(nodes)
+				//.withHeartbeatInterval(Duration.ofSeconds(5))
+				//.withFailureThreshold(2)
+				//.withFailureTimeout(Duration.ofSeconds(1))
+				.build();
+	}
+
+	private MemberId[] getMemberIds(Set<Node> nodes) {
+		List<MemberId> memberIdList = new ArrayList<>();
+		for (Node node : nodes)
+			memberIdList.add(MemberId.from(node.id().id()));
+		return memberIdList.toArray(new MemberId[0]);
+	}
+
+	private Member[] getMembers(Set<Node> nodes) {
+		List<Member> memberList = new ArrayList<>();
+		for (Node node : nodes)
+			memberList.add(Member.builder()
+					.withId(node.id().id())
+					.withAddress(node.address())
+					.build());
+		return memberList.toArray(new Member[0]);
+	}
+
+	private Atomix buildAtomix(ClusterManagerProperties properties, Address localAddress, NodeDiscoveryProvider bootstrapDiscoveryProvider) {
+		// Configuring local cluster member
+		AtomixBuilder atomixBuilder = Atomix.builder();
+
+		// Cluster id
+		String clusterId = properties.getClusterId();
+		if (StringUtils.isNotBlank(clusterId)) {
+			log_info("CLM: Building Atomix: Cluster-id: {}", clusterId);
+			atomixBuilder.withClusterId(clusterId);
+		}
+
+		// Local member id and address
+		String memId = properties.getLocalNode().getId();
+		memId = StringUtils.isBlank(memId) ? createMemberName(localAddress.port()) : memId;
+		MemberId localMemberId = MemberId.from(memId);
+		log_info("CLM: Building Atomix: Local-Member-Id: {}", localMemberId);
+		log_info("CLM: Building Atomix: Local-Member-Address: {}", localAddress);
+		atomixBuilder
+				.withMemberId(localMemberId)
+				.withAddress(localAddress)
+				.withProperties(properties.getLocalNode().getProperties());
+
+		// Configure membership protocol
+		boolean useSwim = properties.isUseSwim();
+		long failureTimeout = Math.max(100L, properties.getFailureTimeout());
+		GroupMembershipProtocol memProto;
+		atomixBuilder
+				.withMembershipProtocol(memProto = useSwim
+						? SwimMembershipProtocol.builder()
+								//.withGossipInterval(Duration.ofMillis(250))
+								//.withGossipFanout(2)
+								.withFailureTimeout(Duration.ofMillis(failureTimeout))
+								.build()
+						: HeartbeatMembershipProtocol.builder()
+								//.withHeartbeatInterval(Duration.ofMillis(1000))
+								.withFailureTimeout(Duration.ofMillis(failureTimeout))
+								//.withFailureThreshold(2)
+								.build()
+				);
+		log_info("CLM: Building Atomix: Membership protocol: {}", memProto.getClass().getSimpleName());
+
+		// Configure Management and Partition groups
+		boolean usePBInMg = properties.isUsePBInMg();
+		boolean usePBInPg = properties.isUsePBInPg();
+		String mgName = properties.getMgName();
+		String pgName = properties.getPgName();
+		if (StringUtils.isBlank(mgName)) mgName = "system";
+		if (StringUtils.isBlank(pgName)) pgName = "data";
+		log_debug("CLM: Building Atomix: Cluster Groups: mg-type-PB={}, pg-type-PB={}, mg-name={}, pg-name={}",
+				usePBInMg, usePBInPg, mgName, pgName);
+		atomixBuilder
+				.withManagementGroup(usePBInMg
+						? PrimaryBackupPartitionGroup.builder(mgName)
+								.withNumPartitions(1)
+								//.withMemberGroupStrategy(MemberGroupStrategy.NODE_AWARE)
+								.build()
+						: RaftPartitionGroup.builder(mgName)
+								.withNumPartitions(1)
+								.withMembers(getMemberIds(bootstrapDiscoveryProvider.getNodes()))
+								//.withMembers(getMembers(bootstrapDiscoveryProvider.getNodes()))
+								//.withDataDirectory(new File("raft-mg"))
+								//.withMemberGroupStrategy(MemberGroupStrategy.NODE_AWARE)
+								.build()
+				)
+				.withPartitionGroups(usePBInPg
+						? PrimaryBackupPartitionGroup.builder(pgName)
+								.withNumPartitions(8)
+								//.withMemberGroupStrategy(MemberGroupStrategy.NODE_AWARE)
+								.build()
+						: RaftPartitionGroup.builder(pgName)
+								.withNumPartitions(8)
+								.withMembers(getMemberIds(bootstrapDiscoveryProvider.getNodes()))
+								//.withMembers(getMembers(bootstrapDiscoveryProvider.getNodes()))
+								//.withDataDirectory(new File("raft-pg"))
+								//.withMemberGroupStrategy(MemberGroupStrategy.NODE_AWARE)
+								.build()
+				);
+
+		// Configure Bootstrap Discovery Provider
+		atomixBuilder
+				//.withMulticastEnabled()
+				.withMembershipProvider(bootstrapDiscoveryProvider);
+
+		// Configure TLS for messaging
+		log_info("CLM: Building Atomix: TLS enabled={}", properties.getTls().isEnabled());
+		if (properties.getTls().isEnabled()) {
+			atomixBuilder
+					.withTlsEnabled(true)
+					.withKeyStore(properties.getTls().getKeystore())
+					.withKeyStorePassword(properties.getTls().getKeystorePassword())
+					.withTrustStore(properties.getTls().getTruststore())
+					.withTrustStorePassword(properties.getTls().getTruststorePassword());
+		}
+
+		return atomixBuilder.build();
+	}
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/cluster/ClusterManagerProperties.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.cluster;
+
+import io.atomix.utils.net.Address;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.List;
+import java.util.Properties;
+
+@Data
+@Configuration
+@ConfigurationProperties(prefix = "cluster")
+public class ClusterManagerProperties {
+	private String clusterId = "local-cluster";
+	private NodeProperties localNode = new NodeProperties();
+	private List<String> memberAddresses;
+
+	private boolean useSwim = true;			// ...else the Heartbeat membership protocol will be used
+	private long failureTimeout = 10000;	// The Atomix default failure timeout for both membership protocols
+	private long testInterval = -1;			// Print cluster node status every X millis (negative numbers should turn off feature)
+
+	private boolean logEnabled;
+	private boolean outEnabled = true;
+
+	private boolean joinOnInit = true;
+	private boolean electionOnJoin;
+
+	private boolean clusterCheckerEnabled = true;
+	private long clusterCheckerDelay = 30000L;
+
+	private boolean usePBInMg = true;
+	private boolean usePBInPg = true;
+	private String mgName = "system";
+	private String pgName = "data";
+
+	private TlsProperties tls = new TlsProperties();
+
+	private ScoreFunctionProperties score;
+
+	@Data
+	@NoArgsConstructor
+	@AllArgsConstructor
+	public static class NodeProperties {
+		private String id;
+		private Address address;
+		private Properties properties = new Properties();
+
+		public NodeProperties(String address) {
+			this.address = Address.from(address);
+		}
+
+		public void setAddress(String address) {
+			this.address = ClusterManager.getAddressFromString(address);
+		}
+	}
+
+	@Data
+	@ToString(exclude = {"keystorePassword", "truststorePassword"})
+	public static class TlsProperties {
+		private boolean enabled;
+		private String keystore;
+		private String keystorePassword;
+		private String truststore;
+		private String truststorePassword;
+		private String keystoreDir;
+	}
+
+	@Data
+	public static class ScoreFunctionProperties {
+		private String formula;
+		private double defaultScore;
+		private Properties defaultArgs;
+		private boolean throwException;
+	}
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/cluster/ClusterTest.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.cluster;
+
+import io.atomix.core.Atomix;
+import lombok.*;
+
+import java.util.stream.Collectors;
+
+@Data
+public class ClusterTest implements Runnable {
+
+	@NonNull
+	private final ClusterManager clusterManager;
+
+	@Getter(AccessLevel.NONE) @Setter(AccessLevel.NONE)
+	private Thread runner;
+	@Getter(AccessLevel.NONE) @Setter(AccessLevel.NONE)
+	private boolean keepRunning;
+	private long delay = 5000;
+
+	public void startTest(long delay) {
+		checkRunning();
+		if (delay < 1) throw new IllegalArgumentException("ClusterTest delay must be positive: " + delay);
+		this.delay = delay;
+		startTest();
+	}
+
+	public synchronized void startTest() {
+		checkRunning();
+		runner = new Thread(this);
+		runner.setDaemon(true);
+		keepRunning = true;
+		runner.start();
+	}
+
+	public synchronized void stopTest() {
+		checkNotRunning();
+		keepRunning = false;
+		runner.interrupt();
+		runner = null;
+	}
+
+	private void checkRunning() {
+		if (keepRunning)
+			throw new IllegalStateException("ClusterTest is already running");
+	}
+
+	private void checkNotRunning() {
+		if (!keepRunning)
+			throw new IllegalStateException("ClusterTest is not running");
+	}
+
+	public void run() {
+		// Start doing work...
+		Atomix atomix = clusterManager.getAtomix();
+		int iterations = 0;
+		while (keepRunning) {
+			iterations++;
+			clusterManager.log_info("-- Iter={} ---------------------------------------", iterations);
+
+			// Get cluster members
+			clusterManager.log_info("-- CLUSTER-MEMBERS: {}", atomix.getMembershipService().getMembers().stream()
+					.map(m -> "\n  "+m.id().id()
+							+ "/" + clusterManager.getBrokerUtil().getNodeStatus(m)
+							+ "/" + m.properties().getProperty("address", "---")
+							+ "/" + (m.isActive()?"active":"inactive")
+							+ (!m.isReachable() ? "/unreachable" : ""))
+					.collect(Collectors.toList()));
+
+			// Sleep for 5 seconds
+			try { Thread.sleep(delay); } catch (Exception e) {}
+		}
+	}
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/cluster/MemberScoreFunction.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.cluster;
+
+import io.atomix.cluster.Member;
+import lombok.Builder;
+import lombok.Data;
+import org.apache.commons.lang3.StringUtils;
+import org.mariuszgromada.math.mxparser.Expression;
+import org.mariuszgromada.math.mxparser.parsertokens.Token;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+@Data
+@Builder
+public class MemberScoreFunction implements Function<Member, Double> {
+    private final String formula;
+    private final double defaultScore;
+    private final Properties argumentDefaults;
+    private boolean throwExceptions;
+
+    public MemberScoreFunction(String formula) {
+        this(formula, -1, new Properties(), false);
+    }
+
+    public MemberScoreFunction(String formula, double defaultScore) {
+        this(formula, defaultScore, new Properties(), false);
+    }
+
+    public MemberScoreFunction(String formula, Properties defaults) {
+        this(formula, -1, defaults, false);
+    }
+
+    public MemberScoreFunction(String formula, double defaultScore, Properties defaults, boolean throwExceptions) {
+        Expression e = new Expression(formula);
+        //e.setVerboseMode();
+        if (!e.checkLexSyntax())
+            throw new IllegalArgumentException("Lexical syntax error in expression: " + e.getErrorMessage());
+        this.formula = formula;
+        this.defaultScore = defaultScore;
+        this.argumentDefaults = defaults;
+        this.throwExceptions = throwExceptions;
+    }
+
+    @Override
+    public Double apply(Member member) {
+        return evaluateExpression(formula, member.properties());
+    }
+
+    protected List<String> getExpressionArguments(Expression e) {
+        // Get argument names
+        boolean lexSyntax = e.checkLexSyntax();
+        boolean genSyntax = e.checkSyntax();
+
+        List<Token> initTokens = e.getCopyOfInitialTokens();
+        List<String> argNames = initTokens.stream()
+                .filter(t -> t.tokenTypeId == Token.NOT_MATCHED)
+                .filter(t -> "argument".equals(t.looksLike))
+                .map(t -> t.tokenStr)
+                .collect(Collectors.toList());
+
+        return argNames;
+    }
+
+    public double evaluateExpression(String formula, Properties args) {
+        try {
+            if (StringUtils.isBlank(formula)) {
+                throw new IllegalArgumentException("Formula is empty or null");
+            }
+
+            // Create MathParser expression
+            Expression e = new Expression(formula);
+            //e.setVerboseMode();
+
+            // Get argument names
+            List<String> argNames = getExpressionArguments(e);
+
+            // Define expression arguments with user provided values
+            //e.removeAllArguments();
+            for (String argName : argNames) {
+                try {
+                    String argStr = args.getProperty(argName, null);
+                    if (StringUtils.isBlank(argStr))
+                        argStr = argumentDefaults.getProperty(argName, null);
+                    if (StringUtils.isBlank(argStr))
+                        throw new IllegalArgumentException("Missing scoring expression argument: " + argName);
+                    double argValue = Double.parseDouble(argStr);
+                    e.defineArgument(argName, argValue);
+                } catch (Exception ex) {
+                    throw ex;
+                }
+            }
+            if (!e.checkSyntax())
+                throw new IllegalArgumentException("Syntax error in expression: " + e.getErrorMessage());
+
+            // Calculate result
+            return e.calculate();
+        } catch (Exception ex) {
+            if (throwExceptions)
+                throw ex;
+            return defaultScore;
+        }
+    }
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/cluster/MemberWithScore.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.cluster;
+
+import io.atomix.cluster.Member;
+import lombok.Data;
+
+@Data
+public class MemberWithScore implements Comparable<MemberWithScore> {
+    public final static MemberWithScore NULL_MEMBER = new MemberWithScore(null, 0);
+
+    private final Member member;
+    private final double score;
+
+    private MemberWithScore(Member m, double s) {
+        member = m;
+        score = s;
+    }
+
+    public MemberWithScore(Member m, MemberScoreFunction scoreFunction) {
+        member = m;
+        score = scoreFunction.apply(m);
+    }
+
+    @Override
+    public int compareTo(MemberWithScore o) {
+        double score1 = this.getScore();
+        double score2 = o.getScore();
+        int result = (int) Math.signum(score1 - score2);
+        if (result == 0) {
+            String uuid1 = this.getMember().properties().getProperty("uuid", "0");
+            String uuid2 = o.getMember().properties().getProperty("uuid", "0");
+            result = uuid1.compareTo(uuid2);
+        }
+        return result;
+    }
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/cluster/TestCallback.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.cluster;
+
+import io.atomix.cluster.ClusterMembershipEvent;
+import io.atomix.cluster.Member;
+import io.atomix.utils.net.Address;
+
+public class TestCallback extends AbstractLogBase implements BrokerUtil.NodeCallback {
+    private String address;
+    private String state = "L1";
+
+    public TestCallback(Address localAddress) {
+        address = localAddress.toString();
+    }
+
+    public void joinedCluster() { }
+    public void leftCluster() { }
+
+    public void initialize() {
+        if ("L2".equals(state)) {
+            log_warn("__TestNode at {}: Already initialized: {}", address, state);
+            return;
+        }
+        state = "initializing L2";
+        out_print("__TestNode at {}: Initializing", address);
+        for (int i = 0; i < (int) (Math.random() * 5 + 5); i++) {
+            out_print(".");
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException ignored) {
+            }
+        }
+        out_println();
+        if ("initializing L2".equals(state)) {
+            state = "L2";
+            log_info("__TestNode at {}: Node is now a Broker: {}", address, state);
+        }
+    }
+
+    public void stepDown() {
+        if ("L1".equals(state)) {
+            log_warn("__TestNode at {}: Already a non-broker node: {}", address, state);
+            return;
+        }
+        state = "clearing L2";
+        out_print("__TestNode at {}: Stepping down", address);
+        for (int i = 0; i < (int) (Math.random() * 4 + 2); i++) {
+            out_print(".");
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException ignored) {
+            }
+        }
+        out_println();
+        if ("clearing L2".equals(state)) {
+            state = "L1";
+            log_info("__TestNode at {}: Node is now a non-broker node: {}", address, state);
+        }
+    }
+
+    public void statusChanged(BrokerUtil.NODE_STATUS oldStatus, BrokerUtil.NODE_STATUS newStatus) {
+        log_info("__TestNode at {}: Status changed: {} --> {}", address, oldStatus, newStatus);
+    }
+
+    public void clusterChanged(ClusterMembershipEvent event) {
+        log_info("__TestNode at {}: Cluster changed: {}: {}", address, event.type(), event.subject().id().id());
+    }
+
+    public String getConfiguration(Member local) {
+        return String.format("ssl://%s:61617", local.address().host());
+    }
+
+    public void setConfiguration(String newConfig) {
+        log_info("__TestNode at {}: New configuration: {}", address, newConfig);
+    }
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/collector/ClientCollectorContext.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.collector;
+
+import gr.iccs.imu.ems.baguette.client.BaguetteClientProperties;
+import gr.iccs.imu.ems.baguette.client.CommandExecutor;
+import gr.iccs.imu.ems.baguette.client.Sshc;
+import gr.iccs.imu.ems.brokercep.event.EventMap;
+import gr.iccs.imu.ems.common.client.SshClient;
+import gr.iccs.imu.ems.common.collector.CollectorContext;
+import gr.iccs.imu.ems.util.ClientConfiguration;
+import gr.iccs.imu.ems.util.GroupingConfiguration;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class ClientCollectorContext implements CollectorContext<BaguetteClientProperties> {
+    private final CommandExecutor commandExecutor;
+
+    public Map<String, GroupingConfiguration> getGroupings() {
+        return commandExecutor.getGroupings();
+    }
+
+    @Override
+    public List<ClientConfiguration> getNodeConfigurations() {
+        return Collections.singletonList(commandExecutor.getClientConfiguration());
+    }
+
+    @Override
+    public Set<Serializable> getNodesWithoutClient() {
+        return commandExecutor.getClientConfiguration()!=null
+                ? commandExecutor.getClientConfiguration().getNodesWithoutClient() : null;
+    }
+
+    @Override
+    public boolean isAggregator() {
+        return commandExecutor.isAggregator();
+    }
+
+    @Override
+    public PUBLISH_RESULT sendEvent(String connectionString, String destinationName, EventMap event, boolean createDestination) {
+        return commandExecutor.sendEvent(connectionString, destinationName, event, createDestination);
+    }
+
+    @Override
+    public SshClient<BaguetteClientProperties> getSshClient() {
+        return new Sshc();
+    }
+
+    @Override
+    public BaguetteClientProperties getSshClientProperties() {
+        return new BaguetteClientProperties();
+    }
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/collector/netdata/NetdataCollector.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.collector.netdata;
+
+import gr.iccs.imu.ems.baguette.client.Collector;
+import gr.iccs.imu.ems.baguette.client.collector.ClientCollectorContext;
+import gr.iccs.imu.ems.common.collector.CollectorContext;
+import gr.iccs.imu.ems.common.collector.netdata.NetdataCollectorProperties;
+import gr.iccs.imu.ems.util.EventBus;
+import gr.iccs.imu.ems.util.GROUPING;
+import gr.iccs.imu.ems.util.GroupingConfiguration;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Collects measurements from Netdata http server
+ */
+@Slf4j
+@Component
+public class NetdataCollector extends gr.iccs.imu.ems.common.collector.netdata.NetdataCollector implements Collector {
+    public NetdataCollector(@NonNull NetdataCollectorProperties properties,
+                            @NonNull CollectorContext collectorContext,
+                            @NonNull TaskScheduler taskScheduler,
+                            @NonNull EventBus<String, Object, Object> eventBus)
+    {
+        super("NetdataCollector", properties, collectorContext, taskScheduler, eventBus);
+        if (!(collectorContext instanceof ClientCollectorContext))
+            throw new IllegalArgumentException("Invalid CollectorContext provided. Expected: ClientCollectorContext, but got "+collectorContext.getClass().getName());
+    }
+
+    public synchronized void activeGroupingChanged(String oldGrouping, String newGrouping) {
+        HashSet<String> topics = new HashSet<>();
+        for (String g : GROUPING.getNames()) {
+            GroupingConfiguration grp = ((ClientCollectorContext)collectorContext).getGroupings().get(g);
+            if (grp!=null)
+                topics.addAll(grp.getEventTypeNames());
+        }
+        log.warn("Collectors::Netdata: activeGroupingChanged: New Allowed Topics for active grouping: {} -- {}", newGrouping, topics);
+        List<String> tmpList = new ArrayList<>(topics);
+        Map<String,String> tmpMap = null;
+        if (properties.getAllowedTopics()!=null) {
+            tmpMap = properties.getAllowedTopics().stream()
+                    .map(s -> s.split(":", 2))
+                    .collect(Collectors.toMap(a -> a[0], a -> a.length>1 ? a[1]: ""));
+        }
+        log.warn("Collectors::Netdata: activeGroupingChanged: New Allowed Topics -- Topics Map: {} -- {}", tmpList, tmpMap);
+        synchronized (this) {
+            this.allowedTopics = tmpList;
+            this.topicMap = tmpMap;
+        }
+    }
+
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/collector/prometheus/PrometheusCollector.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.collector.prometheus;
+
+import gr.iccs.imu.ems.baguette.client.Collector;
+import gr.iccs.imu.ems.baguette.client.collector.ClientCollectorContext;
+import gr.iccs.imu.ems.common.collector.CollectorContext;
+import gr.iccs.imu.ems.common.collector.prometheus.PrometheusCollectorProperties;
+import gr.iccs.imu.ems.util.EventBus;
+import gr.iccs.imu.ems.util.GROUPING;
+import gr.iccs.imu.ems.util.GroupingConfiguration;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Collects measurements from Prometheus exporter
+ */
+@Slf4j
+@Component
+public class PrometheusCollector extends gr.iccs.imu.ems.common.collector.prometheus.PrometheusCollector implements Collector {
+    public PrometheusCollector(@NonNull PrometheusCollectorProperties properties,
+                               @NonNull CollectorContext collectorContext,
+                               @NonNull TaskScheduler taskScheduler,
+                               @NonNull EventBus<String, Object, Object> eventBus)
+    {
+        super("PrometheusCollector", properties, collectorContext, taskScheduler, eventBus);
+        if (!(collectorContext instanceof ClientCollectorContext))
+            throw new IllegalArgumentException("Invalid CollectorContext provided. Expected: ClientCollectorContext, but got "+collectorContext.getClass().getName());
+    }
+
+    public synchronized void activeGroupingChanged(String oldGrouping, String newGrouping) {
+        HashSet<String> topics = new HashSet<>();
+        for (String g : GROUPING.getNames()) {
+            GroupingConfiguration grp = ((ClientCollectorContext)collectorContext).getGroupings().get(g);
+            if (grp!=null)
+                topics.addAll(grp.getEventTypeNames());
+        }
+        log.warn("Collectors::Prometheus: activeGroupingChanged: New Allowed Topics for active grouping: {} -- {}", newGrouping, topics);
+        List<String> tmpList = new ArrayList<>(topics);
+        Map<String,String> tmpMap = null;
+        if (properties.getAllowedTopics()!=null) {
+            tmpMap = properties.getAllowedTopics().stream()
+                    .map(s -> s.split(":", 2))
+                    .collect(Collectors.toMap(a -> a[0], a -> a.length>1 ? a[1]: ""));
+        }
+        log.warn("Collectors::Prometheus: activeGroupingChanged: New Allowed Topics -- Topics Map: {} -- {}", tmpList, tmpMap);
+        synchronized (this) {
+            this.allowedTopics = tmpList;
+            this.topicMap = tmpMap;
+        }
+    }
+
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/plugin/recovery/NodeInfoHelper.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.plugin.recovery;
+
+import com.google.gson.Gson;
+import gr.iccs.imu.ems.baguette.client.CommandExecutor;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Node Info helper -- Retrieves node info from EMS server and caches them
+ */
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class NodeInfoHelper {
+    private final CommandExecutor commandExecutor;
+    private final HashMap<String,Map> nodeInfoCache = new HashMap<>();
+    private final Gson gson = new Gson();
+
+    @SneakyThrows
+    public Map getNodeInfo(String nodeId, @NonNull String nodeAddress) {
+        log.debug("NodeInfoHelper: getNodeInfo(): BEGIN: node-id={}, node-address={}", nodeId, nodeAddress);
+
+        // Get cached node info
+        Map nodeInfo = nodeInfoCache.get(nodeAddress);
+
+        if (nodeInfo==null) {
+            // Get node info from EMS server
+            try {
+                log.debug("NodeInfoHelper: getNodeInfo(): Querying EMS server for Node Info: id={}, address={}", nodeId, nodeAddress);
+                commandExecutor.executeCommand("SEND SERVER-GET-NODE-SSH-CREDENTIALS " + nodeAddress);
+                String response = commandExecutor.getLastInputLine();
+                log.debug("NodeInfoHelper: getNodeInfo(): Node Info from EMS server: id={}, address={}\n{}", nodeId, nodeAddress, response);
+                if (StringUtils.isNotBlank(response)) {
+                    nodeInfo = gson.fromJson(response, Map.class);
+                }
+                nodeInfoCache.put(nodeAddress, nodeInfo);
+            } catch (Exception ex) {
+                log.error("NodeInfoHelper: getNodeInfo(): Exception while querying for node info: node-id={}, node-address={}\n", nodeId, nodeAddress, ex);
+                throw ex;
+            }
+        }
+        //log.debug("NodeInfoHelper: getNodeInfo(): Node info: {}", nodeInfo);
+        return nodeInfo;
+    }
+
+    public void remove(String nodeId, @NonNull String nodeAddress) {
+        log.debug("NodeInfoHelper: remove(): node-id={}, node-address={}", nodeId, nodeAddress);
+        Map nodeInfo = nodeInfoCache.remove(nodeAddress);
+        log.trace("NodeInfoHelper: remove(): Removed: node-id={}, node-address={}", nodeId, nodeAddress);
+    }
+}

ems-core/baguette-client/src/main/java/gr/iccs/imu/ems/baguette/client/plugin/recovery/SelfHealingPlugin.java

@@ -0,0 +1,321 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.client.plugin.recovery;
+
+import gr.iccs.imu.ems.baguette.client.BaguetteClientProperties;
+import gr.iccs.imu.ems.baguette.client.CommandExecutor;
+import gr.iccs.imu.ems.baguette.client.collector.netdata.NetdataCollector;
+import gr.iccs.imu.ems.common.recovery.*;
+import gr.iccs.imu.ems.util.EventBus;
+import gr.iccs.imu.ems.util.PasswordUtil;
+import gr.iccs.imu.ems.util.Plugin;
+import gr.iccs.imu.ems.util.StrUtil;
+import io.atomix.cluster.ClusterMembershipEvent;
+import lombok.*;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DurationFormatUtils;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.context.ApplicationContext;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.stereotype.Component;
+
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Client-side Self-Healing plugin
+ */
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class SelfHealingPlugin implements Plugin, InitializingBean, EventBus.EventConsumer<String,Object,Object> {
+    private final ApplicationContext applicationContext;
+    private final BaguetteClientProperties properties;
+    private final SelfHealingProperties selfHealingProperties;
+    private final CommandExecutor commandExecutor;
+    private final EventBus<String,Object,Object> eventBus;
+    private final PasswordUtil passwordUtil;
+    private final NodeInfoHelper nodeInfoHelper;
+    private final RecoveryContext recoveryContext;
+
+    private boolean started;
+
+    private final HashMap<NodeKey,ScheduledFuture<?>> waitingTasks = new HashMap<>();
+    private final TaskScheduler taskScheduler;
+
+    @Override
+    public void afterPropertiesSet() {
+        log.debug("SelfHealingPlugin: properties: {}", properties);
+        log.debug("SelfHealingPlugin: selfHealingProperties: {}", selfHealingProperties);
+
+        // Initialize recovery context
+        recoveryContext.initialize(properties);
+        log.warn("SelfHealingPlugin: Recovery context: {}", recoveryContext);
+    }
+
+    public synchronized void start() {
+        // check if already running
+        if (started) {
+            log.warn("SelfHealingPlugin: Already started");
+            return;
+        }
+
+        eventBus.subscribe(CommandExecutor.EVENT_CLUSTER_NODE_ADDED, this);
+        eventBus.subscribe(CommandExecutor.EVENT_CLUSTER_NODE_REMOVED, this);
+        eventBus.subscribe(NetdataCollector.NETDATA_NODE_OK, this);
+        eventBus.subscribe(NetdataCollector.NETDATA_NODE_FAILED, this);
+        log.info("SelfHealingPlugin: Started");
+    }
+
+    public synchronized void stop() {
+        if (!started) {
+            log.warn("SelfHealingPlugin: Not started");
+            return;
+        }
+
+        eventBus.unsubscribe(CommandExecutor.EVENT_CLUSTER_NODE_ADDED, this);
+        eventBus.unsubscribe(CommandExecutor.EVENT_CLUSTER_NODE_REMOVED, this);
+        eventBus.unsubscribe(NetdataCollector.NETDATA_NODE_OK, this);
+        eventBus.unsubscribe(NetdataCollector.NETDATA_NODE_FAILED, this);
+
+        // Cancel all waiting recovery tasks
+        waitingTasks.forEach((nodeKey,future) -> {
+            future.cancel(true);
+        });
+        waitingTasks.clear();
+        log.info("SelfHealingPlugin: Stopped");
+    }
+
+    @Override
+    public void onMessage(String topic, Object message, Object sender) {
+        log.debug("SelfHealingPlugin: onMessage(): BEGIN: topic={}, message={}, sender={}", topic, message, sender);
+        if (!selfHealingProperties.isEnabled()) return;
+
+        // Self-Healing for EMS clients
+        if (CommandExecutor.EVENT_CLUSTER_NODE_REMOVED.equals(topic)) {
+            log.debug("SelfHealingPlugin: onMessage(): CLUSTER NODE REMOVED: message={}", message);
+            processClusterNodeRemovedEvent(message);
+        } else
+        if (CommandExecutor.EVENT_CLUSTER_NODE_ADDED.equals(topic)) {
+            log.debug("SelfHealingPlugin: onMessage(): CLUSTER NODE ADDED: message={}", message);
+            processClusterNodeAddedEvent(message);
+        } else
+
+        // Self-healing for Netdata agents
+        if (NetdataCollector.NETDATA_NODE_FAILED.equals(topic)) {
+            log.debug("SelfHealingPlugin: onMessage(): NETDATA NODE PAUSED: message={}", message);
+            processNetdataNodeFailedEvent(message);
+        } else
+        if (NetdataCollector.NETDATA_NODE_OK.equals(topic)) {
+            log.debug("SelfHealingPlugin: onMessage(): NETDATA NODE RESUMED: message={}", message);
+            processNetdataNodeOkEvent(message);
+        } else
+
+        // Unsupported message
+        {
+            log.debug("SelfHealingPlugin: onMessage(): Unsupported message: topic={}, message={}, sender={}",
+                    topic, message, sender);
+        }
+    }
+
+    // ------------------------------------------------------------------------
+
+    private void processClusterNodeRemovedEvent(Object message) {
+        log.debug("SelfHealingPlugin: processClusterNodeRemovedEvent(): BEGIN: message={}", message);
+        if (message instanceof ClusterMembershipEvent) {
+            // Get removed node id and address
+            ClusterMembershipEvent event = (ClusterMembershipEvent)message;
+            String nodeId = event.subject().id().id();
+            String nodeAddress = event.subject().address().host();
+            log.debug("SelfHealingPlugin: processClusterNodeRemovedEvent(): node-id={}, node-address={}", nodeId, nodeAddress);
+            if (StringUtils.isBlank(nodeAddress)) {
+                log.warn("SelfHealingPlugin: processClusterNodeRemovedEvent(): Node address is missing. Cannot recover node. Initial message: {}", event);
+                return;
+            }
+
+            createRecoveryTask(nodeId, nodeAddress, recoveryContext, EmsClientRecoveryTask.class);
+        } else {
+            log.warn("SelfHealingPlugin: processClusterNodeRemovedEvent(): Message is not a {} object. Will ignore it.", ClusterMembershipEvent.class.getSimpleName());
+        }
+    }
+
+    private void processClusterNodeAddedEvent(Object message) {
+        log.debug("SelfHealingPlugin: processClusterNodeAddedEvent(): BEGIN: message={}", message);
+        if (message instanceof ClusterMembershipEvent) {
+            // Get added node id and address
+            ClusterMembershipEvent event = (ClusterMembershipEvent)message;
+            String nodeId = event.subject().id().id();
+            String nodeAddress = event.subject().address().host();
+            log.debug("SelfHealingPlugin: processClusterNodeAddedEvent(): node-id={}, node-address={}", nodeId, nodeAddress);
+            if (StringUtils.isBlank(nodeAddress)) {
+                log.warn("SelfHealingPlugin: processClusterNodeAddedEvent(): Node address is missing. Initial message: {}", event);
+                return;
+            }
+
+            // Cancel any waiting recovery task
+            cancelRecoveryTask(nodeId, nodeAddress, EmsClientRecoveryTask.class, false);
+        } else {
+            log.warn("SelfHealingPlugin: processClusterNodeAddedEvent(): Message is not a {} object. Will ignore it.", ClusterMembershipEvent.class.getSimpleName());
+        }
+    }
+
+    // ------------------------------------------------------------------------
+
+    private void processNetdataNodeFailedEvent(Object message) {
+        log.debug("SelfHealingPlugin: processNetdataNodeFailedEvent(): BEGIN: message={}", message);
+        if (!(message instanceof Map)) {
+            log.warn("SelfHealingPlugin: processNetdataNodeFailedEvent(): Message is not a {} object. Will ignore it.", Map.class.getSimpleName());
+            return;
+        }
+
+        // Get paused node address
+        Object addressValue = StrUtil.castToMapStringObject(message).getOrDefault("address", null);
+        log.debug("SelfHealingPlugin: processNetdataNodeFailedEvent(): node-address={}", addressValue);
+        if (addressValue==null) {
+            log.warn("SelfHealingPlugin: processNetdataNodeFailedEvent(): Node address is missing. Cannot recover node. Initial message: {}", message);
+            return;
+        }
+        String nodeAddress = addressValue.toString();
+
+        if (isLocalAddress(nodeAddress)) {
+            // We are responsible for recovering our local Netdata agent
+            createRecoveryTask(null, "", recoveryContext, NetdataAgentLocalRecoveryTask.class);
+        } else {
+            // Aggregator is responsible for recovering remote Netdata agents
+            createRecoveryTask(null, nodeAddress, recoveryContext, NetdataAgentRecoveryTask.class);
+        }
+    }
+
+    @SneakyThrows
+    private boolean isLocalAddress(String address) {
+        if (address.isEmpty()) return true;
+        if ("127.0.0.1".equals(address)) return true;
+        if ("::1".equals(address)) return true;
+        if ("0:0:0:0:0:0:0:1".equals(address)) return true;
+        InetAddress ia = InetAddress.getByName(address);
+        if (ia.isAnyLocalAddress() || ia.isLoopbackAddress()) return true;
+        try {
+            return NetworkInterface.getByInetAddress(ia) != null;
+        } catch (SocketException se) {
+            return false;
+        }
+    }
+
+    private void processNetdataNodeOkEvent(Object message) {
+        log.debug("SelfHealingPlugin: processNetdataNodeOkEvent(): BEGIN: message={}", message);
+        if (!(message instanceof Map)) {
+            log.warn("SelfHealingPlugin: processNetdataNodeOkEvent(): Message is not a {} object. Will ignore it.", Map.class.getSimpleName());
+            return;
+        }
+
+        // Get resumed node address
+        String nodeAddress = StrUtil.castToMapStringObject(message).getOrDefault("address", "").toString();
+        log.debug("SelfHealingPlugin: processNetdataNodeOkEvent(): node-address={}", nodeAddress);
+        /*if (StringUtils.isBlank(nodeAddress)) {
+            log.warn("SelfHealingPlugin: processNetdataNodeOkEvent(): Node address is missing. Initial message: {}", message);
+            return;
+        }*/
+
+        // Cancel any waiting recovery task
+        @NonNull Class<? extends RecoveryTask> recoverTaskClass =
+                StringUtils.isNotBlank(nodeAddress)
+                        ? NetdataAgentRecoveryTask.class
+                        : NetdataAgentLocalRecoveryTask.class;
+        cancelRecoveryTask(null, nodeAddress, recoverTaskClass, false);
+    }
+
+    // ------------------------------------------------------------------------
+
+    private void createRecoveryTask(String nodeId, @NonNull String nodeAddress, RecoveryContext recoveryContext, @NonNull Class<? extends RecoveryTask> recoveryTaskClass) {
+        // Check if a recovery task has already been scheduled
+        NodeKey nodeKey = new NodeKey(nodeAddress, recoveryTaskClass);
+        synchronized (waitingTasks) {
+            if (waitingTasks.containsKey(nodeKey)) {
+                log.warn("SelfHealingPlugin: createRecoveryTask(): Recovery has already been scheduled for Node: id={}, address={}", nodeId, nodeAddress);
+                return;
+            }
+            waitingTasks.put(nodeKey, null);
+        }
+
+        // Get node info and credentials from EMS server
+        Map nodeInfo = null;
+        if (StringUtils.isNotBlank(nodeAddress)) {
+            nodeInfo = nodeInfoHelper.getNodeInfo(nodeId, nodeAddress);
+            if (nodeInfo == null || nodeInfo.isEmpty()) {
+                log.warn("SelfHealingPlugin: createRecoveryTask(): Node info is null or empty. Cannot recover node.");
+                return;
+            }
+            log.trace("SelfHealingPlugin: createRecoveryTask(): Node info retrieved for node: id={}, address={}", nodeId, nodeAddress);
+        } else {
+            log.debug("SelfHealingPlugin: createRecoveryTask(): Node address is blank. Node info will not be retrieved: id={}, address={}", nodeId, nodeAddress);
+        }
+
+        // Schedule node recovery task
+        final RecoveryTask recoveryTask = applicationContext.getBean(recoveryTaskClass);
+        if (nodeInfo!=null && !nodeInfo.isEmpty())
+            recoveryTask.setNodeInfo(nodeInfo);
+        final AtomicInteger retries = new AtomicInteger(0);
+        Instant firstAttempt;
+        Duration retryDelay;
+        ScheduledFuture<?> future = taskScheduler.scheduleWithFixedDelay(
+                () -> {
+                    try {
+                        log.info("SelfHealingPlugin: Retry #{}: Recovering node: id={}, address={}", retries.get(), nodeId, nodeAddress);
+                        recoveryTask.runNodeRecovery(recoveryContext);
+                        //NOTE: 'recoveryTask.runNodeRecovery()' must send SELF_HEALING_RECOVERY_COMPLETED or _FAILED event
+                    } catch (Exception e) {
+                        log.error("SelfHealingPlugin: EXCEPTION while recovering node: node-address={} -- Exception: ", nodeAddress, e);
+                        eventBus.send(RecoveryConstant.SELF_HEALING_RECOVERY_FAILED, nodeAddress);
+                    }
+                    if (retries.getAndIncrement() >= selfHealingProperties.getRecovery().getMaxRetries()) {
+                        log.warn("SelfHealingPlugin: Max retries reached. No more recovery retries for node: id={}, address={}", nodeId, nodeAddress);
+                        cancelRecoveryTask(nodeId, nodeAddress, recoveryTaskClass, true);
+                        eventBus.send(RecoveryConstant.SELF_HEALING_RECOVERY_GIVE_UP, nodeAddress);
+
+                        // Notify EMS server about giving up recovery due to permanent failure
+                        commandExecutor.notifyEmsServer("RECOVERY GIVE_UP "+nodeId+" @ "+nodeAddress);
+                    }
+                },
+                firstAttempt = Instant.now().plusMillis(selfHealingProperties.getRecovery().getDelay()),
+                retryDelay = Duration.ofMillis(selfHealingProperties.getRecovery().getRetryDelay())
+        );
+        waitingTasks.put(nodeKey, future);
+        log.info("SelfHealingPlugin: createRecoveryTask(): Created recovery task for Node: id={}, address={}, first-attempt-at={}, retry-delay={}",
+                nodeId, nodeAddress, firstAttempt, DurationFormatUtils.formatDurationHMS(retryDelay.toMillis()));
+    }
+
+    private void cancelRecoveryTask(String nodeId, @NonNull String nodeAddress, @NonNull Class<? extends RecoveryTask> recoveryTaskClass, boolean retainNodeKey) {
+        NodeKey nodeKey = new NodeKey(nodeAddress, recoveryTaskClass);
+        synchronized (waitingTasks) {
+            ScheduledFuture<?> future = retainNodeKey ? waitingTasks.put(nodeKey, null) : waitingTasks.remove(nodeKey);
+            if (future != null) {
+                future.cancel(true);
+                nodeInfoHelper.remove(nodeId, nodeAddress);
+                log.info("SelfHealingPlugin: cancelRecoveryTask(): Cancelled recovery task for Node: id={}, address={}", nodeId, nodeAddress);
+            } else
+                log.debug("SelfHealingPlugin: cancelRecoveryTask(): No recovery task is scheduled for Node: id={}, address={}", nodeId, nodeAddress);
+        }
+    }
+
+    @Data
+    @AllArgsConstructor
+    protected static class NodeKey {
+        private String address;
+        @NonNull private Class<?> recoveryTaskClass;
+    }
+}

ems-core/baguette-client/src/main/resources/META-INF/spring.factories

@@ -0,0 +1 @@
+org.springframework.boot.env.EnvironmentPostProcessor=gr.iccs.imu.ems.util.NetUtilPostProcessor

ems-core/baguette-client/src/main/resources/banner-1.txt

@@ -0,0 +1,6 @@
+    ____                         __  __          _________            __ 
+   / __ )____ _____ ___  _____  / /_/ /____     / ____/ (_)__  ____  / /_
+  / __  / __ `/ __ `/ / / / _ \/ __/ __/ _ \   / /   / / / _ \/ __ \/ __/
+ / /_/ / /_/ / /_/ / /_/ /  __/ /_/ /_/  __/  / /___/ / /  __/ / / / /_  
+/_____/\__,_/\__, /\__,_/\___/\__/\__/\___/   \____/_/_/\___/_/ /_/\__/  
+            /____/                                                       

ems-core/baguette-client/src/main/resources/banner.txt

@@ -0,0 +1,8 @@
+  ____                         _   _          _____ _ _            _   
+ |  _ \                       | | | |        / ____| (_)          | |  
+ | |_) | __ _  __ _ _   _  ___| |_| |_ ___  | |    | |_  ___ _ __ | |_ 
+ |  _ < / _` |/ _` | | | |/ _ \ __| __/ _ \ | |    | | |/ _ \ '_ \| __|
+ | |_) | (_| | (_| | |_| |  __/ |_| ||  __/ | |____| | |  __/ | | | |_ 
+ |____/ \__,_|\__, |\__,_|\___|\__|\__\___|  \_____|_|_|\___|_| |_|\__|
+               __/ |                                                   
+              |___/                                                    

ems-core/baguette-server/pom.xml

@@ -0,0 +1,105 @@
+<!--
+  ~ Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+  ~
+  ~ This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+  ~ Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+  ~ If a copy of the MPL was not distributed with this file, you can obtain one at
+  ~ https://www.mozilla.org/en-US/MPL/2.0/
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>gr.iccs.imu.ems</groupId>
+        <artifactId>ems-core</artifactId>
+		<version>${revision}</version>
+	</parent>
+
+	<artifactId>baguette-server</artifactId>
+	<name>EMS - Baguette Server</name>
+
+	<dependencies>
+		<dependency>
+			<groupId>gr.iccs.imu.ems</groupId>
+			<artifactId>broker-cep</artifactId>
+			<version>${project.version}</version>
+			<scope>compile</scope>
+		</dependency>
+		<dependency>
+			<groupId>gr.iccs.imu.ems</groupId>
+			<artifactId>translator</artifactId>
+			<version>${project.version}</version>
+			<scope>compile</scope>
+			<exclusions>
+				<exclusion>
+					<groupId>*</groupId>
+					<artifactId>*</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>gr.iccs.imu.ems</groupId>
+			<artifactId>common</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/org.apache.sshd/apache-sshd -->
+		<dependency>
+			<groupId>org.apache.sshd</groupId>
+			<artifactId>apache-sshd</artifactId>
+			<version>${apache-sshd.version}</version>
+			<type>pom</type>
+			<exclusions>
+				<exclusion>
+					<groupId>org.slf4j</groupId>
+					<artifactId>slf4j-jdk14</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.bouncycastle</groupId>
+					<artifactId>*</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.springframework</groupId>
+					<artifactId>*</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.sshd</groupId>
+			<artifactId>sshd-scp</artifactId>
+			<version>${apache-sshd.version}</version>
+		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/org.projectlombok/lombok -->
+		<dependency>
+			<groupId>org.projectlombok</groupId>
+			<artifactId>lombok</artifactId>
+			<scope>provided</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter</artifactId>
+		</dependency>
+		<!-- https://mvnrepository.com/artifact/javax.validation/validation-api -->
+		<dependency>
+			<groupId>javax.validation</groupId>
+			<artifactId>validation-api</artifactId>
+			<version>2.0.1.Final</version>
+		</dependency>
+
+		<!-- Apache Commons Text (for StringSubstitutor) -->
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-text</artifactId>
+		</dependency>
+
+		<!-- For importing: class org.glassfish.jersey.internal.guava.InetAddresses -->
+		<dependency>
+			<groupId>org.glassfish.jersey.core</groupId>
+			<artifactId>jersey-common</artifactId>
+			<version>3.1.3</version>
+		</dependency>
+	</dependencies>
+
+</project>

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/BaguetteServer.java

@@ -0,0 +1,553 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server;
+
+import gr.iccs.imu.ems.baguette.server.properties.BaguetteServerProperties;
+import gr.iccs.imu.ems.brokercep.BrokerCepService;
+import gr.iccs.imu.ems.common.recovery.RecoveryConstant;
+import gr.iccs.imu.ems.common.selfhealing.SelfHealingManager;
+import gr.iccs.imu.ems.translate.TranslationContext;
+import gr.iccs.imu.ems.util.*;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringSubstitutor;
+import org.slf4j.event.Level;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.time.Instant;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Baguette Server
+ */
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class BaguetteServer implements InitializingBean, EventBus.EventConsumer<String, Object, Object> {
+    private final BaguetteServerProperties config;
+    private final PasswordUtil passwordUtil;
+    private final NodeRegistry nodeRegistry;
+
+    private final EventBus<String,Object,Object> eventBus;
+    @Getter
+    private final SelfHealingManager<NodeRegistryEntry> selfHealingManager;
+    private final TaskScheduler taskScheduler;
+
+    private Sshd server;
+
+    private Map<String, Set<String>> groupingTopicsMap;
+    private Map<String, Map<String, Set<String>>> groupingRulesMap;
+    private Map<String, Map<String, Set<String>>> topicConnections;
+    private Map<String, Double> constants;
+    private Set<FunctionDefinition> functionDefinitions;
+    private String upperwareGrouping;
+    private String upperwareBrokerUrl;
+    private BrokerCepService brokerCepService;
+
+    @Override
+    public void afterPropertiesSet() {
+        // Generate a new, random username/password pair and add it to provided credentials
+        generateUsernamePassword();
+    }
+
+    private void generateUsernamePassword() {
+        String genUsername = "user-"+UUID.randomUUID();
+        String genPassword = RandomStringUtils.randomAlphanumeric(32, 64);
+        CredentialsMap credentials = config.getCredentials();
+        credentials.put(genUsername, genPassword, true);
+        log.info("BaguetteServer: Generated new username/password: username={}, password={}",
+                genUsername, credentials.getPasswordEncoder()!=null
+                        ? credentials.getPasswordEncoder().encode(genPassword)
+                        : passwordUtil.encodePassword(genPassword));
+    }
+
+    // Configuration getter methods
+    public Set<String> getGroupingNames() {
+        return getGroupingNames(true);
+    }
+
+    public Set<String> getGroupingNames(boolean removeUpperware) {
+        Set<String> groupings = new HashSet<>();
+        groupings.addAll(groupingTopicsMap.keySet());
+        groupings.addAll(groupingRulesMap.keySet());
+        groupings.addAll(topicConnections.keySet());
+        // remove upperware grouping (i.e. GLOBAL)
+        if (removeUpperware) groupings.remove(upperwareGrouping);
+        return groupings;
+    }
+
+    private List<GROUPING> getGroupingsSorted(boolean removeUpperware, boolean ascending) {
+        List<GROUPING> list = getGroupingNames(removeUpperware).stream()
+                .map(GROUPING::valueOf)
+                .sorted()
+                .collect(Collectors.toList());
+        if (ascending) Collections.reverse(list);
+        return list;
+    }
+
+    private List<String> getGroupingNamesSorted(boolean removeUpperware, boolean ascending) {
+        return getGroupingsSorted(removeUpperware, ascending).stream()
+                .map(GROUPING::name)
+                .collect(Collectors.toList());
+    }
+
+    private String getLowestLevelGroupingName() {
+        List<String> list = getGroupingNamesSorted(false, true);
+        return !list.isEmpty() ? list.get(0) : null;
+    }
+
+    public BaguetteServerProperties getConfiguration() {
+        return config;
+    }
+
+    public Set<String> getTopicsForGrouping(String grouping) {
+        return groupingTopicsMap.get(grouping);
+    }
+
+    public Map<String, Set<String>> getRulesForGrouping(String grouping) {
+        return groupingRulesMap.get(grouping);
+    }
+
+    public Map<String, Set<String>> getTopicConnectionsForGrouping(String grouping) {
+        return topicConnections.get(grouping);
+    }
+
+    public Map<String, Double> getConstants() {
+        return constants;
+    }
+
+    public Set<FunctionDefinition> getFunctionDefinitions() {
+        return functionDefinitions;
+    }
+
+    public String getUpperwareGrouping() { return upperwareGrouping; }
+
+    public String getUpperwareBrokerUrl() { return upperwareBrokerUrl; }
+
+    public String getBrokerUsername() { return brokerCepService.getBrokerUsername(); }
+
+    public String getBrokerPassword() { return brokerCepService.getBrokerPassword(); }
+
+    public BrokerCepService getBrokerCepService() { return brokerCepService; }
+
+    public String getServerPubkey() { return server.getPublicKey(); }
+
+    public String getServerPubkeyFingerprint() { return server.getPublicKeyFingerprint(); }
+
+    public String getServerPubkeyAlgorithm() { return server.getPublicKeyAlgorithm(); }
+
+    public String getServerPubkeyFormat() { return server.getPublicKeyFormat(); }
+
+    public NodeRegistry getNodeRegistry() { return nodeRegistry; }
+
+    // Server control methods
+    public synchronized void startServer(ServerCoordinator coordinator) throws IOException {
+        if (server == null) {
+            eventBus.subscribe(RecoveryConstant.SELF_HEALING_RECOVERY_GIVE_UP, this);
+
+            log.info("BaguetteServer.startServer(): Starting SSH server...");
+            nodeRegistry.setCoordinator(coordinator);
+            Sshd server = new Sshd();
+            server.start(config, coordinator, eventBus, nodeRegistry);
+            server.setNodeRegistry(getNodeRegistry());
+            this.server = server;
+            log.info("BaguetteServer.startServer(): Starting SSH server... done");
+        } else {
+            log.info("BaguetteServer.startServer(): SSH server is already running");
+        }
+    }
+
+    public synchronized void stopServer() throws IOException {
+        if (server != null) {
+            eventBus.unsubscribe(RecoveryConstant.SELF_HEALING_RECOVERY_GIVE_UP, this);
+
+            log.info("BaguetteServer.setServerConfiguration(): stopping SSH server...");
+            server.stop();
+            this.server = null;
+            nodeRegistry.setCoordinator(null);
+            log.info("BaguetteServer.setServerConfiguration(): stopping SSH server... done");
+        } else {
+            log.info("BaguetteServer.stop(): No SSH server instance is running");
+        }
+    }
+
+    public synchronized void restartServer(ServerCoordinator coordinator) throws IOException {
+        stopServer();
+        startServer(coordinator);
+    }
+
+    public synchronized boolean isServerRunning() {
+        return server != null;
+    }
+
+    @Override
+    public void onMessage(String topic, Object message, Object sender) {
+        log.trace ("BaguetteServer.onMessage: BEGIN: topic={}, message={}, sender={}", topic, message, sender);
+
+        String nodeAddress = (message!=null) ? message.toString() : null;
+        log.trace("BaguetteServer.onMessage: nodeAddress={}", nodeAddress);
+
+        if (RecoveryConstant.SELF_HEALING_RECOVERY_GIVE_UP.equals(topic)) {
+            if (StringUtils.isNotBlank(nodeAddress)) {
+                NodeRegistryEntry node = nodeRegistry.getNodeByAddress(nodeAddress);
+                if (node!=null) {
+                    node.nodeFailed(null);
+                    log.info("BaguetteServer.onMessage: Marked Node as Failed: {}", nodeAddress);
+                } else {
+                    log.warn("BaguetteServer.onMessage: Node with Address not found: {}", nodeAddress);
+                    log.debug("BaguetteServer.onMessage: Node addresses: {}", nodeRegistry.getNodeAddresses());
+                }
+            }
+        } else {
+            log.warn("BaguetteServer.onMessage: Event from unexpected topic received. Ignoring it: {}", topic);
+        }
+    }
+
+    // Topology configuration methods
+    public synchronized void setTopologyConfiguration(
+            TranslationContext _TC,
+            Map<String, Double> constants,
+            String upperwareGrouping,
+            BrokerCepService brokerCepService)
+            throws IOException
+    {
+        log.debug("BaguetteServer.setTopologyConfiguration(): BEGIN");
+
+        // Set new configuration
+        this.groupingTopicsMap = _TC.getG2T();
+        this.groupingRulesMap = _TC.getG2R();
+        this.topicConnections = _TC.getTopicConnections();
+        this.constants = constants;
+        this.functionDefinitions = _TC.getFunctionDefinitions();
+        this.upperwareGrouping = upperwareGrouping;
+        this.upperwareBrokerUrl = brokerCepService.getBrokerCepProperties().getBrokerUrlForClients();
+        this.brokerCepService = brokerCepService;
+
+        // Print new configuration
+        log.debug("BaguetteServer.setTopologyConfiguration(): Grouping-to-Topics (G2T): {}", groupingTopicsMap);
+        log.debug("BaguetteServer.setTopologyConfiguration(): Grouping-to-Rules (G2R): {}", groupingRulesMap);
+        log.debug("BaguetteServer.setTopologyConfiguration(): Topic-Connections: {}", topicConnections);
+        log.debug("BaguetteServer.setTopologyConfiguration(): Constants: {}", constants);
+        log.debug("BaguetteServer.setTopologyConfiguration(): Function-Definitions: {}", functionDefinitions);
+        log.debug("BaguetteServer.setTopologyConfiguration(): Upperware-grouping: {}", upperwareGrouping);
+        log.debug("BaguetteServer.setTopologyConfiguration(): Upperware-broker-url: {}", upperwareBrokerUrl);
+        log.debug("BaguetteServer.setTopologyConfiguration(): Broker-credentials: username={}, password={}",
+                brokerCepService.getBrokerUsername(), passwordUtil.encodePassword(brokerCepService.getBrokerPassword()));
+
+        // Stop any running instance of SSH server
+        stopServer();
+
+        // Clear node registry
+        nodeRegistry.clearNodes();
+
+        log.debug("BaguetteServer.setTopologyConfiguration(): Baguette server configuration: {}", config);
+        log.debug("BaguetteServer.setTopologyConfiguration(): Baguette Server credentials: {}", config.getCredentials());
+
+        // Initialize server coordinator
+        log.debug("BaguetteServer.setTopologyConfiguration(): Initializing Baguette protocol coordinator...");
+        ServerCoordinator coordinator = createServerCoordinator(config, _TC, upperwareGrouping);
+        log.debug("BaguetteServer.setTopologyConfiguration(): Coordinator: {}", coordinator.getClass().getName());
+        coordinator.initialize(_TC, upperwareGrouping, this, () ->
+                {
+                    log.info("****************************************");
+                    log.info("****  MONITORING TOPOLOGY IS READY  ****");
+                    log.info("****************************************");
+                }
+        );
+
+        // Start a new instance of SSH server
+        startServer(coordinator);
+
+        log.debug("BaguetteServer.setTopologyConfiguration(): END");
+    }
+
+    protected static ServerCoordinator createServerCoordinator(BaguetteServerProperties config, TranslationContext _TC, String upperwareGrouping) {
+        // Initialize coordinator class and parameters for backward compatibility
+        Class<ServerCoordinator> coordinatorClass = config.getCoordinatorClass();
+        Map<String, String> coordinatorParams = config.getCoordinatorParameters();
+
+        // Check if Coordinator Id has been specified (this overrides)
+        for (String id : config.getCoordinatorId()) {
+            if (StringUtils.isBlank(id))
+                throw new IllegalArgumentException("Coordinator Id cannot be null or blank");
+
+            // Get coordinator class and parameters by Id
+            BaguetteServerProperties.CoordinatorConfig coordConfig = config.getCoordinatorConfig().get(id);
+            if (coordConfig == null)
+                throw new IllegalArgumentException("Not found coordinator configuration with id: " + id);
+            coordinatorClass = coordConfig.getCoordinatorClass();
+            if (coordinatorClass == null)
+                throw new IllegalArgumentException("Not found coordinator class in configuration with id: " + id);
+            coordinatorParams = coordConfig.getParameters();
+
+            // Initialize coordinator instance
+            ServerCoordinator coordinator = createServerCoordinator(id, coordinatorClass, coordinatorParams, _TC, upperwareGrouping);
+
+            if (coordinator != null)
+                return coordinator;
+            // else try the next coordinator in configuration
+        }
+
+        if (coordinatorClass == null)
+            throw new IllegalArgumentException("Either coordinator class or coordinator id must be specified");
+
+        // Initialize coordinator class and parameters for backward compatibility
+        ServerCoordinator coordinator = createServerCoordinator(null, coordinatorClass, coordinatorParams, _TC, upperwareGrouping);
+        if (coordinator == null) {
+            log.error("No configured coordinator supports Translation Context.\nCoordinator Id's: {}\nDefault coordinator: {}\nTranslation Context:\n{}",
+                    config.getCoordinatorId(), coordinatorClass, _TC);
+            throw new IllegalArgumentException("No configured coordinator supports Translation Context");
+        }
+        return coordinator;
+    }
+
+    @SneakyThrows
+    private static ServerCoordinator createServerCoordinator(String id, Class<ServerCoordinator> coordinatorClass, Map<String,String> coordinatorParams, TranslationContext _TC, String upperwareGrouping) {
+        log.debug("createServerCoordinator: Instantiating coordinator with id: {}", id);
+
+        // Initialize coordinator instance
+        ServerCoordinator coordinator = coordinatorClass.getConstructor().newInstance();
+
+        // Set coordinator parameters
+        coordinator.setProperties(coordinatorParams);
+
+        // Check if coordinator supports this Translation Context
+        if (!coordinator.isSupported(_TC)) {
+            log.debug("createServerCoordinator: Coordinator does not support Translation Context: id={}", id);
+            return null;
+        }
+
+        log.debug("createServerCoordinator: Coordinator supports Translation Context: id={}", id);
+        return coordinator;
+    }
+
+    public void sendToActiveClients(String command) {
+        server.sendToActiveClients(command);
+    }
+
+    public void sendToClient(String clientId, String command) {
+        server.sendToClient(clientId, command);
+    }
+
+    public void sendToActiveClusters(String command) {
+        server.sendToActiveClusters(command);
+    }
+
+    public void sendToCluster(String clusterId, String command) {
+        server.sendToCluster(clusterId, command);
+    }
+
+    public Object readFromClient(String clientId, String command, Level logLevel) {
+        return server.readFromClient(clientId, command, logLevel);
+    }
+
+    public List<String> getActiveClients() {
+        return ClientShellCommand.getActive().stream()
+                .map(c -> {
+                    NodeRegistryEntry entry = getNodeRegistryEntryFromClientShellCommand(c);
+                    return formatClientList(c, entry);
+                })
+                .sorted()
+                .collect(Collectors.toList());
+    }
+
+    public Map<String, Map<String, String>> getActiveClientsMap() {
+        return ClientShellCommand.getActive().stream()
+                .map(c -> {
+                    NodeRegistryEntry entry = getNodeRegistryEntryFromClientShellCommand(c);
+                    return prepareClientMap(c, entry);
+                })
+                .sorted(Comparator.comparing(m -> m.get("id")))
+                .collect(Collectors.toMap(m -> m.get("id"), m -> m,
+                        (u,v) -> { throw new IllegalStateException(String.format("Duplicate key %s", u)); },
+                        LinkedHashMap::new));
+    }
+
+    private NodeRegistryEntry getNodeRegistryEntryFromClientShellCommand(ClientShellCommand c) {
+        NodeRegistryEntry entry = c.getNodeRegistryEntry();
+        if (entry==null)
+            entry = getNodeRegistry().getNodeByAddress(c.getClientIpAddress());
+        log.debug("getNodeRegistryEntryFromClientShellCommand: CSC ip-address: {}", c.getClientIpAddress());
+        log.debug("getNodeRegistryEntryFromClientShellCommand: CSC NR entry: {}", entry!=null ? entry.getPreregistration() : null);
+        /*if (entry==null) {
+            log.warn("getNodeRegistryEntryFromClientShellCommand: WARN: ** NOT SECURE ** CSC client-id: {}", c.getClientId());
+            entry = getNodeRegistry().getNodeByClientId(c.getClientId());
+            log.debug("getNodeRegistryEntryFromClientShellCommand: WARN: ** NOT SECURE ** CSC NR entry: {}", entry!=null ? entry.getPreregistration() : null);
+        }*/
+        return entry;
+    }
+
+    public List<String> getNodesWithoutClient() {
+        return createClientList(new HashSet<>(Collections.singletonList(NodeRegistryEntry.STATE.NOT_INSTALLED)));
+    }
+
+    public Map<String, Map<String, String>> getNodesWithoutClientMap() {
+        return createClientMap(new HashSet<>(Collections.singletonList(NodeRegistryEntry.STATE.NOT_INSTALLED)));
+    }
+
+    public List<String> getIgnoredNodes() {
+        return createClientList(new HashSet<>(Collections.singletonList(NodeRegistryEntry.STATE.IGNORE_NODE)));
+    }
+
+    public Map<String, Map<String, String>> getIgnoredNodesMap() {
+        return createClientMap(new HashSet<>(Collections.singletonList(NodeRegistryEntry.STATE.IGNORE_NODE)));
+    }
+
+    public List<String> getPassiveNodes() {
+        return createClientList(new HashSet<>(Arrays.asList(NodeRegistryEntry.STATE.NOT_INSTALLED, NodeRegistryEntry.STATE.IGNORE_NODE)));
+    }
+
+    public Map<String, Map<String, String>> getPassiveNodesMap() {
+        return createClientMap(new HashSet<>(Arrays.asList(NodeRegistryEntry.STATE.NOT_INSTALLED, NodeRegistryEntry.STATE.IGNORE_NODE)));
+    }
+
+    public List<String> getAllNodes() {
+        return createClientList(new HashSet<>(Arrays.asList(NodeRegistryEntry.STATE.values())));
+    }
+
+    public Map<String, Map<String, String>> getAllNodesMap() {
+        return createClientMap(new HashSet<>(Arrays.asList(NodeRegistryEntry.STATE.values())));
+    }
+
+    private List<String> createClientList(Set<NodeRegistryEntry.STATE> states) {
+        return nodeRegistry.getNodes().stream()
+                .filter(entry->states.contains(entry.getState()))
+                .map(entry -> {
+                    log.debug("createClientList: Node ip-address: {}", entry.getIpAddress());
+                    log.debug("createClientList: Node preregistration info: {}", entry.getPreregistration());
+                    ClientShellCommand c = getClientShellCommandFromNodeRegistryEntry(entry);
+                    return formatClientList(c, entry);
+                })
+                .sorted()
+                .collect(Collectors.toList());
+    }
+
+    private Map<String, Map<String, String>> createClientMap(Set<NodeRegistryEntry.STATE> states) {
+        return nodeRegistry.getNodes().stream()
+                .filter(entry -> states.contains(entry.getState()))
+                .sorted(Comparator.comparing(NodeRegistryEntry::getClientId))
+                .collect(Collectors.toMap(NodeRegistryEntry::getClientId, entry -> {
+                    log.debug("createClientMap: Node ip-address: {}", entry.getIpAddress());
+                    log.debug("createClientMap: Node preregistration info: {}", entry.getPreregistration());
+                    ClientShellCommand c = getClientShellCommandFromNodeRegistryEntry(entry);
+                    return prepareClientMap(c, entry);
+                }, (u,v) -> { throw new IllegalStateException(String.format("Duplicate key %s", u)); }, LinkedHashMap::new));
+    }
+
+    private ClientShellCommand getClientShellCommandFromNodeRegistryEntry(NodeRegistryEntry entry) {
+        return StringUtils.isNotBlank(entry.getIpAddress())
+                ? ClientShellCommand.getActiveByIpAddress(entry.getIpAddress()) : null;
+    }
+
+    private String formatClientList(ClientShellCommand c, NodeRegistryEntry entry) {
+        final StringBuilder sb = new StringBuilder();
+        prepareClientMap(c, entry).forEach((k,v)->{
+            if ("id".equals(k)) sb.append(v);
+            else if ("node-port".equals(k)) sb.append(":").append(v);
+            else sb.append(" ").append(v);
+        });
+        return sb.toString();
+    }
+
+    private Map<String, String> prepareClientMap(ClientShellCommand c, NodeRegistryEntry entry) {
+        // Get node hostname
+        String address = entry!=null ? entry.getIpAddress() : c.getClientIpAddress();
+        String hostname = entry!=null ? entry.getHostname() : null;
+        if (StringUtils.isBlank(hostname)) {
+            if (c!=null)
+                hostname = c.getClientClusterNodeHostname();
+            if (StringUtils.isNotBlank(hostname)) {
+                if (c!=null) c.setClientClusterNodeHostname(hostname);
+                if (entry!=null) entry.setHostname(hostname);
+            }
+
+            // Resolve hostname in a separate thread to avoid blocking this method (and the Web Admin updates)
+            if (config.isResolveHostname() && StringUtils.isBlank(hostname)) {
+                taskScheduler.schedule(()->{
+                    try {
+                        String _hostname = InetAddress.getByName(address).getHostName();
+                        if (StringUtils.isNotBlank(_hostname)) {
+                            if (c!=null) c.setClientClusterNodeHostname(_hostname);
+                            if (entry!=null) entry.setHostname(_hostname);
+                        }
+                    } catch (Exception e) {
+                        log.warn("Failed to resolve client hostname from IP address: {}\n", address, e);
+                    }
+                }, Instant.now());
+            }
+        }
+
+        // Prepare node info map
+        Map<String,String> properties = new LinkedHashMap<>();
+        properties.put("id", c!=null ? c.getId() : entry.getClientId());
+        properties.put("ip-address", address);
+        properties.put("node-hostname", c!=null ? c.getClientClusterNodeHostname() : hostname);
+        properties.put("node-port", Integer.toString(c!=null ? c.getClientClusterNodePort() : -1));
+        properties.put("node-status", c!=null ? c.getClientNodeStatus() : null);
+        properties.put("node-zone", (entry!=null && entry.getClusterZone()!=null) ? entry.getClusterZone().getId() : null);  //c.getClientZone()!=null ? c.getClientZone().getId() : null
+        properties.put("grouping", c!=null ? c.getClientGrouping() : (entry.getState()==NodeRegistryEntry.STATE.NOT_INSTALLED ? getLowestLevelGroupingName() : null));
+        properties.put("reference", entry!=null ? entry.getReference() : null);
+        properties.put("node-id", c!=null ? c.getClientProperty("node-id") : null);
+        properties.put("node-state", entry!=null && entry.getState()!=null ? entry.getState().toString() : null);
+        properties.put("errors", entry!=null && entry.getErrors()!=null
+                ? entry.getErrors().stream()
+                        .filter(Objects::nonNull)
+                        .map(Object::toString)
+                        .collect(Collectors.joining(" | "))
+                : null);
+        return properties;
+    }
+
+    public void sendConstants(Map<String, Double> constants) {
+        server.sendConstants(constants);
+    }
+
+    public NodeRegistryEntry registerClient(Map<String,?> nodeInfoMap) throws UnknownHostException {
+        log.debug("BaguetteServer.registerClient(): node-info={}", nodeInfoMap);
+
+        Map<String,Object> nodeInfo = new HashMap<>(nodeInfoMap);
+
+        // Create client id and random UUID
+        String clientId = nodeInfoMap.get("CLIENT_ID")!=null && StringUtils.isNotBlank(nodeInfoMap.get("CLIENT_ID").toString())
+                ? nodeInfoMap.get("CLIENT_ID").toString()
+                : generateClientIdFromNodeInfo(nodeInfo);
+        Object randomUuid = UUID.randomUUID().toString();
+        nodeInfo.put("random", randomUuid);
+        log.debug("BaguetteServer.registerClient(): client-id={}, random-UUID={}", clientId, randomUuid);
+
+        // Add node info into node registry
+        return nodeRegistry.addNode(nodeInfo, clientId);
+    }
+
+    public String generateClientIdFromNodeInfo(Map<String, ?> nodeInfo) {
+        String clientId;
+        String formatter = getConfiguration().getClientIdFormat();
+        if (StringUtils.isBlank(formatter)) {
+            log.debug("BaguetteServer.registerClient(): No formatter specified. A random uuid will be returned");
+            clientId = UUID.randomUUID().toString();
+        } else {
+            String escape = Optional.ofNullable(getConfiguration().getClientIdFormatEscape()).orElse("~");
+            formatter = formatter.replace(escape,"$");
+            log.debug("BaguetteServer.registerClient(): formatter={}", formatter);
+            clientId = StringSubstitutor.replace(formatter, nodeInfo);
+        }
+        return clientId;
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/ClientShellCommand.java

@@ -0,0 +1,735 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.gson.Gson;
+import gr.iccs.imu.ems.baguette.server.coordinator.cluster.IClusterZone;
+import gr.iccs.imu.ems.common.recovery.RecoveryConstant;
+import gr.iccs.imu.ems.util.*;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.sshd.common.session.Session;
+import org.apache.sshd.common.session.SessionListener;
+import org.apache.sshd.server.channel.ChannelSession;
+import org.apache.sshd.server.command.Command;
+import org.apache.sshd.server.Environment;
+import org.apache.sshd.server.ExitCallback;
+import org.apache.sshd.server.session.ServerSession;
+import org.apache.sshd.server.session.ServerSessionAware;
+import org.cryptacular.util.CertUtil;
+import org.slf4j.event.Level;
+
+import javax.validation.constraints.NotBlank;
+import java.io.*;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.security.cert.X509Certificate;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class ClientShellCommand implements Command, Runnable, ServerSessionAware {
+
+    private final static Object LOCK = new Object();
+    private final static AtomicLong counter = new AtomicLong(0);
+    private final static Set<ClientShellCommand> activeCmdList = new HashSet<>();
+    private final static Map<String,ClientShellCommand> activeCmdMap = new HashMap<>();
+    private final static long INPUT_CHECK_DELAY = 100;
+
+    public static Set<ClientShellCommand> getActive() {
+        return Collections.unmodifiableSet(activeCmdList);
+    }
+
+    public static Set<String> getActiveIds() {
+        return Collections.unmodifiableSet(activeCmdMap.keySet());
+    }
+
+    public static ClientShellCommand getActiveByIpAddress(@NotBlank String address) {
+        return activeCmdMap.get(address);
+    }
+
+    public static ClientShellCommand getActiveById(@NotBlank String id) {
+        return activeCmdList.stream().filter(csc->csc.getId().equals(id)).findFirst().orElse(null);
+    }
+
+    private InputStream in;
+    private PrintStream out;
+    private PrintStream err;
+    private ExitCallback callback;
+    private final AtomicBoolean callbackCalled = new AtomicBoolean(false);
+
+    @Getter @Setter
+    private String id;
+    @Getter @Setter
+    private boolean echoOn = false;
+
+    private String clientId;
+    @Getter private String clientBrokerUrl;
+    @Getter private String clientBrokerUsername;
+    @Getter private String clientBrokerPassword;
+    private String clientIpAddress;
+    private String clientHostname;
+    private String clientCanonicalHostname;
+    private int clientPort = -1;
+    @Getter private String clientCertificate;   // Broker certificate of Client
+
+    @Getter @Setter private int clientClusterNodePort;
+    @Getter @Setter private String clientClusterNodeAddress;
+    @Getter @Setter private String clientClusterNodeHostname;
+    @Getter @Setter private IClusterZone clientZone;
+    @Getter private String clientNodeStatus;
+    @Getter private String clientGrouping;
+    private final Properties clientProperties = new Properties();
+
+    private final ServerCoordinator coordinator;
+    private final boolean clientAddressOverrideAllowed;
+    @Getter
+    private ServerSession session;
+    @Getter @Setter
+    private boolean closeConnection = false;
+
+    private final Map<String,Object> inputsMap = new HashMap<>();
+    private final EventBus<String,Object,Object> eventBus;
+    @Getter
+    private Exception lastException;
+    @JsonIgnore
+    private final transient NodeRegistry nodeRegistry;
+    @Setter
+    private NodeRegistryEntry nodeRegistryEntry;
+
+    @Getter
+    private Map<String, Object> clientStatistics;
+
+    public ClientShellCommand(ServerCoordinator coordinator, boolean allowClientOverrideItsAddress, EventBus<String,Object,Object> eventBus, NodeRegistry registry) {
+        synchronized (LOCK) {
+            id = String.format("#%05d", counter.getAndIncrement());
+        }
+        this.coordinator = coordinator;
+        this.clientAddressOverrideAllowed = allowClientOverrideItsAddress;
+        this.eventBus = eventBus;
+        this.nodeRegistry = registry;
+    }
+
+    @JsonIgnore
+    public NodeRegistry getNodeRegistry() {
+        return nodeRegistry;
+    }
+
+    public void setSession(ServerSession session) {
+        log.info("{}--> Got session : {}", id, session);
+        this.session = session;
+        eventBus.send("BAGUETTE_SERVER_CLIENT_SESSION_STARTED", this);
+
+		/*try {
+			String clientIpAddr = ((InetSocketAddress)session.getIoSession().getRemoteAddress()).getAddress().getHostAddress();
+			int clientPort = ((InetSocketAddress)session.getIoSession().getRemoteAddress()).getPort();
+			log.info("{}--> Client connection : {}:{}", id, clientIpAddr, clientPort);
+			String username = session.getUsername();
+			log.info("{}--> Client session username: {}", username);
+		} catch (Exception ex) {}*/
+
+        session.addSessionListener(new SessionListener() {
+            @Override
+            public void sessionException(Session session, Throwable t) {
+                log.warn("{}--> SessionListener: sessionException Throwable: ", id, t);
+            }
+            @Override
+            public void sessionClosed(Session session) {
+                log.info("{}--> SessionListener: sessionClosed", id);
+            }
+        });
+
+        // Initialize NodeRegistryEntry for this CSC
+        initNodeRegistryEntry();
+    }
+
+    private void initNodeRegistryEntry() {
+        String address = getClientIpAddress();
+        NodeRegistryEntry entry = coordinator.getServer().getNodeRegistry().getNodeByAddress(address);
+        log.debug("{}--> initNodeRegistryEntry: Node registry entry for CSC: address={}, entry={}", id, address, entry);
+        log.trace("{}--> initNodeRegistryEntry: Current nodeRegistryEntry: {}", id, entry);
+        if (entry!=null) {
+            setNodeRegistryEntry(entry);
+        } else {
+            log.error("{}--> initNodeRegistryEntry: No node registry entry found for client: address={}", id, address);
+            log.error("{}--> initNodeRegistryEntry: Marked client session for immediate close: address={}", id, address);
+            setCloseConnection(true);
+        }
+    }
+
+    public void setInputStream(InputStream in) {
+        this.in = in;
+    }
+
+    public void setOutputStream(OutputStream out) {
+        this.out = new PrintStream(out, true);
+    }
+
+    public void setErrorStream(OutputStream err) {
+        this.err = new PrintStream(err, true);
+    }
+
+    public void setExitCallback(ExitCallback callback) {
+        this.callback = callback;
+    }
+
+    @Override
+    public void start(ChannelSession channelSession, Environment environment) throws IOException {
+        new Thread(this).start();
+    }
+
+    @Override
+    public void destroy(ChannelSession channelSession) throws Exception {
+    }
+
+    public void run() {
+        // Check if session has been marked for immediate close
+        if (closeConnection) {
+            log.warn("{}--> Exiting immediately because 'closeConnection' flag is set", id);
+            eventBus.send("BAGUETTE_SERVER_CLIENT_SESSION_CLOSING_IMMEDIATELY", this);
+            coordinator.unregister(this);
+            if (this.session!=null && this.session.isOpen()) {
+                try {
+                    this.session.close();
+                } catch (IOException e) {
+                    log.warn("Closing session caused on exception: ", e);
+                }
+                this.session = null;
+            }
+            if (!callbackCalled.getAndSet(true)) {
+                callback.onExit(2);
+            }
+            log.info("{}--> Thread stopped immediately", id);
+            eventBus.send("BAGUETTE_SERVER_CLIENT_SESSION_CLOSED_IMMEDIATELY", this);
+            return;
+        }
+
+        // Add this CSC in active list
+        synchronized (activeCmdList) {
+            if (activeCmdMap.containsKey(getClientIpAddress()) || activeCmdMap.containsValue(this))
+                throw new IllegalArgumentException("ClientShellCommand has already been registered");
+            activeCmdList.add(this);
+            activeCmdMap.put(getClientIpAddress(), this);
+        }
+        eventBus.send("BAGUETTE_SERVER_CLIENT_STARTING", this);
+        getNodeRegistryEntry().nodeRegistering(null);
+
+        // Process client input
+        try {
+            log.info("{}==> Thread started", id);
+            out.printf("CLIENT (%s) : START\n", id);
+
+            this.clientIpAddress = getClientIpAddress();
+
+            // Enter the main processing loop
+            BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+            String line;
+            boolean helloReceived = false;
+            while ((line = reader.readLine()) != null) {
+                line = line.trim();
+                log.debug("{}--> {}", id, line);
+
+                // Echo command (if configured)
+                //if (echoOn) out.printf("CLIENT (%s) : ECHO : %s\n", id, line);
+                if (echoOn) out.printf("ECHO %s\n", line);
+                //if (line.equalsIgnoreCase("exit")) break;
+
+                if (!helloReceived && line.startsWith("-HELLO FROM CLIENT:")) {
+                    // Process the Greeting line from client -- It must be the first line received
+                    helloReceived = true;
+                    getClientInfoFromGreeting(line.substring("-HELLO FROM CLIENT:".length()));
+
+                    // Register CSC to Coordinator
+                    coordinator.register(this);
+                    eventBus.send("BAGUETTE_SERVER_CLIENT_REGISTERED", this);
+                    getNodeRegistryEntry().nodeRegistered(null);
+
+                    // Instruct client to start sending statistics
+                    sendCommand("SEND-STATS START");
+                } else {
+                    // Process the subsequent lines from client -- After the Greeting line
+                    processClientInput(line);
+                }
+            }
+            // Client connection closed
+            eventBus.send("BAGUETTE_SERVER_CLIENT_EXITING", this);
+            getNodeRegistryEntry().nodeExiting(null);
+
+            log.info("{}==> Signaling client to exit", id);
+            out.println("EXIT");
+
+        } catch (Exception ex) {
+            log.warn("{}==> EXCEPTION : ", id, ex);
+            out.printf("EXCEPTION %s\n", ex);
+            this.lastException = ex;
+            eventBus.send("BAGUETTE_SERVER_CLIENT_EXCEPTION", this);
+            NodeRegistryEntry entry = getNodeRegistryEntry();
+            if (entry.getState()==NodeRegistryEntry.STATE.REGISTERING) entry.nodeRegistrationError(ex);
+            else entry.nodeDisconnected(ex);
+        } finally {
+            // Remove CSC from active list
+            synchronized (activeCmdList) {
+                activeCmdList.remove(this);
+                activeCmdMap.remove(getClientIpAddress());
+            }
+            log.info("{}--> Thread stops", id);
+
+            // Unregister from Coordinator
+            coordinator.unregister(this);
+            eventBus.send("BAGUETTE_SERVER_CLIENT_UNREGISTERED", this);
+
+            // Invoke callback if provided
+            if (!callbackCalled.getAndSet(true)) {
+                callback.onExit(0);
+            }
+            eventBus.send("BAGUETTE_SERVER_CLIENT_EXITED", this);
+            if (getNodeRegistryEntry().getState()==NodeRegistryEntry.STATE.EXITING)
+                getNodeRegistryEntry().nodeExited(null);
+        }
+    }
+
+    private void processClientInput(String line) throws IOException, ClassNotFoundException {
+        if (line.startsWith("-INPUT:")) {
+            String input = line.substring("-INPUT:".length());
+            String[] part = input.split(":",2 );
+            inputsMap.put(part[0].trim(), SerializationUtil.deserializeFromString(part[1]));
+        } else if (StringUtils.startsWithIgnoreCase(line, "SERVER-")) {
+            String[] lineArgs = line.split(" ", 2);
+            if ("SERVER-GET-NODE-SSH-CREDENTIALS".equalsIgnoreCase(lineArgs[0].trim()) && lineArgs.length>1) {
+                String nodeAddress = lineArgs[1].trim();
+                if (!nodeAddress.isEmpty()) {
+                    NodeRegistryEntry entry = nodeRegistry.getNodeByAddress(nodeAddress);
+                    if (entry!=null) {
+                        Map<String, String> preregInfo = entry.getPreregistration();
+                        log.debug("{}--> NODE PRE-REGISTRATION INFO: address={}\n{}", getId(), nodeAddress, preregInfo);
+
+                        if (preregInfo!=null) {
+                            String preregInfoStr = new Gson().toJson(preregInfo);
+                            log.trace("{}--> NODE PRE-REGISTRATION INFO STRING: STR={}\n{}", getId(), nodeAddress, preregInfoStr);
+                            sendToClient(preregInfoStr);
+                        } else {
+                            log.warn("{}--> NO PRE-REGISTRATION INFO FOR NODE: {}", getId(), nodeAddress);
+                            sendToClient("{}");
+                        }
+                    } else {
+                        log.warn("{}--> UNKNOWN NODE: {}", getId(), nodeAddress);
+                        sendToClient("{}");
+                    }
+                }
+            }
+        } else if (line.startsWith("-NOTIFY-GROUPING-CHANGE:")) {
+            String newGrouping = line.substring("-NOTIFY-GROUPING-CHANGE:".length()).trim();
+            log.info("{}--> Client grouping changed: {} --> {}", getId(), clientGrouping, newGrouping);
+            if (StringUtils.isNotBlank(newGrouping) && ! StringUtils.equals(clientGrouping, newGrouping))
+                this.clientGrouping = newGrouping;
+        } else if (line.startsWith("-NOTIFY-STATUS-CHANGE:")) {
+            String newNodeStatus = line.substring("-NOTIFY-STATUS-CHANGE:".length()).trim();
+            log.info("{}--> Client status changed: {} --> {}", getId(), clientNodeStatus, newNodeStatus);
+            if (StringUtils.isNotBlank(newNodeStatus) && ! StringUtils.equals(clientNodeStatus, newNodeStatus))
+                this.clientNodeStatus = newNodeStatus;
+        } else if (line.startsWith("-NOTIFY-X:")) {
+            String message = line.substring("-NOTIFY-X:".length()).trim();
+            String[] part = message.split(" ", 2);
+            String command = part[0].trim();
+            String args = part.length>1 ? part[1] : null;
+            log.info("{}--> Client notification: CMD={}, ARGS={}", getId(), command, args);
+
+            if ("DEBUG".equalsIgnoreCase(command)) {
+                log.debug("{}--> {}", getId(), args);
+            } else
+            if ("INFO".equalsIgnoreCase(command)) {
+                log.info("{}--> {}", getId(), args);
+            } else
+            if ("WARN".equalsIgnoreCase(command)) {
+                log.warn("{}--> {}", getId(), args);
+            } else
+            if ("ERROR".equalsIgnoreCase(command)) {
+                log.error("{}--> {}", getId(), args);
+            } else
+            if ("RECOVERY".equalsIgnoreCase(command)) {
+                args = args==null ? "" : args;
+                part = args.split(" ", 2);
+                String notificationType = part[0].trim();
+                String clientData = part.length>1 ? part[1] : null;
+                if (StringUtils.isNotBlank(notificationType) && StringUtils.isNotBlank(clientData)) {
+                    log.info("{}--> Client Recovery Notification: {}: {}", getId(), notificationType, clientData);
+                    if ("GIVE_UP".equalsIgnoreCase(notificationType)) {
+                        String[] tmp = clientData.split("@", 2);
+                        String nodeId = tmp[0].trim();
+                        String nodeAddress = tmp.length>1 ? tmp[1].trim() : null;
+                        if (StringUtils.isNotBlank(nodeAddress))
+                            eventBus.send(RecoveryConstant.SELF_HEALING_RECOVERY_GIVE_UP, nodeAddress, "Client_" + getId());
+                        else
+                            log.warn("{}--> Missing Node Address in Client Recovery Notification: {}", getId(), args);
+                    } else
+                        log.warn("{}--> UNKNOWN Client Recovery Notification: {}", getId(), args);
+                } else {
+                    log.warn("{}--> INVALID Client Recovery Notification: {}", getId(), args);
+                }
+            } else
+            {
+                log.warn("{}--> UNKNOWN Client Notification type: {}", getId(), message);
+            }
+
+        } else if (line.startsWith("-CLIENT-PROPERTY-CHANGE:")) {
+            String[] part = line.substring("-CLIENT-PROPERTY-CHANGE:".length()).trim().split(" ", 2);
+            String propertyName = part[0];
+            String propertyValue = part.length > 1 ? part[1] : null;
+            String oldValue = clientProperties.getProperty(propertyName);
+            if (StringUtils.isNotBlank(propertyName)) {
+                log.info("{}--> Client property changed: {} = {} --> {}", getId(), propertyName, oldValue, propertyValue);
+                clientProperties.put(propertyName.trim(), propertyValue);
+            } else {
+                log.warn("{}--> Invalid Client property: input line: ", line);
+            }
+        } else if (line.startsWith("-STATS:")) {
+            String statsStr = line.substring("-STATS:".length());
+            Object statsObj = SerializationUtil.deserializeFromString(statsStr);
+            if (statsObj instanceof Map) {
+                Map<String, Object> statsMap = StrUtil.castToMapStringObject(statsObj);
+                statsMap.put("_received_at_server_timestamp", System.currentTimeMillis());
+                log.debug("{}--> Client STATS received: {}", getId(), statsMap);
+                this.clientStatistics = statsMap;
+            } else if (statsObj==null) {
+                log.debug("{}--> Client STATS object is NULL", getId());
+            } else {
+                log.error("{}--> Unsupported Client STATS object: class={}, object={}", getId(), statsObj.getClass().getName(), statsObj);
+            }
+        } else if (line.equalsIgnoreCase("READY")) {
+            coordinator.clientReady(this);
+        } else {
+            coordinator.processClientInput(this, line);
+        }
+    }
+
+    protected void getClientInfoFromGreeting(String greetingInfo) {
+        if (StringUtils.isBlank(greetingInfo)) return;
+        String[] clientInfo = greetingInfo.trim().split(" ");
+
+        for (String s : clientInfo) {
+            if (StringUtils.isBlank(s)) continue;
+            if (s.startsWith("id=")) {
+                this.clientId = s.substring("id=".length()).replace("~~", " ");
+                log.info("{}--> Client Id: {}", id, clientId);
+            } else
+            if (s.startsWith("broker=")) {
+                this.clientBrokerUrl = s.substring("broker=".length());
+                log.info("{}--> Broker URL: {}", id, clientBrokerUrl);
+            } else
+            if (s.startsWith("address=")) {
+                if (clientAddressOverrideAllowed) {
+                    String addr = s.substring("address=".length());
+                    if (StringUtils.isNotBlank(addr)) {
+                        this.clientIpAddress = addr.trim();
+                        log.info("{}--> Effective IP: {}", id, clientIpAddress);
+                    }
+                }
+            } else
+            if (s.startsWith("port=")) {
+                if (clientAddressOverrideAllowed) {
+                    try {
+                        int port = Integer.parseInt(s.substring("port=".length()));
+                        if (port>0 && port<65536) {
+                            this.clientPort = port;
+                            log.info("{}--> Effective Port: {}", id, clientPort);
+                        }
+                    } catch (Exception ex) {
+                        log.warn("{}--> Invalid Port value: {}: {}", id, s.substring("port=".length()), ex.getMessage());
+                    }
+                }
+            } else
+            if (s.startsWith("username=")) {
+                this.clientBrokerUsername = s.substring("username=".length());
+                log.info("{}--> Broker Username: {}", id, clientBrokerUsername);
+            } else
+            if (s.startsWith("password=")) {
+                this.clientBrokerPassword = s.substring("password=".length());
+                log.info("{}--> Broker Password: {}", id, PasswordUtil.getInstance().encodePassword(clientBrokerPassword));
+            } else
+            if (s.startsWith("cert=")) {
+                this.clientCertificate = s.substring("cert=".length())
+                        .replace("~~", " ")
+                        .replace("##", "\r\n")
+                        .replace("$$", "\n");
+                log.info("{}--> Broker Cert.: {}", id, clientCertificate);
+
+                // Get certificate alias from client Id or IP address
+                String alias = /*StringUtils.isNotBlank(clientId)
+                        ? clientId.trim()
+                        :*/ getClientIpAddress();
+                log.info("{}--> Adding/Replacing client certificate in Truststore: alias={}", id, alias);
+
+                if (StringUtils.isNotEmpty(clientCertificate)) {
+                    // Add certificate to truststore
+                    try {
+                        X509Certificate cert = (X509Certificate) coordinator
+                                .getServer()
+                                .getBrokerCepService()
+                                .addOrReplaceCertificateInTruststore(alias, clientCertificate);
+                        log.info("{}--> Added/Replaced client certificate in Truststore: alias={}, CN={}, certificate-names={}",
+                                id, alias, cert.getSubjectX500Principal().getName(), CertUtil.subjectNames(cert));
+                    } catch (Exception e) {
+                        log.warn("{}--> EXCEPTION while adding/replacing certificate in Trust store: alias={}, exception: ",
+                                clientId, alias, e);
+                    }
+                } else {
+                    log.info("{}--> Client PEM certificate is empty. Leaving truststore unchanged", id);
+                }
+            } else {
+                log.warn("{}--> Unknown HELLO argument will be ignored: {}", id, s);
+            }
+        }
+
+        if (StringUtils.isBlank(this.clientId) || "null".equalsIgnoreCase(this.clientId))
+            this.clientId = getClientId();
+        if (StringUtils.isBlank(this.clientIpAddress) || "null".equalsIgnoreCase(this.clientIpAddress))
+            this.clientIpAddress = getClientIpAddress();
+        if (this.clientPort<=0 || this.clientPort>65535)
+            this.clientPort = getClientPort();
+    }
+
+    public String getClientId() {
+        if (StringUtils.isNotBlank(clientId)) return clientId;
+        clientId = getId();
+        return clientId;
+    }
+
+    public String getClientIpAddress() {
+        if (StringUtils.isNotBlank(clientIpAddress)) return clientIpAddress;
+        clientIpAddress = ((InetSocketAddress) getSession().getIoSession().getRemoteAddress()).getAddress().getHostAddress();
+        return clientIpAddress;
+    }
+
+    public String getClientHostname() {
+        if (StringUtils.isNotBlank(clientHostname)) return clientHostname;
+        clientHostname = ((InetSocketAddress) getSession().getIoSession().getRemoteAddress()).getAddress().getHostName();
+        return clientHostname;
+    }
+
+    public String getClientCanonicalHostname() {
+        if (StringUtils.isNotBlank(clientCanonicalHostname)) return clientCanonicalHostname;
+        clientCanonicalHostname = ((InetSocketAddress) getSession().getIoSession().getRemoteAddress()).getAddress().getCanonicalHostName();
+        return clientCanonicalHostname;
+    }
+
+    public int getClientPort() {
+        if (clientPort > 0) return clientPort;
+        clientPort = ((InetSocketAddress) getSession().getIoSession().getRemoteAddress()).getPort();
+        return clientPort;
+    }
+
+    public String getClientProperty(@NonNull String propertyName) { return clientProperties.getProperty(propertyName); }
+    public String getClientProperty(@NonNull String propertyName, String defaultValue) { return clientProperties.getProperty(propertyName, defaultValue); }
+
+    public NodeRegistryEntry getNodeRegistryEntry() {
+        if (nodeRegistryEntry!=null)
+            return nodeRegistryEntry;
+
+        //XXX:BUG: Following code seems not working...
+        String clientId = getClientId();
+        if (StringUtils.isNotBlank(clientId)) {
+            return nodeRegistry.getNodeByClientId(clientId);
+        }
+        return null;
+    }
+
+    public void sendToClient(String msg) {
+        sendToClient(msg, Level.INFO);
+    }
+
+    public void sendToClient(String msg, Level logLevel) {
+        if (msg == null || (msg = msg.trim()).isEmpty()) return;
+        switch (logLevel) {
+            case TRACE -> log.trace("{}==> PUSH : {}", id, msg);
+            case DEBUG -> log.debug("{}==> PUSH : {}", id, msg);
+            case WARN -> log.warn("{}==> PUSH : {}", id, msg);
+            case ERROR -> log.error("{}==> PUSH : {}", id, msg);
+            default -> log.info("{}==> PUSH : {}", id, msg);
+        }
+        out.println(msg);
+    }
+
+    public void sendCommand(String cmd) {
+        sendToClient(cmd);
+    }
+
+    public void sendCommand(String cmd, Level logLevel) {
+        sendToClient(cmd, logLevel);
+    }
+
+    public void sendCommand(String[] cmd) {
+        sendToClient(String.join(" ", cmd));
+    }
+
+    public void sendCommand(String[] cmd, Level logLevel) {
+        sendToClient(String.join(" ", cmd), logLevel);
+    }
+
+    public Object readFromClient(String cmd, Level logLevel) {
+        String uuid = UUID.randomUUID().toString();
+        log.trace("ClientShellCommand.readFromClient: uuid={}, cmd={}", uuid, cmd);
+        Object oldValue = inputsMap.remove(uuid);
+        log.trace("ClientShellCommand.readFromClient: uuid={}, old-inputMap-value={}", uuid, oldValue);
+        log.trace("ClientShellCommand.readFromClient: uuid={}, inputMap-BEFORE={}", uuid, inputsMap);
+        sendCommand(cmd+" "+uuid, logLevel);
+        log.trace("ClientShellCommand.readFromClient: uuid={}, Command sent to client", uuid);
+        while (!inputsMap.containsKey(uuid)) {
+            log.trace("ClientShellCommand.readFromClient: uuid={}, No input, waiting 500ms", uuid);
+            try { Thread.sleep(INPUT_CHECK_DELAY); } catch (InterruptedException e) { }
+        }
+        log.trace("ClientShellCommand.readFromClient: uuid={}, inputMap-BEFORE={}", uuid, inputsMap);
+        Object input = inputsMap.remove(uuid);
+        log.trace("ClientShellCommand.readFromClient: uuid={}, Input found: {}", uuid, input);
+        return input;
+    }
+
+    protected String _propertiesToBase64(Properties params) {
+        if (params != null && params.size() > 0) {
+            StringWriter writer = new StringWriter();
+            try {
+                params.store(writer, null);
+            } catch (IOException e) {
+                log.error("Could not serialize parameters: ", e);
+            }
+            String paramsStr = writer.getBuffer().toString();
+            return Base64.getEncoder().encodeToString(paramsStr.getBytes(StandardCharsets.UTF_8));
+        }
+        return null;
+    }
+
+    public void sendParams(Properties params) {
+        log.debug("sendParams: id={}, parameters={}", id, params);
+        String paramsStr = _propertiesToBase64(params);
+        if (paramsStr != null) {
+            sendToClient("SET-PARAMS " + paramsStr);
+        }
+    }
+
+    /**
+     * Write an object to a Base64 string.
+     */
+    public static String serializeToString(Serializable o) throws IOException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        ObjectOutputStream oos = new ObjectOutputStream(baos);
+        oos.writeObject(o);
+        oos.close();
+        return Base64.getEncoder().encodeToString(baos.toByteArray());
+    }
+
+    /**
+     * Read the object from Base64 string.
+     */
+    public static Object unserializeFromString(String s) throws IOException, ClassNotFoundException {
+        byte[] data = Base64.getDecoder().decode(s);
+        ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(data));
+        Object o = ois.readObject();
+        ois.close();
+        return o;
+    }
+
+    public static void sendClientConfigurationToClients(@NonNull ClientConfiguration cc, @NonNull List<ClientShellCommand> clients) {
+        List<String> clientIds = clients.stream().map(ClientShellCommand::getClientId).collect(Collectors.toList());
+        log.debug("sendClientConfigurationToClients: clients={}, client-config={}", clientIds, cc);
+        try {
+            String ccStr = serializeToString(cc);
+            log.debug("sendClientConfigurationToClients: Serialization of Client configuration: {}", ccStr);
+            ccStr = "SET-CLIENT-CONFIG " + ccStr;
+            for (ClientShellCommand csc : clients) {
+                log.info("sendClientConfigurationToClients: Sending Client configuration to client: {}", csc.getClientId());
+                csc.sendToClient(ccStr);
+            }
+            log.info("sendClientConfigurationToClients: Client configuration sent to clients: {}", clientIds);
+        } catch (IOException ex) {
+            log.error("sendClientConfigurationToClients: Exception while serializing Client configuration: ", ex);
+            log.error("sendClientConfigurationToClients: SET-CLIENT-CONFIG command *NOT* sent to clients");
+        }
+    }
+
+    public void sendClientConfiguration(ClientConfiguration cc) {
+        log.debug("sendClientConfiguration: id={}, client-config={}", id, cc);
+        try {
+            String ccStr = serializeToString(cc);
+            log.info("sendClientConfiguration: Serialization of Client configuration: {}", ccStr);
+            sendToClient("SET-CLIENT-CONFIG " + ccStr);
+        } catch (IOException ex) {
+            log.error("sendClientConfiguration: Exception while serializing Client configuration: ", ex);
+            log.error("sendClientConfiguration: SET-CLIENT-CONFIG command *NOT* sent to client");
+        }
+    }
+
+    public void sendGroupingConfiguration(String grouping, Map<String, GroupingConfiguration.BrokerConnectionConfig> connectionConfigs, BaguetteServer server) {
+        GroupingConfiguration gc = GroupingConfigurationHelper.newGroupingConfiguration(grouping, connectionConfigs, server);
+        sendGroupingConfiguration(gc);
+    }
+
+    public void sendGroupingConfiguration(GroupingConfiguration gc) {
+        String grouping = gc.getName();
+        log.debug("sendGroupingConfiguration: id={}, grouping={}, grouping-config={}", id, grouping, gc);
+        try {
+            String allStr = serializeToString(gc);
+            log.info("sendGroupingConfiguration: Serialization of Grouping configuration for {}: {}", grouping, allStr);
+            sendToClient("SET-GROUPING-CONFIG " + allStr);
+        } catch (IOException ex) {
+            log.error("sendGroupingConfiguration: Exception while serializing Grouping configuration: ", ex);
+            log.error("sendGroupingConfiguration: SET-GROUPING-CONFIG command *NOT* sent to client");
+        }
+    }
+
+    public void sendConstants(Map<String, Double> constants) {
+        log.debug("sendConstants: constants={}", constants);
+        HashMap<String, Object> all = new HashMap<>();
+        all.put("constants", constants);
+
+        try {
+            String allStr = serializeToString(all);
+            log.info("sendConstants: Serialization of Constants: {}", allStr);
+            sendToClient("SET-CONSTANTS " + allStr);
+        } catch (IOException ex) {
+            log.error("sendConstants: Exception while serializing Constants: ", ex);
+            log.error("sendConstants: SET-CONSTANTS command *NOT* sent to client");
+        }
+    }
+
+    public void setClientId(String id) {
+        if (id != null && !id.trim().isEmpty())
+            sendToClient("SET-ID " + id.trim());
+    }
+
+    public void setRole(String role) {
+        if (role != null && !role.trim().isEmpty()) sendToClient("SET-ROLE " + role.trim().toUpperCase());
+    }
+
+    public void setActiveGrouping(String grouping) {
+        if (grouping != null && !grouping.trim().isEmpty())
+            sendToClient("SET-ACTIVE-GROUPING " + grouping.trim().toUpperCase());
+    }
+
+    public void stop(String msg) {
+        log.info("{}==> STOP : {}", id, msg);
+        out.println("EXIT " + msg);
+        if (!callbackCalled.getAndSet(true)) {
+            callback.onExit(1);
+        }
+    }
+
+    public String toString() {
+        return "ClientShellCommand_" + id;
+    }
+
+    public String toStringCluster() {
+        return getClientClusterNodeAddress()+":"+getClientClusterNodePort();
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/GroupingConfigurationHelper.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server;
+
+import gr.iccs.imu.ems.util.GroupingConfiguration;
+
+import java.util.Map;
+
+import static gr.iccs.imu.ems.util.GroupingConfiguration.BrokerConnectionConfig;
+
+/**
+ * Baguette Client Configuration creation helper
+ */
+public class GroupingConfigurationHelper {
+    public static GroupingConfiguration newGroupingConfiguration(String groupingName, Map<String,BrokerConnectionConfig> connectionConfigs, BaguetteServer server) {
+        return GroupingConfiguration.builder()
+                .name( groupingName )
+                .properties(null)
+                .brokerConnections(connectionConfigs)
+                .eventTypeNames( server.getTopicsForGrouping(groupingName) )
+                .rules( server.getRulesForGrouping(groupingName) )
+                .connections( server.getTopicConnectionsForGrouping(groupingName) )
+                .constants( server.getConstants() )
+                .functionDefinitions( server.getFunctionDefinitions() )
+                .brokerUsername( server.getBrokerUsername() )
+                .brokerPassword( server.getBrokerPassword() )
+                .build();
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistry.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Node Registry
+ */
+@Slf4j
+@Service
+public class NodeRegistry {
+    private final Map<String,NodeRegistryEntry> registry = new LinkedHashMap<>();
+    @Getter @Setter
+    private ServerCoordinator coordinator;
+
+    public synchronized NodeRegistryEntry addNode(Map<String,Object> nodeInfo, String clientId) throws UnknownHostException {
+        String hostnameOrAddress = getIpAddressFromNodeInfo(nodeInfo);
+        String ipAddress = hostnameOrAddress;
+
+        // Get IP address from provided hostname or address
+        Throwable errorObj = null;
+        try {
+            log.debug("NodeRegistry.addNode(): Resolving IP address from provided hostname/address: {}", hostnameOrAddress);
+            InetAddress host = InetAddress.getByName(hostnameOrAddress);
+            log.trace("NodeRegistry.addNode(): InetAddress for provided hostname/address: {},  InetAddress: {}", hostnameOrAddress, host);
+            String resolvedIpAddress = host.getHostAddress();
+            log.info("NodeRegistry.addNode(): Provided-Address={},  Resolved-IP-Address={}", hostnameOrAddress, resolvedIpAddress);
+            ipAddress = resolvedIpAddress;
+        } catch (UnknownHostException e) {
+            log.error("NodeRegistry.addNode(): EXCEPTION while resolving IP address from provided hostname/address: {}\n", ipAddress, e);
+            errorObj = e;
+            //throw e;
+        }
+        nodeInfo.put("original-address", hostnameOrAddress);
+        nodeInfo.put("address", ipAddress);
+
+        // Check if an entry with the same IP address is already registered
+        NodeRegistryEntry entry = registry.get(ipAddress);
+        if (entry!=null) {
+            log.debug("NodeRegistry.addNode(): Node already pre-registered: ip-address={}\nOld Node Info: {}\nNew Node Info: {}",
+                    ipAddress, entry, nodeInfo);
+            if (coordinator!=null && coordinator.allowAlreadyPreregisteredNode(nodeInfo)) {
+                log.info("NodeRegistry.addNode(): PREVIOUS NODE INFO WILL BE OVERWRITTEN: ip-address={}\nOld Node Info: {}\nNew Node Info: {}",
+                        ipAddress, entry, nodeInfo);
+            } else {
+                log.error("NodeRegistry.addNode(): Node already pre-registered and coordinator does not allow new pre-registration requests to overwrite the existing one: ip-address={}\nOld Node Info: {}\nNew Node Info: {}",
+                        ipAddress, entry, nodeInfo);
+                throw new IllegalStateException("NODE ALREADY PRE-REGISTERED: "+ipAddress);
+            }
+        }
+
+        // Create and register node registry entry
+        entry = new NodeRegistryEntry(ipAddress, clientId, coordinator.getServer()).nodePreregistration(nodeInfo);
+        if (errorObj!=null) entry.getErrors().add(errorObj);
+        nodeInfo.put("baguette-client-id", clientId);
+        registry.put(ipAddress, entry);
+        log.debug("NodeRegistry.addNode(): Added info for node at address: {}\nNode info: {}", ipAddress, nodeInfo);
+        return entry;
+    }
+
+    public synchronized void removeNode(NodeRegistryEntry nodeEntry) {
+        String ipAddress = nodeEntry.getIpAddress();
+        removeNode(ipAddress);
+    }
+
+    public synchronized void removeNode(Map<String,Object> nodeInfo) {
+        String ipAddress = getIpAddressFromNodeInfo(nodeInfo);
+        removeNode(ipAddress);
+    }
+
+    public synchronized void removeNode(String ipAddress) {
+        registry.remove(ipAddress);
+        log.debug("NodeRegistry.removeNode(): Removed info for node at address: {}", ipAddress);
+    }
+
+    private String getIpAddressFromNodeInfo(Map<String,Object> nodeInfo) {
+        Object value = nodeInfo.get("ip-address");
+        if (value==null || StringUtils.isBlank(value.toString())) value = nodeInfo.get("address");
+        if (value==null || StringUtils.isBlank(value.toString())) value = nodeInfo.get("ip");
+        if (value==null || StringUtils.isBlank(value.toString())) return null;
+        return value.toString();
+    }
+
+    public synchronized void clearNodes() {
+        registry.clear();
+        log.debug("NodeRegistry.clearNodes(): Cleared node info registry");
+    }
+
+    public NodeRegistryEntry getNodeByAddress(String ipAddress) {
+        NodeRegistryEntry entry = registry.get(ipAddress);
+        log.debug("NodeRegistry.getNodeByAddress(): Returning info for node at address: {}\nNode Info: {}", ipAddress, entry);
+        return entry;
+    }
+
+    public NodeRegistryEntry getNodeByReference(String ref) {
+        return registry.values().stream()
+                .filter(n->n.getReference().equals(ref))
+                .findAny().orElse(null);
+    }
+
+    public NodeRegistryEntry getNodeByClientId(String clientId) {
+        return registry.values().stream()
+                .filter(n->n.getClientId().equals(clientId))
+                .findAny().orElse(null);
+    }
+
+    public Collection<String> getNodeAddresses() {
+        return registry.keySet();
+    }
+
+    public Collection<NodeRegistryEntry> getNodes() {
+        return registry.values();
+    }
+
+    public Collection<String> getNodeReferences() {
+        return registry.values().stream().map(NodeRegistryEntry::getReference).collect(Collectors.toList());
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/NodeRegistryEntry.java

@@ -0,0 +1,169 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import gr.iccs.imu.ems.baguette.server.coordinator.cluster.IClusterZone;
+import gr.iccs.imu.ems.util.StrUtil;
+import lombok.*;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.*;
+
+@Data
+@AllArgsConstructor
+@RequiredArgsConstructor
+public class NodeRegistryEntry {
+    public enum STATE { PREREGISTERED, IGNORE_NODE, INSTALLING, NOT_INSTALLED, INSTALLED, INSTALL_ERROR,
+        WAITING_REGISTRATION, REGISTERING, REGISTERED, REGISTRATION_ERROR, DISCONNECTED, EXITING, EXITED, NODE_FAILED
+    };
+    @Getter private final String ipAddress;
+    @Getter private final String clientId;
+    @JsonIgnore
+    private final transient BaguetteServer baguetteServer;
+    @Getter private String hostname;
+    @Getter private STATE state = null;
+    @Getter private Date stateLastUpdate;
+    @Getter private String reference = UUID.randomUUID().toString();
+    @Getter private List<Object> errors = new LinkedList<>();
+    @JsonIgnore
+    @Getter private transient Map<String, String> preregistration = new LinkedHashMap<>();
+    @JsonIgnore
+    @Getter private transient Map<String, String> installation = new LinkedHashMap<>();
+    @JsonIgnore
+    @Getter private transient Map<String, String> registration = new LinkedHashMap<>();
+    @JsonIgnore
+    @Getter @Setter private transient IClusterZone clusterZone;
+
+    @JsonIgnore
+    public BaguetteServer getBaguetteServer() {
+        return baguetteServer;
+    }
+
+    public String getNodeId() {
+        return getPreregistration().get("id");
+    }
+
+    public String getNodeAddress() {
+        return ipAddress!=null ? ipAddress : getPreregistration().get("address");
+    }
+
+    public String getNodeIdOrAddress() {
+        return StringUtils.isNotBlank(getNodeId()) ? getNodeId() : getNodeAddress();
+    }
+
+    public String getNodeIdAndAddress() {
+        return getNodeId()+" @ "+getNodeAddress();
+    }
+
+    private void setState(@NonNull STATE s) {
+        state = s;
+        stateLastUpdate = new Date();
+    }
+
+    public void refreshReference() { reference = UUID.randomUUID().toString(); }
+
+    public NodeRegistryEntry nodePreregistration(Map<String,Object> nodeInfo) {
+        preregistration.clear();
+        preregistration.putAll(StrUtil.deepFlattenMap(nodeInfo));
+        setState(STATE.PREREGISTERED);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeIgnore(Object nodeInfo) {
+        installation.clear();
+        installation.put("ignore-node", nodeInfo!=null ? nodeInfo.toString() : null);
+        setState(STATE.IGNORE_NODE);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeInstalling(Object nodeInfo) {
+        installation.clear();
+        installation.put("installation-task", nodeInfo!=null ? nodeInfo.toString() : "INSTALLING");
+        setState(STATE.INSTALLING);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeNotInstalled(Object nodeInfo) {
+        installation.clear();
+        installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "NOT_INSTALLED");
+        setState(STATE.NOT_INSTALLED);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeInstallationComplete(Object nodeInfo) {
+        installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "SUCCESS");
+        setState(STATE.INSTALLED);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeInstallationError(Object nodeInfo) {
+        installation.put("installation-task-result", nodeInfo!=null ? nodeInfo.toString() : "ERROR");
+        setState(STATE.INSTALL_ERROR);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeRegistering(Map<String,Object> nodeInfo) {
+        registration.clear();
+        registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
+        setState(STATE.REGISTERING);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeRegistered(Map<String,Object> nodeInfo) {
+        //registration.clear();
+        registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
+        setState(STATE.REGISTERED);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeRegistrationError(Map<String,Object> nodeInfo) {
+        registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
+        setState(STATE.REGISTRATION_ERROR);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeRegistrationError(Throwable t) {
+        registration.putAll(StrUtil.deepFlattenMap(Collections.singletonMap("exception", t)));
+        setState(STATE.REGISTRATION_ERROR);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeDisconnected(Map<String,Object> nodeInfo) {
+        registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
+        setState(STATE.DISCONNECTED);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeDisconnected(Throwable t) {
+        registration.putAll(StrUtil.deepFlattenMap(Collections.singletonMap("exception", t)));
+        setState(STATE.DISCONNECTED);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeExiting(Map<String,Object> nodeInfo) {
+        registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
+        setState(STATE.EXITING);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeExited(Map<String,Object> nodeInfo) {
+        registration.putAll(StrUtil.deepFlattenMap(nodeInfo));
+        setState(STATE.EXITED);
+        return this;
+    }
+
+    public NodeRegistryEntry nodeFailed(Map<String,Object> failInfo) {
+        if (failInfo!=null)
+            registration.putAll(StrUtil.deepFlattenMap(failInfo));
+        setState(STATE.NODE_FAILED);
+        return this;
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/ServerCoordinator.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server;
+
+import gr.iccs.imu.ems.translate.TranslationContext;
+import gr.iccs.imu.ems.util.GroupingConfiguration;
+
+import java.util.Map;
+
+import static gr.iccs.imu.ems.util.GroupingConfiguration.BrokerConnectionConfig;
+
+public interface ServerCoordinator {
+    default boolean isSupported(TranslationContext tc) { return true; }
+
+    default boolean supportsAggregators() { return false; }
+
+    void initialize(TranslationContext tc, String upperwareGrouping, BaguetteServer server, Runnable callback);
+
+    default void setProperties(Map<String, String> p) { }
+
+    default boolean processClientInput(ClientShellCommand csc, String line) { return false; }
+
+    BaguetteServer getServer();
+
+    int getPhase();
+
+    default boolean allowAlreadyPreregisteredNode(Map<String,Object> nodeInfo) { return true; }
+
+    default boolean allowAlreadyRegisteredNode(ClientShellCommand csc) { return true; }
+
+    default boolean allowNotPreregisteredNode(ClientShellCommand csc) { return true; }
+
+    default void preregister(NodeRegistryEntry entry) { }
+
+    void register(ClientShellCommand c);
+
+    void unregister(ClientShellCommand c);
+
+    void clientReady(ClientShellCommand c);
+
+    void start();
+
+    void stop();
+
+    default void sendGroupingConfigurations(Map<String,BrokerConnectionConfig> connectionConfigs, ClientShellCommand c, BaguetteServer server) {
+        for (String grouping : server.getGroupingNames()) {
+            GroupingConfiguration gc = GroupingConfigurationHelper.newGroupingConfiguration(grouping, connectionConfigs, server);
+            c.sendGroupingConfiguration(gc);
+        }
+    }
+
+    default BrokerConnectionConfig getGroupingBrokerConfig(String grouping, ClientShellCommand c) {
+        String brokerUrl = c.getClientBrokerUrl();
+        String brokerCert = c.getClientCertificate();
+        String username = c.getClientBrokerUsername();
+        String password = c.getClientBrokerPassword();
+        return new BrokerConnectionConfig(grouping, brokerUrl, brokerCert, username, password);
+    }
+    default BrokerConnectionConfig getUpperwareBrokerConfig(BaguetteServer server) {
+        String brokerUrl = server.getUpperwareBrokerUrl();
+        String brokerCert = server.getBrokerCepService().getBrokerCertificate();
+        String username = server.getBrokerUsername();
+        String password = server.getBrokerPassword();
+        return new BrokerConnectionConfig(server.getUpperwareGrouping(), brokerUrl, brokerCert, username, password);
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/Sshd.java

@@ -0,0 +1,320 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server;
+
+import gr.iccs.imu.ems.baguette.server.coordinator.cluster.ClusteringCoordinator;
+import gr.iccs.imu.ems.baguette.server.properties.BaguetteServerProperties;
+import gr.iccs.imu.ems.util.EventBus;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.text.StringEscapeUtils;
+import org.apache.sshd.common.PropertyResolverUtils;
+import org.apache.sshd.common.config.keys.KeyUtils;
+import org.apache.sshd.common.keyprovider.KeyPairProvider;
+import org.apache.sshd.common.session.SessionHeartbeatController;
+import org.apache.sshd.core.CoreModuleProperties;
+import org.apache.sshd.mina.MinaServiceFactoryFactory;
+import org.apache.sshd.server.SshServer;
+import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
+import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
+import org.slf4j.event.Level;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.security.KeyPair;
+import java.security.PublicKey;
+import java.time.Duration;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Custom SSH server
+ */
+@Slf4j
+public class Sshd {
+    @Getter private ServerCoordinator coordinator;
+    private BaguetteServerProperties configuration;
+    private SshServer sshd;
+    private String serverPubkey;
+    private String serverPubkeyFingerprint;
+    private String serverPubkeyAlgorithm;
+    private String serverPubkeyFormat;
+    private KeyPairProvider serverKeyProvider;
+
+    private boolean heartbeatOn;
+    private long heartbeatPeriod;
+
+    private EventBus<String,Object,Object> eventBus;
+    @Getter @Setter
+    private NodeRegistry nodeRegistry;
+
+    public void start(BaguetteServerProperties configuration, ServerCoordinator coordinator, EventBus<String,Object,Object> eventBus, NodeRegistry registry) throws IOException {
+        log.info("** SSH server **");
+        this.coordinator = coordinator;
+        this.configuration = configuration;
+        this.eventBus = eventBus;
+        this.nodeRegistry = registry;
+
+        // Configure SSH server
+        int port = configuration.getServerPort();
+        String serverKeyFilePath = configuration.getServerKeyFile();
+        log.info("SSH server: Public IP address: {}", configuration.getServerAddress());
+        log.info("SSH server:  Starting on port: {}", port);
+        log.info("SSH server:   Server key file: {}", new File(serverKeyFilePath).getAbsolutePath());
+
+        // Create SSHD and set port
+        sshd = SshServer.setUpDefaultServer();
+        sshd.setPort(port);
+
+        // Setup server's key provider
+        _loadPubkeyAndFingerprint();
+        sshd.setKeyPairProvider(this.serverKeyProvider);
+
+        // Setup server's shell factory (for custom Shell commands)
+        sshd.setShellFactory(channelSession -> {
+            ClientShellCommand csc = new ClientShellCommand(coordinator, configuration.isClientAddressOverrideAllowed(), eventBus, nodeRegistry);
+            //csc.setId( "#-"+System.currentTimeMillis() );
+            log.debug("SSH server: Shell Factory: create invoked : New ClientShellCommand id: {}", csc.getId());
+            return csc;
+        });
+
+        // Setup password authenticator
+        sshd.setPasswordAuthenticator((username, password, session) -> {
+            //public boolean authenticate(String username, String password, ServerSession session)
+            String pwd = Optional.ofNullable(configuration.getCredentials().get(username.trim())).orElse("");
+            return pwd.equals(password);
+        });
+
+        // Set session timeout
+        sshd.setSessionHeartbeat(SessionHeartbeatController.HeartbeatType.IGNORE, Duration.ofMillis(configuration.getHeartbeatPeriod()));
+        //PropertyResolverUtils.updateProperty(sshd, CoreModuleProperties.HEARTBEAT_INTERVAL.getName(), configuration.getHeartbeatPeriod());
+        PropertyResolverUtils.updateProperty(sshd, CoreModuleProperties.IDLE_TIMEOUT.getName(), Long.MAX_VALUE);
+        PropertyResolverUtils.updateProperty(sshd, CoreModuleProperties.SOCKET_KEEPALIVE.getName(), true);
+        log.debug("SSH server: Set IDLE_TIMEOUT to MAX, and KEEP-ALIVE to true, and HEARTBEAT to {}", configuration.getHeartbeatPeriod());
+
+        // Explicitly set IO service factory factory to prevent conflict between MINA and Netty options
+        sshd.setIoServiceFactoryFactory(new MinaServiceFactoryFactory());
+
+        // Start SSH server and accept connections
+        sshd.start();
+        log.info("SSH server: Ready");
+
+        // Start application-level heartbeat service (additional to the SSH and Socket heartbeats)
+        if (configuration.isHeartbeatEnabled()) {
+            long heartbeatPeriod = configuration.getHeartbeatPeriod();
+            startHeartbeat(heartbeatPeriod);
+        }
+
+        // Start coordinator
+        coordinator.start();
+    }
+
+    public void stop() throws IOException {
+        // Stop coordinator
+        coordinator.stop();
+
+        // Don't accept new connections
+        log.info("SSH server: Stopping SSH server...");
+        sshd.setShellFactory(null);
+
+        // Signal heartbeat service to stop
+        stopHeartbeat();
+
+        // Close active client connections
+        for (ClientShellCommand csc : ClientShellCommand.getActive()) {
+            csc.stop("Server exits");
+        }
+
+        sshd.stop();
+        log.info("SSH server: Stopped");
+    }
+
+    public void startHeartbeat(long period) {
+        heartbeatOn = true;
+        Thread heartbeat = new Thread(
+                new Runnable() {
+                    private long period;
+
+                    public void run() {
+                        log.info("--> Heartbeat: Started: period={}ms", period);
+                        while (heartbeatOn && period > 0) {
+                            try {
+                                Thread.sleep(period);
+                            } catch (InterruptedException ex) {
+                            }
+                            String msg = String.format("Heartbeat %d", System.currentTimeMillis());
+                            log.debug("--> Heartbeat: {}", msg);
+                            for (ClientShellCommand csc : ClientShellCommand.getActive()) {
+                                csc.sendToClient(msg, Level.DEBUG);
+                            }
+                        }
+                        log.info("--> Heartbeat: Stopped");
+                    }
+
+                    public Runnable setPeriod(long period) {
+                        this.period = period;
+                        return this;
+                    }
+                }
+                        .setPeriod(period)
+        );
+        heartbeat.setDaemon(true);
+        heartbeat.start();
+    }
+
+    public void stopHeartbeat() {
+        heartbeatOn = false;
+    }
+
+    protected void broadcastToClients(String msg) {
+        for (ClientShellCommand csc : ClientShellCommand.getActive()) {
+            log.info("SSH server: Sending to {} : {}", csc.getId(), msg);
+            csc.sendToClient(msg);
+        }
+    }
+
+    public void sendToActiveClients(String command) {
+        for (ClientShellCommand csc : ClientShellCommand.getActive()) {
+            log.info("SSH server: Sending to client {} : {}", csc.getId(), command);
+            csc.sendToClient(command);
+        }
+    }
+
+    public void sendToClient(String clientId, String command) {
+        for (ClientShellCommand csc : ClientShellCommand.getActive()) {
+            if (csc.getId().equals(clientId)) {
+                log.info("SSH server: Sending to client {} : {}", csc.getId(), command);
+                csc.sendToClient(command);
+            }
+        }
+    }
+
+    public void sendToActiveClusters(String command) {
+        if (!(coordinator instanceof ClusteringCoordinator)) return;
+        ((ClusteringCoordinator)coordinator).getClusters().forEach(cluster -> {
+            log.info("SSH server: Sending to cluster {} : {}", cluster.getId(), command);
+            sendToCluster(cluster.getId(), command);
+        });
+    }
+
+    public void sendToCluster(String clusterId, String command) {
+        if (!(coordinator instanceof ClusteringCoordinator)) return;
+        ((ClusteringCoordinator)coordinator).getCluster(clusterId).getNodes().forEach(csc -> {
+            log.info("SSH server: Sending to client {} : {}", csc.getId(), command);
+            csc.sendToClient(command);
+        });
+    }
+
+    public Object readFromClient(String clientId, String command, Level logLevel) {
+        log.trace("SSH server: Sending and Reading to/from client {}: {}", clientId, command);
+        for (ClientShellCommand csc : ClientShellCommand.getActive()) {
+            log.trace("SSH server: Check CSC: csc-id={}, client={}", csc.getId(), clientId);
+            if (csc.getId().equals(clientId)) {
+                log.debug("SSH server: Sending and Reading to/from client {} : {}", csc.getId(), command);
+                return csc.readFromClient(command, logLevel);
+            }
+        }
+        return null;
+    }
+
+    public List<String> getActiveClients() {
+        return ClientShellCommand.getActive().stream()
+                .map(c -> String.format("%s %s %s:%d", c.getId(),
+                        c.getClientIpAddress(),
+                        c.getClientClusterNodeHostname(),
+                        c.getClientClusterNodePort()))
+                .sorted()
+                .collect(Collectors.toList());
+    }
+
+    public Map<String, Map<String, String>> getActiveClientsMap() {
+        return ClientShellCommand.getActive().stream()
+                //.sorted((final ClientShellCommand c1, final ClientShellCommand c2) -> c1.getId().compareTo(c2.getId()))
+                .collect(Collectors.toMap(ClientShellCommand::getId, c -> {
+                    Map<String,String> properties = new LinkedHashMap<>();
+                    //properties.put("id", c.getId());
+                    properties.put("ip-address", c.getClientIpAddress());
+                    properties.put("node-hostname", c.getClientClusterNodeHostname());
+                    properties.put("node-port", Integer.toString(c.getClientClusterNodePort()));
+                    return properties;
+                }));
+    }
+
+    public void sendConstants(Map<String, Double> constants) {
+        for (ClientShellCommand csc : ClientShellCommand.getActive()) {
+            log.info("SSH server: Sending constants to client {} : {}", csc.getId(), constants);
+            csc.sendConstants(constants);
+        }
+    }
+
+    public String getPublicKey() {
+        if (serverPubkey==null) _loadPubkeyAndFingerprint();
+        return serverPubkey;
+    }
+
+    public String getPublicKeyFingerprint() {
+        if (serverPubkeyFingerprint==null) _loadPubkeyAndFingerprint();
+        return serverPubkeyFingerprint;
+    }
+
+    public String getPublicKeyAlgorithm() {
+        if (serverPubkey==null) _loadPubkeyAndFingerprint();
+        return serverPubkeyAlgorithm;
+    }
+
+    public String getPublicKeyFormat() {
+        if (serverPubkey==null) _loadPubkeyAndFingerprint();
+        return serverPubkeyFormat;
+    }
+
+    @SneakyThrows
+    private synchronized void _loadPubkeyAndFingerprint() {
+        if (serverPubkey!=null) return;
+
+        String serverKeyFilePath = configuration.getServerKeyFile();
+        log.debug("_loadPubkeyAndFingerprint(): Server Key file: {}", serverKeyFilePath);
+        File serverKeyFile = new File(serverKeyFilePath);
+
+        // Create and configure a new SimpleGeneratorHostKeyProvider instance
+        SimpleGeneratorHostKeyProvider simpleGeneratorHostKeyProvider =
+                new SimpleGeneratorHostKeyProvider(serverKeyFile.toPath());
+        //simpleGeneratorHostKeyProvider.setStrictFilePermissions(true);    // 'true' by default
+
+        //  Create or load the Baguette server key pair
+        List<KeyPair> keys = simpleGeneratorHostKeyProvider.loadKeys(null);
+        if (keys.size()!=1)
+            throw new IllegalArgumentException("Server key file contains 0 or >1 keys: #keys="+keys.size()+", file="+serverKeyFilePath);
+        KeyPair serverKey = keys.get(0);
+        PublicKey publicKey = serverKey.getPublic();
+
+        // Write Baguette server public key as PEM string
+        StringWriter writer = new StringWriter();
+        JcaPEMWriter pemWriter = new JcaPEMWriter(writer);
+        pemWriter.writeObject(publicKey);
+        pemWriter.flush();
+
+        // Store public key PEM and fingerprint for future use
+        this.serverPubkey = StringEscapeUtils.escapeJson(writer.toString().trim());
+        this.serverPubkeyFormat = publicKey.getFormat();
+        this.serverPubkeyAlgorithm = publicKey.getAlgorithm();
+        this.serverPubkeyFingerprint = KeyUtils.getFingerPrint(publicKey);
+        this.serverKeyProvider = simpleGeneratorHostKeyProvider;
+        log.debug("_loadPubkeyAndFingerprint(): Server public key: \n{}", serverPubkey);
+        log.debug("_loadPubkeyAndFingerprint():       Fingerprint: {}", serverPubkeyFingerprint);
+        log.debug("_loadPubkeyAndFingerprint():         Algorithm: {}", serverPubkeyAlgorithm);
+        log.debug("_loadPubkeyAndFingerprint():            Format: {}", serverPubkeyFormat);
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/NoopCoordinator.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator;
+
+import gr.iccs.imu.ems.baguette.server.BaguetteServer;
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.baguette.server.ServerCoordinator;
+import gr.iccs.imu.ems.baguette.server.properties.BaguetteServerProperties;
+import gr.iccs.imu.ems.translate.TranslationContext;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class NoopCoordinator implements ServerCoordinator {
+    protected BaguetteServer server;
+    protected BaguetteServerProperties config;
+    protected Runnable callback;
+    protected boolean started;
+
+    @Override
+    public void initialize(final TranslationContext TC, String upperwareGrouping, BaguetteServer server, Runnable callback) {
+        if (_logInvocation("initialize", null, false)) return;
+        this.server = server;
+        this.config = server.getConfiguration();
+        this.callback = callback;
+    }
+
+    @Override
+    public BaguetteServer getServer() {
+        return server;
+    }
+
+    @Override
+    public void start() {
+        if (_logInvocation("start", null, false)) return;
+        started = true;
+
+        if (callback != null) {
+            log.info("{}: start(): Invoking callback", getClass().getSimpleName());
+            callback.run();
+        }
+    }
+
+    @Override
+    public void stop() {
+        if (!_logInvocation("stop", null, true)) return;
+        started = false;
+    }
+
+    public boolean isStarted() {
+        return started;
+    }
+
+    @Override
+    public int getPhase() {
+        return -1;
+    }
+
+    @Override
+    public synchronized void preregister(NodeRegistryEntry entry) {
+        _logInvocation("preregister", entry, true);
+    }
+
+    @Override
+    public synchronized void register(ClientShellCommand c) {
+        _logInvocation("register", c, true);
+    }
+
+    @Override
+    public synchronized void unregister(ClientShellCommand c) {
+        _logInvocation("unregister", c, true);
+    }
+
+    @Override
+    public synchronized void clientReady(ClientShellCommand c) {
+        _logInvocation("clientReady", c, true);
+    }
+
+    protected boolean _logInvocation(String methodName, Object o, boolean checkStarted) {
+        String className = getClass().getSimpleName();
+        String str =  (o==null) ? "" : (
+                o instanceof ClientShellCommand ? String.format(". CSC: %s", o) : (
+                        o instanceof NodeRegistryEntry ? String.format(". NRE: %s", o) :
+                                String.format(". Object: %s", o)
+                        )
+                );
+        if (checkStarted && !started) {
+            log.warn("{}: {}(): Coordinator has not been started{}", className, methodName, str);
+        } else
+        if (!checkStarted && started) {
+            log.warn("{}: {}(): Coordinator is already running{}", className, methodName, str);
+        } else {
+            log.info("{}: {}(): Method invoked{}", className, methodName, str);
+        }
+        return started;
+    }
+
+    public void sleep(long millis) {
+        try { Thread.sleep(millis); } catch (Exception ignored) { }
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/ServerCoordinatorTimeWin.java

@@ -0,0 +1,217 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator;
+
+import gr.iccs.imu.ems.baguette.server.BaguetteServer;
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.ServerCoordinator;
+import gr.iccs.imu.ems.translate.TranslationContext;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@Slf4j
+public class ServerCoordinatorTimeWin implements ServerCoordinator {
+    private final ServerCoordinatorTimeWin LOCK = this;
+    private BaguetteServer server;
+    private Runnable callback;
+    private boolean started;
+    private long registrationWindow;
+    private boolean registrationWindowEnded;
+    private Thread timeout;
+    private int numClients;
+    private int phase;
+    private List<ClientShellCommand> clients;
+    private ClientShellCommand broker;
+    private int readyClients;
+    private String brokerCfgIpAddressCmd;
+    private String brokerCfgPortCmd;
+
+    public void initialize(final TranslationContext TC, String upperwareGrouping, BaguetteServer server, Runnable callback) {
+        this.server = server;
+        this.registrationWindow = server.getConfiguration().getRegistrationWindow();
+        this.callback = callback;
+        this.clients = new ArrayList<>();
+    }
+
+    public BaguetteServer getServer() {
+        return server;
+    }
+
+    public void start() {
+        timeout = new Thread(
+                new Runnable() {
+                    private long delay;
+
+                    public void run() {
+                        log.info("ServerCoordinatorTimeWin: REGISTRATION PERIOD STARTS");
+                        started = true;
+                        registrationWindowEnded = false;
+                        try {
+                            Thread.sleep(delay);
+                        } catch (InterruptedException ex) {
+                            log.info("ServerCoordinatorTimeWin: INTERRUPTED: Registration stopped");
+                            return;
+                        }
+                        log.info("ServerCoordinatorTimeWin: REGISTRATION PERIOD ENDS");
+
+                        List<ClientShellCommand> registeredIntime;
+                        synchronized (LOCK) {
+                            registeredIntime = new ArrayList<>(clients);
+                        }
+                        if (registeredIntime.size() > 0) {
+                            startPhase1(registeredIntime);
+                        } else {
+                            registrationWindowEnded = true;
+                            log.warn("ServerCoordinatorTimeWin: No clients have been registered");
+                            log.warn("ServerCoordinatorTimeWin: The first client to register will become BROKER");
+                        }
+                    }
+
+                    public Runnable setDelay(long delay) {
+                        this.delay = delay;
+                        return this;
+                    }
+                }
+                        .setDelay(registrationWindow)
+        );
+        timeout.setDaemon(true);
+        timeout.start();
+        log.info("ServerCoordinatorTimeWin: START");
+    }
+
+    public void stop() {
+        started = false;
+        if (timeout.isAlive()) timeout.interrupt();
+    }
+
+    public boolean isStarted() {
+        return started;
+    }
+
+    public int getPhase() {
+        return phase;
+    }
+
+    public synchronized void register(ClientShellCommand c) {
+        if (!started) return;
+        //if (phase!=0) return;
+        clients.add(c);
+        numClients++;
+        if (phase == 0 && numClients == 1 && registrationWindowEnded) startPhase1(clients);
+        else if (phase != 0) {
+            c.sendToClient(brokerCfgIpAddressCmd);
+            c.sendToClient(brokerCfgPortCmd);
+            c.sendToClient("ROLE CLIENT");
+        }
+        log.info("ServerCoordinatorTimeWin: register: {} clients registered", numClients);
+    }
+
+    public synchronized void unregister(ClientShellCommand c) {
+        if (!started) return;
+        //if (phase!=0) return;
+        clients.remove(c);
+        numClients--;
+        log.info("ServerCoordinatorTimeWin: unregister: {} clients registered", numClients);
+    }
+
+    protected synchronized void startPhase1(List<ClientShellCommand> registeredIntime) {
+        if (phase != 0) return;
+        log.info("ServerCoordinatorTimeWin: Phase #1");
+        phase = 1;
+
+        // Pick a random client for Broker
+        int howmany = registeredIntime.size();
+        int sel = (int) Math.round((howmany - 1) * Math.random());
+        if (sel >= howmany) sel = howmany - 1;
+        broker = registeredIntime.get(sel);
+        log.info("ServerCoordinatorTimeWin: Client {} will become BROKER", broker.getId());
+
+        // Push broker IP address to all clients
+        try {
+            //java.net.InetSocketAddress brokerSocketAddress = (java.net.InetSocketAddress) broker.getSession().getIoSession().getRemoteAddress();
+            //String brokerIpAddress = brokerSocketAddress.getAddress().getHostAddress();
+            //int brokerPort = brokerSocketAddress.getPort();
+            String brokerIpAddress = broker.getClientIpAddress();
+            int brokerPort = broker.getClientPort();
+            if (brokerIpAddress == null || brokerIpAddress.trim().isEmpty() || brokerPort <= 0)
+                throw new Exception("ServerCoordinatorTimeWin: startPhase1(): Unable to get broker IP address or Port: " + broker);
+            this.brokerCfgIpAddressCmd = String.format("SET-PARAM bin/broker.cfg-template BROKER_IP_ADDR %s bin/broker.cfg", brokerIpAddress);
+            this.brokerCfgPortCmd = String.format("SET-PARAM bin/broker.cfg-template BROKER_PORT %d bin/broker.cfg", brokerPort);
+        } catch (Exception ex) {
+            this.brokerCfgIpAddressCmd = null;
+            this.brokerCfgPortCmd = null;
+            log.error("ServerCoordinatorTimeWin: startPhase1(): Error while getting broker IP address and port: {}", broker);
+        }
+
+        // Signal BROKER to prepare
+        phase = 2;
+        broker.sendToClient("ROLE BROKER");
+    }
+
+    public synchronized void clientReady(ClientShellCommand c) {
+        if (getPhase()==2) _brokerReady(c);
+        else _clientReady(c);
+    }
+
+    private void _brokerReady(ClientShellCommand c) {
+        if (!started) return;
+        if (phase != 2) return;
+        log.info("ServerCoordinatorTimeWin: Broker is ready");
+        phase = 3;
+        readyClients = 1;
+        if (readyClients == numClients) {
+            phase = 4;
+            signalTopologyReady();
+        } else {
+            Thread runner = new Thread(new Runnable() {
+                public void run() {
+                    // Signal all clients except broker to prepare
+                    for (ClientShellCommand c : clients) {
+                        if (c != broker) {
+                            c.sendToClient(brokerCfgIpAddressCmd);
+                            c.sendToClient(brokerCfgPortCmd);
+                            c.sendToClient("ROLE CLIENT");
+                        }
+                    }
+                }
+            });
+            runner.setDaemon(true);
+            runner.start();
+        }
+    }
+
+    private void _clientReady(ClientShellCommand c) {
+        if (!started) return;
+        if (phase != 3) return;
+        readyClients++;
+        log.info("ServerCoordinatorTimeWin: {} of {} clients are ready", readyClients, numClients);
+        if (readyClients == numClients) {
+            phase = 4;
+            signalTopologyReady();
+        }
+    }
+
+    protected void signalTopologyReady() {
+        if (phase != 4) return;
+        log.info("ServerCoordinatorTimeWin: Invoking callback");
+        phase = 5;
+        Thread runner = new Thread(new Runnable() {
+            public void run() {
+                // Invoke callback
+                callback.run();
+                log.info("ServerCoordinatorTimeWin: FINISHED");
+            }
+        });
+        runner.setDaemon(true);
+        runner.start();
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/ServerCoordinatorWaitAll.java

@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator;
+
+import gr.iccs.imu.ems.baguette.server.BaguetteServer;
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.ServerCoordinator;
+import gr.iccs.imu.ems.translate.TranslationContext;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Vector;
+
+@Slf4j
+public class ServerCoordinatorWaitAll implements ServerCoordinator {
+    private BaguetteServer server;
+    private Runnable callback;
+    private int expectedClients;
+    private int numClients;
+    private int phase;
+    private Vector<ClientShellCommand> clients;
+    private ClientShellCommand broker;
+    private int readyClients;
+
+    public void initialize(final TranslationContext TC, String upperwareGrouping, BaguetteServer server, Runnable callback) {
+        this.server = server;
+        this.expectedClients = server.getConfiguration().getNumberOfInstances();
+        this.callback = callback;
+        this.clients = new Vector<>();
+        log.info("initialize: Done");
+    }
+
+    public BaguetteServer getServer() {
+        return server;
+    }
+
+    public void start() {
+    }
+
+    public void stop() {
+    }
+
+    public int getPhase() {
+        return phase;
+    }
+
+    public synchronized void register(ClientShellCommand c) {
+        if (phase != 0) return;
+        clients.add(c);
+        numClients++;
+        log.info("ServerCoordinatorWaitAll: {} of {} clients registered", numClients, expectedClients);
+        if (numClients == expectedClients) {
+            startPhase1();
+        }
+    }
+
+    public synchronized void unregister(ClientShellCommand c) {
+        if (phase != 0) return;
+        clients.remove(c);
+        numClients--;
+    }
+
+    protected synchronized void startPhase1() {
+        if (phase != 0) return;
+        log.info("ServerCoordinatorWaitAll: Phase #1");
+        phase = 1;
+        Thread runner = new Thread(new Runnable() {
+            public void run() {
+                // Pick a random client for Broker
+                int sel = (int) Math.round((numClients - 1) * Math.random());
+                if (sel >= numClients) sel = numClients - 1;
+                broker = clients.get(sel);
+                log.info("ServerCoordinatorWaitAll: Client #{} will become BROKER", broker.getId());
+
+                // Signal BROKER to prepare
+                phase = 2;
+                broker.sendToClient("ROLE BROKER");
+            }
+        });
+        runner.setDaemon(true);
+        runner.start();
+    }
+
+    public synchronized void clientReady(ClientShellCommand c) {
+        if (getPhase()==2) _brokerReady(c);
+        else _clientReady(c);
+    }
+
+    private void _brokerReady(ClientShellCommand c) {
+        if (phase != 2) return;
+        log.info("ServerCoordinatorWaitAll: Broker is ready");
+        phase = 3;
+        readyClients = 1;
+        if (readyClients == expectedClients) {
+            phase = 4;
+            signalTopologyReady();
+        } else {
+            Thread runner = new Thread(new Runnable() {
+                public void run() {
+                    // Signal all clients except broker to prepare
+                    for (ClientShellCommand c : clients) {
+                        if (c != broker) {
+                            c.sendToClient("ROLE CLIENT");
+                        }
+                    }
+                }
+            });
+            runner.setDaemon(true);
+            runner.start();
+        }
+    }
+
+    private void _clientReady(ClientShellCommand c) {
+        if (phase != 3) return;
+        readyClients++;
+        log.info("ServerCoordinatorWaitAll: {} of {} clients are ready", readyClients, expectedClients);
+        if (readyClients == expectedClients) {
+            phase = 4;
+            signalTopologyReady();
+        }
+    }
+
+    protected void signalTopologyReady() {
+        if (phase != 4) return;
+        log.info("ServerCoordinatorWaitAll: Invoking callback");
+        phase = 5;
+        Thread runner = new Thread(new Runnable() {
+            public void run() {
+                // Invoke callback
+                callback.run();
+                log.info("ServerCoordinatorWaitAll: FINISHED");
+            }
+        });
+        runner.setDaemon(true);
+        runner.start();
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/TestCoordinator.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator;
+
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static gr.iccs.imu.ems.util.GroupingConfiguration.BrokerConnectionConfig;
+
+@Slf4j
+public class TestCoordinator extends NoopCoordinator {
+    @Override
+    public synchronized void register(ClientShellCommand c) {
+        if (!_logInvocation("register", c, true)) return;
+        _do_register(c);
+    }
+
+    protected synchronized void _do_register(ClientShellCommand c) {
+        // prepare configuration
+        Map<String, BrokerConnectionConfig> connCfgMap = new LinkedHashMap<>();
+        BrokerConnectionConfig groupingConn = getUpperwareBrokerConfig(server);
+        connCfgMap.put(server.getUpperwareGrouping(), groupingConn);
+        log.trace("ClusteringCoordinator: GLOBAL broker config.: {}", groupingConn);
+
+        connCfgMap.put("PER_CLOUD", groupingConn = getGroupingBrokerConfig("PER_CLOUD", c));
+        log.trace("TestCoordinator.test(): {} broker config.: {}", "PER_CLOUD", groupingConn);
+
+        // prepare Broker-CEP configuration
+        log.info("TestCoordinator.test(): --------------------------------------------------");
+        log.info("TestCoordinator.test(): Sending grouping configurations...");
+        sendGroupingConfigurations(connCfgMap, c, server);
+        log.info("TestCoordinator.test(): Sending grouping configurations... done");
+
+        // Set active grouping and send an event
+        String grouping = "PER_INSTANCE";
+        try {
+            Thread.sleep(500);
+        } catch (Exception ex) {
+        }
+        log.info("TestCoordinator.test(): --------------------------------------------------");
+        log.info("TestCoordinator.test(): Setting active grouping: {}", grouping);
+        c.setActiveGrouping(grouping);
+
+        try {
+            Thread.sleep(5000);
+        } catch (Exception ex) {
+        }
+        log.info("TestCoordinator.test(): --------------------------------------------------");
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/TwoLevelCoordinator.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator;
+
+import gr.iccs.imu.ems.baguette.server.BaguetteServer;
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.common.selfhealing.SelfHealingManager;
+import gr.iccs.imu.ems.translate.TranslationContext;
+import gr.iccs.imu.ems.util.GROUPING;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static gr.iccs.imu.ems.util.GroupingConfiguration.BrokerConnectionConfig;
+
+@Slf4j
+public class TwoLevelCoordinator extends NoopCoordinator {
+    private GROUPING globalGrouping;
+    private GROUPING nodeGrouping;
+
+    @Override
+    public boolean isSupported(final TranslationContext _TC) {
+        // Check if there are at least 2 levels in architecture
+        Set<String> groupings = _TC.getG2R().keySet();
+        if (!groupings.contains("GLOBAL")) return false;
+        return groupings.size()>1;
+    }
+
+    @Override
+    public void initialize(final TranslationContext TC, String upperwareGrouping, BaguetteServer server, Runnable callback) {
+        if (!isSupported(TC))
+            throw new IllegalArgumentException("Passed Translation Context is not supported");
+
+        super.initialize(TC, upperwareGrouping, server, callback);
+        List<GROUPING> groupings = TC.getG2R().keySet().stream()
+                .map(GROUPING::valueOf)
+                .sorted()
+                .collect(Collectors.toList());
+        log.debug("TwoLevelCoordinator.initialize(): Groupings: {}", groupings);
+        this.globalGrouping = groupings.get(0);
+        this.nodeGrouping = groupings.get(1);
+        log.info("TwoLevelCoordinator.initialize(): Groupings: top-level={}, node-level={}",
+                globalGrouping, nodeGrouping);
+
+        // Configure Self-Healing manager
+        server.getSelfHealingManager().setMode(SelfHealingManager.MODE.ALL);
+    }
+
+    @Override
+    public boolean processClientInput(ClientShellCommand csc, String line) {
+        if (StringUtils.isBlank(line)) return false;
+        log.info("TwoLevelCoordinator: Client: {} @ {} -- Input: {}",
+                csc.getId(), csc.getClientIpAddress(), line);
+        return true;
+    }
+
+    @Override
+    public synchronized void register(ClientShellCommand csc) {
+        if (!_logInvocation("register", csc, true)) return;
+
+        // prepare configuration
+        Map<String,BrokerConnectionConfig> connCfgMap = new LinkedHashMap<>();
+        BrokerConnectionConfig groupingConn = getUpperwareBrokerConfig(server);
+        connCfgMap.put(server.getUpperwareGrouping(), groupingConn);
+        log.trace("TwoLevelCoordinator: GLOBAL broker config.: {}", groupingConn);
+
+        // collect client configurations per grouping
+        for (String groupingName : server.getGroupingNames()) {
+            groupingConn = getGroupingBrokerConfig(groupingName, csc);
+            connCfgMap.put(groupingName, groupingConn);
+            log.trace("TwoLevelCoordinator: {} broker config.: {}", groupingName, groupingConn);
+        }
+
+        // send grouping configurations to client
+        log.info("TwoLevelCoordinator: --------------------------------------------------");
+        log.info("TwoLevelCoordinator: Sending grouping configurations to client {}...\n{}", csc.getId(), connCfgMap);
+        sendGroupingConfigurations(connCfgMap, csc, server);
+        log.info("TwoLevelCoordinator: Sending grouping configurations to client {}... done", csc.getId());
+        sleep(500);
+
+        // Set active grouping
+        String grouping = nodeGrouping.name();
+        log.info("TwoLevelCoordinator: --------------------------------------------------");
+        log.info("TwoLevelCoordinator: Setting active grouping of client {}: {}", csc.getId(), grouping);
+        csc.setActiveGrouping(grouping);
+        log.info("TwoLevelCoordinator: --------------------------------------------------");
+    }
+
+    @Override
+    public synchronized void unregister(ClientShellCommand csc) {
+        if (!_logInvocation("unregister", csc, true)) return;
+        log.info("TwoLevelCoordinator: --------------------------------------------------");
+        log.info("TwoLevelCoordinator: Client unregistered: {} @ {}", csc.getId(), csc.getClientIpAddress());
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/cluster/AtLeastTwoZoneManagementStrategy.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator.cluster;
+
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * A smarter than default Zone Management Strategy.
+ * It groups clients based on domain name, or last byte of IP Address. If neither is available it assigns client
+ * in a new zone identified by a random UUID.
+ * When a zone contains only one client, no cluster initialization is instructed.
+ * When a zone contains exactly two clients, they are both initialized as cluster nodes.
+ * If only one client is left in a zone, it is instructed to leave cluster.
+ */
+@Slf4j
+public class AtLeastTwoZoneManagementStrategy implements IZoneManagementStrategy {
+    @Override
+    public void notPreregisteredNode(ClientShellCommand csc) {
+        log.warn("AtLeastTwoZoneManagementStrategy: Unexpected node connected: {} @ {}", csc.getId(), csc.getClientIpAddress());
+    }
+
+    @Override
+    public void alreadyRegisteredNode(ClientShellCommand csc) {
+        log.warn("AtLeastTwoZoneManagementStrategy: Node connection from an already registered IP address: {} @ {}", csc.getId(), csc.getClientIpAddress());
+    }
+
+    @Override
+    public synchronized void nodeAdded(ClientShellCommand csc, ClusteringCoordinator coordinator, IClusterZone zone) {
+        if (zone.getNodes().size() < 2)
+            return;
+
+        if (zone.getNodes().size()==2) {
+            // Instruct first node to join cluster first (in fact to initialize it)
+            ClientShellCommand firstNode = zone.getNodes().get(0);
+            log.info("AtLeastTwoZoneManagementStrategy: First node to join cluster: client={}, zone={}", firstNode.getId(), zone.getId());
+            joinToCluster(firstNode, coordinator, zone);
+        }
+
+        // Instruct new node to join cluster
+        log.info("AtLeastTwoZoneManagementStrategy: Node to join cluster: client={}, zone={}", csc.getId(), zone.getId());
+        joinToCluster(csc, coordinator, zone);
+
+        // Instruct aggregator election if at least 2 nodes are present in the zone
+        if (zone.getNodes().size()==2) {
+            log.info("AtLeastTwoZoneManagementStrategy: Elect aggregator: zone={}", zone.getId());
+            coordinator.sleep(5000);
+            coordinator.electAggregator(zone);
+        }
+    }
+
+    private void joinToCluster(ClientShellCommand csc, ClusteringCoordinator coordinator, IClusterZone zone) {
+        coordinator.sendClusterKey(csc, zone);
+        coordinator.instructClusterJoin(csc, zone, false);
+
+        coordinator.sleep(1000);
+        csc.sendCommand("CLUSTER-EXEC broker list");
+        //coordinator.sleep(1000);
+        //csc.sendCommand("CLUSTER-TEST");
+    }
+
+    @Override
+    public synchronized void nodeRemoved(ClientShellCommand csc, ClusteringCoordinator coordinator, IClusterZone zone) {
+        // Instruct node to leave cluster
+        log.info("AtLeastTwoZoneManagementStrategy: Node to leave cluster: client={}, zone={}", csc.getId(), zone.getId());
+        coordinator.instructClusterLeave(csc, zone);
+
+        if (zone.getNodes().size()==1) {
+            // Instruct last node to leave cluster (and terminate cluster)
+            ClientShellCommand lastNode = zone.getNodes().get(0);
+            log.info("AtLeastTwoZoneManagementStrategy: Last node to leave cluster: client={}, zone={}", lastNode.getId(), zone.getId());
+            coordinator.instructClusterLeave(lastNode, zone);
+        }
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/cluster/ClusterSelfHealing.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator.cluster;
+
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.common.selfhealing.SelfHealingManager;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Slf4j
+@RequiredArgsConstructor
+public class ClusterSelfHealing {
+    private final SelfHealingManager<NodeRegistryEntry> selfHealingManager;
+
+    // ------------------------------------------------------------------------
+    // Server-side self-healing methods
+    // ------------------------------------------------------------------------
+
+    List<NodeRegistryEntry> getAggregatorCapableNodesInZone(IClusterZone zone) {
+        // Get the normal nodes in the zone that can be Aggregators (i.e. Aggregator and candidates)
+        List<NodeRegistryEntry> aggregatorCapableNodes = zone.findAggregatorCapableNodes();
+        if (log.isTraceEnabled()) {
+            log.trace("getAggregatorCapableNodesInZone: nodes={}", zone.getNodes().stream().map(ClientShellCommand::getNodeRegistryEntry).collect(Collectors.toList()));
+            log.trace("getAggregatorCapableNodesInZone: aggregatorCapableNodes={}", aggregatorCapableNodes);
+        }
+        return aggregatorCapableNodes;
+    }
+
+    void updateNodesSelfHealingMonitoring(IClusterZone zone, List<NodeRegistryEntry> aggregatorCapableNodes) {
+        if (aggregatorCapableNodes.size()>1) {
+            // If zone has >1 aggregator-capable nodes (i.e. Aggregator and Candidates) then stop monitoring them for server-side self-healing
+            // Aggregator will monitor them for client-side self-healing
+            List<NodeRegistryEntry> nodes = zone.getNodes().stream().map(ClientShellCommand::getNodeRegistryEntry).collect(Collectors.toList());
+            log.info("updateNodesSelfHealingMonitoring: Stop self-healing monitor for zone nodes: zone={}, clients={}",
+                    zone.getId(), nodes.stream().map(NodeRegistryEntry::getIpAddress).collect(Collectors.toList()));
+            selfHealingManager.removeAllNodes(nodes);
+        } else if (aggregatorCapableNodes.size()==1) {
+            // If zone has exactly 1 aggregator-capable node (i.e. Aggregator) then start monitoring it for server-side self-healing
+            // If Aggregator fails then EMS server must recover it
+            NodeRegistryEntry lastNode = aggregatorCapableNodes.get(0);
+            log.info("updateNodesSelfHealingMonitoring: Start self-healing monitor for the first/last node of zone: zone={}, client={}, address={}", zone.getId(), lastNode.getClientId(), lastNode.getIpAddress());
+            selfHealingManager.addNode(lastNode);
+        }
+    }
+
+    void removeResourceLimitedNodeSelfHealingMonitoring(IClusterZone zone, List<NodeRegistryEntry> aggregatorCapableNodes) {
+        // Remove self-healing responsibility of RL nodes from EMS server, if there are aggregator-capable nodes in the zone (since one will be/become Aggregator)
+        List<NodeRegistryEntry> clientlessNodes = zone.getNodesWithoutClient();
+        log.trace("removeResourceLimitedNodeSelfHealingMonitoring: AC-nodes: {}", aggregatorCapableNodes);
+        log.trace("removeResourceLimitedNodeSelfHealingMonitoring: RL-nodes: {}", clientlessNodes);
+        if (! clientlessNodes.isEmpty() && ! aggregatorCapableNodes.isEmpty()) {
+            if (log.isTraceEnabled()) {
+                log.trace("removeResourceLimitedNodeSelfHealingMonitoring: Zone has aggregators-capable node(s) and nodes without client: zone={}, nodes-without-client={}, aggregator-capable-nodes={}",
+                        zone.getId(), clientlessNodes.stream().map(NodeRegistryEntry::getIpAddress).collect(Collectors.toList()),
+                        aggregatorCapableNodes.stream().map(NodeRegistryEntry::getIpAddress).collect(Collectors.toList()));
+            }
+
+            boolean containsNodesWithoutClient = selfHealingManager.containsAny(zone.getNodesWithoutClient());
+            log.trace("removeResourceLimitedNodeSelfHealingMonitoring: containsAny={}", containsNodesWithoutClient);
+            if (containsNodesWithoutClient) {
+                // Remove RL nodes self-healing responsibility from EMS server
+                List<String> zoneNodesWithoutClient = zone.getNodesWithoutClient().stream().map(NodeRegistryEntry::getIpAddress).collect(Collectors.toList());
+                log.info("removeResourceLimitedNodeSelfHealingMonitoring: Zone has nodes without client. Will remove self-healing responsibility from EMS server: {}", zoneNodesWithoutClient);
+                selfHealingManager.removeAllNodes(zone.getNodesWithoutClient());
+                log.debug("removeResourceLimitedNodeSelfHealingMonitoring: Removed self-healing responsibility from EMS server, for zone nodes without client: {}", zoneNodesWithoutClient);
+            } else {
+                log.trace("removeResourceLimitedNodeSelfHealingMonitoring: No nodes without client have been assigned to EMS server: zone={}", zone.getId());
+            }
+        }
+    }
+
+    void addResourceLimitedNodeSelfHealingMonitoring(IClusterZone zone, List<NodeRegistryEntry> aggregatorCapableNodes) {
+        // Add self-healing responsibility of RL nodes to EMS server, if there are no aggregator-capable nodes in the zone
+        List<NodeRegistryEntry> clientlessNodes = zone.getNodesWithoutClient();
+        log.trace("addResourceLimitedNodeSelfHealingMonitoring: AC-nodes: {}", aggregatorCapableNodes);
+        log.trace("addResourceLimitedNodeSelfHealingMonitoring: RL-nodes: {}", clientlessNodes);
+        if (! clientlessNodes.isEmpty() && aggregatorCapableNodes.isEmpty()) {
+            if (log.isTraceEnabled()) {
+                log.trace("addResourceLimitedNodeSelfHealingMonitoring: Zone has no aggregator-capable nodes but it has nodes without client: zone={}, nodes-without-client={}, aggregator-capable-nodes={}",
+                        zone.getId(), clientlessNodes.stream().map(NodeRegistryEntry::getIpAddress).collect(Collectors.toList()),
+                        aggregatorCapableNodes.stream().map(NodeRegistryEntry::getIpAddress).collect(Collectors.toList()));
+            }
+
+            // Add RL nodes self-healing responsibility to EMS server
+            List<String> zoneNodesWithoutClient = zone.getNodesWithoutClient().stream().map(NodeRegistryEntry::getIpAddress).collect(Collectors.toList());
+            log.info("removeNodeFromTopology: Zone has only members without client. Will move self-healing responsibility to EMS server: {}", zoneNodesWithoutClient);
+            selfHealingManager.addAllNodes(zone.getNodesWithoutClient());
+            log.debug("removeNodeFromTopology: Moved self-healing responsibility to EMS server, for nodes without client: {}", zoneNodesWithoutClient);
+        }
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/cluster/ClusterZone.java

@@ -0,0 +1,202 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator.cluster;
+
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.util.ClientConfiguration;
+import gr.iccs.imu.ems.util.KeystoreUtil;
+import gr.iccs.imu.ems.util.PasswordUtil;
+import lombok.*;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.bouncycastle.operator.OperatorCreationException;
+
+import javax.validation.constraints.NotBlank;
+import java.io.File;
+import java.io.IOException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Data
+public class ClusterZone implements IClusterZone {
+    private final String id;
+    private final int startPort;
+    private final int endPort;
+
+    @Getter(AccessLevel.NONE)
+    private final AtomicInteger currentPort = new AtomicInteger(1200);
+    @Getter(AccessLevel.NONE)
+    private final Map<String, ClientShellCommand> nodes = new LinkedHashMap<>();
+    @Getter(AccessLevel.NONE)
+    private final Map<String, Integer> addressPortCache = new HashMap<>();
+    @Getter(AccessLevel.NONE)
+    private final Map<String, NodeRegistryEntry> nodesWithoutClient = new LinkedHashMap<>();
+
+    private final String clusterId;
+    private final String clusterKeystoreBase64;
+    private final File clusterKeystoreFile;
+    private final String clusterKeystoreType;
+    private final String clusterKeystorePassword;
+    @Getter @Setter
+    private ClientShellCommand aggregator;
+
+    public ClusterZone(@NotBlank String id, int startPort, int endPort, String keystoreFileName)
+            throws IOException, CertificateException, KeyStoreException, NoSuchAlgorithmException, OperatorCreationException
+    {
+        checkArgs(id, startPort, endPort);
+        this.id = id;
+        this.startPort = startPort;
+        this.endPort = endPort;
+        currentPort.set(startPort);
+
+        this.clusterId = RandomStringUtils.randomAlphanumeric(64);
+        this.clusterKeystoreFile = new File(keystoreFileName);
+        this.clusterKeystoreType = "JKS";
+        this.clusterKeystorePassword = RandomStringUtils.randomAlphanumeric(64);
+        log.info("New ClusterZone:  zone: {}", id);
+        log.info("                  file: {}", clusterKeystoreFile);
+        log.info("                  type: {}", clusterKeystoreType);
+        log.debug("             password: {}", PasswordUtil.getInstance().encodePassword(clusterKeystorePassword));
+
+        log.trace("ClusterZone.<init>: Cluster Keystore: file={}, type={}, pass={}", clusterKeystoreFile.getCanonicalPath(), clusterKeystoreType, clusterKeystorePassword);
+        log.trace("ClusterZone.<init>: Cluster Id: {}", clusterId);
+        this.clusterKeystoreBase64 = KeystoreUtil
+                .getKeystore(clusterKeystoreFile.getCanonicalPath(), clusterKeystoreType, clusterKeystorePassword)
+                .createIfNotExist()
+                .createKeyAndCert(clusterId, "CN=" + clusterId, "")
+                .readFileAsBase64();
+        log.debug("       Base64 content: {}",
+                StringUtils.isNotBlank(clusterKeystoreBase64) ? "Not empty" : "!!! Empty !!!");
+        if (log.isTraceEnabled())
+            log.trace("ClusterZone.<init>: Cluster Keystore: Base64: {}", PasswordUtil.getInstance().encodePassword(clusterKeystoreBase64));
+    }
+
+    private void checkArgs(String id, int startPort, int endPort) {
+        if (StringUtils.isBlank(id))
+            throw new IllegalArgumentException("Zone id cannot be null or blank: zone-id="+id);
+        if (startPort<1 || endPort<1 || startPort>65535 || endPort>65535)
+            throw new IllegalArgumentException("Zone start/end port must be between 1 and 65535: zone-id="+id+", start="+startPort+", end="+endPort);
+        if (startPort > endPort)
+            throw new IllegalArgumentException("Zone start port must be less than or equal to end port: zone-id="+id+", start="+startPort+", end="+endPort);
+    }
+
+    public int getPortForAddress(String address) {
+        return addressPortCache.computeIfAbsent(address, k -> {
+            int port = currentPort.incrementAndGet();
+            if (port>endPort)
+                throw new IllegalStateException("Zone ports exhausted: "+id);
+            log.debug("Mapped address-to-port: {} -> {}", address, port);
+            return port;
+        });
+    }
+
+    public void clearAddressToPortCache() {
+        addressPortCache.clear();
+    }
+
+    // Nodes management
+    public void addNode(@NonNull ClientShellCommand csc) {
+        synchronized (Objects.requireNonNull(csc)) {
+            nodes.put(csc.getClientIpAddress(), csc);
+            csc.setClientZone(this);
+            csc.getNodeRegistryEntry().setClusterZone(this);
+        }
+    }
+
+    public void removeNode(@NonNull ClientShellCommand csc) {
+        synchronized (Objects.requireNonNull(csc)) {
+            nodes.remove(csc.getClientIpAddress());
+            if (csc.getClientZone()==this)
+                csc.setClientZone(null);
+            if (csc.getNodeRegistryEntry()!=null && csc.getNodeRegistryEntry().getClusterZone()==this)
+                csc.getNodeRegistryEntry().setClusterZone(null);
+            if (aggregator==csc)
+                setAggregator(null);
+        }
+    }
+
+    public Set<String> getNodeAddresses() {
+        return new HashSet<>(nodes.keySet());
+    }
+
+    public List<ClientShellCommand> getNodes() {
+        return new ArrayList<>(nodes.values());
+    }
+
+    public ClientShellCommand getNodeByAddress(String address) {
+        return nodes.get(address);
+    }
+
+    public List<NodeRegistryEntry> findAggregatorCapableNodes() {
+        return this.nodes.values().stream()
+                .filter(Objects::nonNull)
+                .map(ClientShellCommand::getNodeRegistryEntry)
+                .filter(Objects::nonNull)
+                .filter(entry -> entry.getState()==NodeRegistryEntry.STATE.REGISTERED || entry.getState()==NodeRegistryEntry.STATE.REGISTERING)
+                .collect(Collectors.toList());
+    }
+
+    // Nodes-without-Clients management
+    public void addNodeWithoutClient(@NonNull NodeRegistryEntry entry) {
+        synchronized (Objects.requireNonNull(entry)) {
+            String address = entry.getIpAddress();
+            if (address == null) address = entry.getNodeAddress();
+            if (address == null) throw new IllegalArgumentException("Node address not found in Preregistration info");
+            nodesWithoutClient.put(address, entry);
+            entry.setClusterZone(this);
+            sendClientConfigurationToZoneClients();
+        }
+    }
+
+    public void removeNodeWithoutClient(@NonNull NodeRegistryEntry entry) {
+        synchronized (Objects.requireNonNull(entry)) {
+            String address = entry.getIpAddress();
+            if (address == null) address = entry.getNodeAddress();
+            if (address == null) throw new IllegalArgumentException("Node address not found in Preregistration info");
+            nodesWithoutClient.remove(address);
+            if (entry.getClusterZone() == this)
+                entry.setClusterZone(null);
+            sendClientConfigurationToZoneClients();
+        }
+    }
+
+    public Set<String> getNodeWithoutClientAddresses() {
+        return new HashSet<>(nodesWithoutClient.keySet());
+    }
+
+    public List<NodeRegistryEntry> getNodesWithoutClient() {
+        return new ArrayList<>(nodesWithoutClient.values());
+    }
+
+    public NodeRegistryEntry getNodeWithoutClientByAddress(String address) {
+        return nodesWithoutClient.get(address);
+    }
+
+    public ClientConfiguration getClientConfiguration() {
+        return ClientConfiguration.builder()
+                .nodesWithoutClient(new HashSet<>(nodesWithoutClient.keySet()))
+                .build();
+    }
+
+    public ClientConfiguration sendClientConfigurationToZoneClients() {
+        ClientConfiguration cc = ClientConfiguration.builder()
+                .nodesWithoutClient(new HashSet<>(nodesWithoutClient.keySet()))
+                .build();
+        ClientShellCommand.sendClientConfigurationToClients(cc , getNodes());
+        return cc;
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/cluster/ClusterZoneDetector.java

@@ -0,0 +1,166 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator.cluster;
+
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringSubstitutor;
+import org.springframework.context.expression.MapAccessor;
+import org.springframework.expression.spel.standard.SpelExpressionParser;
+import org.springframework.expression.spel.support.StandardEvaluationContext;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Detects the Cluster/Zone the given node must be added,
+ * using node's pre-registration info and a set of configured rules
+ */
+@Slf4j
+public class ClusterZoneDetector implements IClusterZoneDetector {
+    private final static List<String> DEFAULT_ZONE_DETECTION_RULES = Arrays.asList(
+            "'${zone:-}'",
+            "'${zone-id:-}'",
+            "'${region:-}'",
+            "'${region-id:-}'",
+            "'${cloud:-}'",
+            "'${cloud-id:-}'",
+            "'${provider:-}'",
+            "'${provider-id:-}'",
+            "T(java.time.OffsetDateTime).now().toString()",
+//            "'Cluster_'+T(java.lang.System).currentTimeMillis()",
+//            "'Cluster_'+T(java.util.UUID).randomUUID()",
+            ""
+    );
+    private final static RULE_TYPE DEFAULT_RULES_TYPE = RULE_TYPE.SPEL;
+    private final static List<String> DEFAULT_ZONES = Collections.singletonList("DEFAULT_CLUSTER");
+    private final static ASSIGNMENT_TO_DEFAULT_CLUSTERS DEFAULT_ASSIGNMENT_TO_DEFAULT_CLUSTERS = ASSIGNMENT_TO_DEFAULT_CLUSTERS.RANDOM;
+
+    enum RULE_TYPE { SPEL, MAP }
+    enum ASSIGNMENT_TO_DEFAULT_CLUSTERS { RANDOM, SEQUENTIAL }
+
+    private RULE_TYPE clusterDetectionRulesType = DEFAULT_RULES_TYPE;
+    private List<String> clusterDetectionRules = DEFAULT_ZONE_DETECTION_RULES;
+    private List<String> defaultClusters = DEFAULT_ZONES;
+    private ASSIGNMENT_TO_DEFAULT_CLUSTERS assignmentToDefaultClusters = DEFAULT_ASSIGNMENT_TO_DEFAULT_CLUSTERS;
+
+    private SpelExpressionParser parser = new SpelExpressionParser();
+    private AtomicInteger currentDefaultCluster = new AtomicInteger(0);
+
+    @Override
+    public void setProperties(Map<String, String> zoneConfig) {
+        log.debug("ClusterZoneDetector: setProperties: BEGIN: config: {}", zoneConfig);
+
+        // Get rules type (Map keys or SpEL expressions)
+        RULE_TYPE rulesType = RULE_TYPE.valueOf(
+                zoneConfig.getOrDefault("cluster-detector-rules-type", DEFAULT_RULES_TYPE.toString()).toUpperCase());
+
+        // Get rules texts and separator
+        String separator = zoneConfig.getOrDefault("cluster-detector-rules-separator", ",");
+        String rulesStr = zoneConfig.getOrDefault("cluster-detector-rules", null);
+        if (StringUtils.isNotBlank(rulesStr)) {
+            List<String> rulesList = Arrays.stream(rulesStr.split(separator))
+                    .filter(StringUtils::isNotBlank)
+                    .map(String::trim)
+                    .map(String::trim)
+                    .collect(Collectors.toList());
+            clusterDetectionRules = (rulesList.size()>0) ? rulesList : DEFAULT_ZONE_DETECTION_RULES;
+            clusterDetectionRulesType = (rulesList.size()>0) ? rulesType : DEFAULT_RULES_TYPE;
+        }
+
+        // Get the default cluster(s)
+        List<String> defaultsList = Arrays.stream(zoneConfig.getOrDefault("default-clusters", "").split(","))
+                .filter(StringUtils::isNotBlank)
+                .map(String::trim)
+                .collect(Collectors.toList());
+        defaultClusters = (defaultsList.size()>0) ? defaultsList : DEFAULT_ZONES;
+
+        // Get assignment method to default clusters
+        assignmentToDefaultClusters = ASSIGNMENT_TO_DEFAULT_CLUSTERS.valueOf(
+                zoneConfig.getOrDefault("assignment-to-default-clusters", DEFAULT_ASSIGNMENT_TO_DEFAULT_CLUSTERS.toString().toUpperCase()));
+
+        log.debug("ClusterZoneDetector: setProperties: clusterDetectionRulesType: {}", clusterDetectionRulesType);
+        log.debug("ClusterZoneDetector: setProperties: clusterDetectionRules: {}", clusterDetectionRules);
+        log.debug("ClusterZoneDetector: setProperties: defaultClusters: {}", defaultClusters);
+        log.debug("ClusterZoneDetector: setProperties: assignmentToDefaultClusters: {}", assignmentToDefaultClusters);
+    }
+
+    @Override
+    public String getZoneIdFor(ClientShellCommand csc) {
+        log.trace("ClusterZoneDetector: getZoneIdFor: BEGIN: CSC: {}", csc);
+        return csc.getClientZone()==null || StringUtils.isBlank(csc.getClientZone().getId())
+                ? getZoneIdFor(csc.getNodeRegistryEntry())
+                : csc.getClientZone().getId();
+    }
+
+    @Override
+    public String getZoneIdFor(NodeRegistryEntry entry) {
+        log.trace("ClusterZoneDetector: getZoneIdFor: BEGIN: NRE: {}", entry);
+        final Map<String, String> info = entry.getPreregistration();
+
+        // Select and initialize the right valueMapper based on rules type
+        log.trace("ClusterZoneDetector: getZoneIdFor: PREREGISTRATION-INFO: {}", info);
+        Function<String,String> valueMapper;
+        switch (clusterDetectionRulesType) {
+            case SPEL:
+                StandardEvaluationContext context = new StandardEvaluationContext(info);
+                context.addPropertyAccessor(new MapAccessor());
+                valueMapper = expression -> {
+                    log.trace("ClusterZoneDetector: getZoneIdFor: Expression: {}", expression);
+                    expression = StringSubstitutor.replace(expression, info);
+                    expression = StringSubstitutor.replaceSystemProperties(expression);
+                    log.trace("ClusterZoneDetector: getZoneIdFor: SpEL expr.: {}", expression);
+                    String result = parser.parseRaw(expression).getValue(context, String.class);
+                    log.trace("ClusterZoneDetector: getZoneIdFor:     Result: {}", result);
+                    return StringUtils.isBlank(result) ? null : result.trim();
+                };
+                break;
+            case MAP:
+                valueMapper = info::get;
+                break;
+            default:
+                throw new IllegalArgumentException("Unsupported RULE_TYPE: "+ clusterDetectionRulesType);
+        }
+
+        // Process rules one-by-one, using valueMapper, until one rule yields a non-blank value
+        String zoneId = clusterDetectionRules.stream()
+                .filter(StringUtils::isNotBlank)
+                .peek(s -> log.trace("ClusterZoneDetector: getZoneIdFor: RULE: {}", s))
+                .map(valueMapper)
+                .peek(s -> log.trace("ClusterZoneDetector: getZoneIdFor: RESULT: {}", s))
+                .filter(StringUtils::isNotBlank)
+                .findFirst()
+                .orElse(null);
+        log.debug("ClusterZoneDetector: getZoneIdFor: Intermediate: zoneId: {}", zoneId);
+
+        // If all rules yielded blank values then a default cluster id will be selected, using the assignment method
+        if (StringUtils.isBlank(zoneId)) {
+            switch (assignmentToDefaultClusters) {
+                case RANDOM:
+                    zoneId = defaultClusters.get((int) (Math.random() * defaultClusters.size()));
+                    break;
+                case SEQUENTIAL:
+                    zoneId = defaultClusters.get(currentDefaultCluster.getAndUpdate(operand -> (operand + 1) % defaultClusters.size()));
+                    break;
+                default:
+                    throw new IllegalArgumentException("Unsupported ASSIGNMENT_TO_DEFAULT_CLUSTERS: "+assignmentToDefaultClusters);
+            }
+        }
+        log.debug("ClusterZoneDetector: getZoneIdFor: END: zoneId: {}", zoneId);
+        return zoneId;
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/cluster/ClusteringCoordinator.java

@@ -0,0 +1,446 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator.cluster;
+
+import gr.iccs.imu.ems.baguette.server.BaguetteServer;
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.baguette.server.coordinator.NoopCoordinator;
+import gr.iccs.imu.ems.common.selfhealing.SelfHealingManager;
+import gr.iccs.imu.ems.translate.TranslationContext;
+import gr.iccs.imu.ems.util.ClientConfiguration;
+import gr.iccs.imu.ems.util.GROUPING;
+import lombok.NonNull;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringSubstitutor;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static gr.iccs.imu.ems.util.GroupingConfiguration.BrokerConnectionConfig;
+
+@Slf4j
+public class ClusteringCoordinator extends NoopCoordinator {
+    private final static String DEFAULT_ZONE = "default_zone";
+
+    private final Map<String, ClusterZone> topologyMap = new HashMap<>();
+
+    private IClusterZoneDetector clusterZoneDetector;
+    private IZoneManagementStrategy zoneManagementStrategy;
+    private int zoneStartPort = 1200;
+    private int zoneEndPort = 65535;
+    private String zoneKeystoreFileNameFormatter = "logs/cluster_${TIMESTAMP}_${ZONE_ID}.p12";
+
+    private GROUPING topLevelGrouping;
+    private GROUPING aggregatorGrouping;
+    private GROUPING lastLevelGrouping;
+
+    private final Map<String, NodeRegistryEntry> ignoredNodes = new LinkedHashMap<>();
+    private ClusterSelfHealing clusterSelfHealing;
+
+    public Collection<String> getClusterIdSet() { return topologyMap.keySet(); }
+    public Collection<IClusterZone> getClusters() { return topologyMap.values().stream().map(c->(IClusterZone)c).collect(Collectors.toList()); }
+    public IClusterZone getCluster(String id) { return topologyMap.get(id); }
+
+    @Override
+    public boolean isSupported(final TranslationContext _TC) {
+        log.trace("ClusteringCoordinator.isSupported: TC: {}", _TC);
+
+        // Check if it is a 3-level architecture
+        Set<String> groupings = _TC.getG2R().keySet();
+        log.trace("ClusteringCoordinator.isSupported: Groupings: {}", groupings);
+        log.trace("ClusteringCoordinator.isSupported: Contains GLOBAL: {}", groupings.contains("GLOBAL"));
+        log.trace("ClusteringCoordinator.isSupported: Num of Levels: {}", groupings.size());
+
+        if (!groupings.contains("GLOBAL")) return false;
+        return groupings.size()==3;
+    }
+
+    @Override
+    public boolean supportsAggregators() {
+        return true;
+    }
+
+    @Override
+    public void initialize(final TranslationContext TC, String upperwareGrouping, BaguetteServer server, Runnable callback) {
+        if (!isSupported(TC))
+            throw new IllegalArgumentException("Passed Translation Context is not supported");
+
+        super.initialize(TC, upperwareGrouping, server, callback);
+        List<GROUPING> groupings = TC.getG2R().keySet().stream()
+                .map(GROUPING::valueOf)
+                .sorted()
+                .collect(Collectors.toList());
+        log.debug("ClusteringCoordinator.initialize(): Groupings: {}", groupings);
+        this.topLevelGrouping = groupings.get(0);
+        this.aggregatorGrouping = groupings.get(1);
+        this.lastLevelGrouping = groupings.get(2);
+        log.info("ClusteringCoordinator.initialize(): Groupings: top-level={}, aggregator={}, last-level={}",
+                topLevelGrouping, aggregatorGrouping, lastLevelGrouping);
+
+        // Configure Self-Healing manager
+        clusterSelfHealing = new ClusterSelfHealing(server.getSelfHealingManager());
+        server.getSelfHealingManager().setMode(SelfHealingManager.MODE.INCLUDED);
+    }
+
+    @SneakyThrows
+    public void setProperties(Map<String, String> zoneConfig) {
+        log.debug("Zone configuration: {}", zoneConfig);
+        zoneManagementStrategy = zoneConfig.containsKey("zone-management-strategy-class")
+                ? (IZoneManagementStrategy) Class.forName(zoneConfig.get("zone-management-strategy-class")).getConstructor().newInstance()
+                : new DefaultZoneManagementStrategy();
+        zoneStartPort = zoneConfig.containsKey("zone-port-start")
+                ? Integer.parseInt(zoneConfig.get("zone-port-start")) : zoneStartPort;
+        zoneEndPort = zoneConfig.containsKey("zone-port-end")
+                ? Integer.parseInt(zoneConfig.get("zone-port-end")) : zoneEndPort;
+        zoneKeystoreFileNameFormatter = zoneConfig.containsKey("zone-keystore-file-name-formatter")
+                ? zoneConfig.get("zone-keystore-file-name-formatter") : zoneKeystoreFileNameFormatter;
+
+        // Initialize Cluster Detector
+        String clusterDetectorClass = zoneConfig.get("cluster-detector-class");
+        if (StringUtils.isNotBlank(clusterDetectorClass)) {
+            Class<?> clazz = Class.forName(clusterDetectorClass);
+            if (clazz.isAssignableFrom(IClusterZoneDetector.class))
+                clusterZoneDetector = (IClusterZoneDetector) clazz.getConstructor().newInstance();
+            else
+                throw new IllegalArgumentException("Invalid Cluster Detector class. Not implementing IClusterZoneDetector interface: "+clazz.getName());
+        } else {
+            clusterZoneDetector = new ClusterZoneDetector();
+        }
+        clusterZoneDetector.setProperties(zoneConfig);
+        log.info("Cluster Detector class: {}", clusterZoneDetector.getClass().getName());
+    }
+
+    @Override
+    public boolean processClientInput(ClientShellCommand csc, String line) {
+        if (StringUtils.isBlank(line)) return false;
+        String[] args = Arrays.stream(line.trim().split("[ \t\r\n]+")).filter(StringUtils::isNotBlank).map(String::trim).toArray(String[]::new);
+        if (!"CLUSTER".equalsIgnoreCase(args[0])) return false;
+        if ("AGGREGATOR".equalsIgnoreCase(args[1])) {
+            String clientId1 = csc.getId();
+            String clientId2 = csc.getClientId();
+            String clientId3 = args[2];
+            log.trace("processClientInput: csc.zone: {}", csc.getClientZone()!=null ? csc.getClientZone().getId() : null);
+            log.trace("processClientInput: topology-map: {}", topologyMap.keySet());
+            ClusterZone zone = findZone(csc);
+            log.trace("processClientInput: zone={}", zone);
+            zone.setAggregator(csc);
+            log.info("Updated aggregator of zone: {} -- New aggregator: {} @ {} ({})",
+                    zone.getId(), clientId1, csc.getClientIpAddress(), clientId2);
+        }
+        return true;
+    }
+
+    private ClusterZone findZone(ClientShellCommand csc) {
+        String zoneId = clusterZoneDetector.getZoneIdFor(csc);
+        return topologyMap.get(zoneId);
+    }
+
+    @Override
+    public boolean allowAlreadyPreregisteredNode(Map<String,Object> nodeInfo) {
+        return zoneManagementStrategy.allowAlreadyPreregisteredNode(nodeInfo);
+    }
+
+    @Override
+    public boolean allowAlreadyRegisteredNode(ClientShellCommand csc) {
+        return zoneManagementStrategy.allowAlreadyRegisteredNode(csc);
+    }
+
+    @Override
+    public boolean allowNotPreregisteredNode(ClientShellCommand csc) {
+        return zoneManagementStrategy.allowNotPreregisteredNode(csc);
+    }
+
+    @Override
+    public synchronized void preregister(@NonNull NodeRegistryEntry entry) {
+        log.debug("ClusteringCoordinator: preregister: BEGIN: NRE:\n{}", entry);
+
+        if (!_logInvocation("preregister", entry.getNodeIdAndAddress(), true)) return;
+
+        // Check if client has been preregistered (or connected without being expected)
+        /*if (zoneManagementStrategy.allowNotPreregisteredNode(entry)) {
+            log.warn("Non-Preregistered node will be preregistered: {} @ {}", entry.getClientId(), entry.getIpAddress());
+            zoneManagementStrategy.notPreregisteredNode(entry);
+        }*/
+
+        log.debug("ClusteringCoordinator: preregister: Checking node State: node={}, state={}", entry.getNodeIdAndAddress(), entry.getState());
+        if (entry.getState()==NodeRegistryEntry.STATE.IGNORE_NODE) {
+            // Add in ignored nodes list
+            log.info("ClusteringCoordinator: preregister: Ignoring node: node={}, state={}", entry.getNodeIdAndAddress(), entry.getState());
+            ignoredNodes.put(entry.getIpAddress(), entry);
+        } else
+        if (entry.getState()==NodeRegistryEntry.STATE.NOT_INSTALLED) {
+            // Append to Nodes without EMS client (e.g. Edge devices, resource-limited VM's)
+            log.debug("ClusteringCoordinator: preregister: Adding node without EMS client: node={}, state={}", entry.getNodeIdAndAddress(), entry.getState());
+
+            // Assign node-without-client in a zone
+            String zoneId = clusterZoneDetector.getZoneIdFor(entry);
+            log.debug("ClusteringCoordinator: preregister: New entry: node={}, zone-id={}", entry.getNodeIdAndAddress(), zoneId);
+            if (log.isTraceEnabled()) {
+                log.trace("preregister: topologyMap: BEFORE: keys={}", topologyMap.keySet());
+                log.trace("preregister: topologyMap: containsKey: key={}, result={}", zoneId, topologyMap.containsKey(zoneId));
+            }
+            ClusterZone zone = topologyMap.computeIfAbsent(zoneId, this::createClusterZone);
+            log.trace("ClusteringCoordinator: preregister: Zone members without client: BEFORE: {}", zone.getNodesWithoutClient());
+            zone.addNodeWithoutClient(entry);
+            log.trace("ClusteringCoordinator: preregister: Zone members without client:  AFTER: {}", zone.getNodesWithoutClient());
+        } else
+        if (entry.getState()==NodeRegistryEntry.STATE.INSTALLED) {
+            // Append to normal Node with EMS client
+            log.debug("ClusteringCoordinator: preregister: Node with EMS client: node={}, state={}", entry.getNodeIdAndAddress(), entry.getState());
+            // No need to do something
+        } else {
+            // Other states are ignored
+            log.warn("ClusteringCoordinator: preregister: No preregistration due to node state: node={}, state={}", entry.getNodeIdAndAddress(), entry.getState());
+        }
+    }
+
+    @SneakyThrows
+    private ClusterZone createClusterZone(@NonNull String id) {
+        Map<String,String> values = new HashMap<>();
+        values.put("TIMESTAMP", ""+System.currentTimeMillis());
+        values.put("ZONE_ID", id.replaceAll("[^A-Za-z0-9_]", "_"));
+        String keystoreFile = StringSubstitutor.replace(zoneKeystoreFileNameFormatter, values);
+        return new ClusterZone(id, zoneStartPort, zoneEndPort, keystoreFile);
+    }
+
+    @Override
+    public synchronized void register(ClientShellCommand csc) {
+        if (!_logInvocation("register", csc, true)) return;
+
+        // Check if client has been preregistered (or connected without being expected)
+        NodeRegistryEntry preregEntry = server.getNodeRegistry().getNodeByAddress(csc.getClientIpAddress());
+        log.debug("Preregistered info for node: {} @ {}:\n{}", csc.getId(), csc.getClientIpAddress(), preregEntry);
+        if (preregEntry==null && zoneManagementStrategy.allowNotPreregisteredNode(csc)) {
+            log.warn("Non Preregistered node connected: {} @ {}", csc.getId(), csc.getClientIpAddress());
+            log.warn("Preregistered nodes: {}", server.getNodeRegistry().getNodes().stream()
+                    .map(entry->entry.getState()+"/"+entry.getIpAddress()+"/"+entry.getNodeIdAndAddress()+"/"+entry.getClientId())
+                    .collect(Collectors.toList()));
+            zoneManagementStrategy.notPreregisteredNode(csc);
+        } else if (preregEntry==null) {
+            log.warn("Non Preregistered node is refused connection: {} @ {}", csc.getId(), csc.getClientIpAddress());
+            csc.setCloseConnection(true);
+            return;
+        }
+
+        // Check if client has already been registered (i.e. is still connected)
+        ClientShellCommand regEntry = topologyMap.values().stream()
+                .map(zone->zone.getNodeByAddress(csc.getClientIpAddress()))
+                .filter(Objects::nonNull)
+                .findAny().orElse(null);
+        log.debug("Registered CSC for node: {} @ {}:\n{}", csc.getId(), csc.getClientIpAddress(), regEntry);
+        if (regEntry!=null && allowAlreadyRegisteredNode(csc)) {
+            log.warn("Already Registered node connected: {} @ {}", csc.getId(), csc.getClientIpAddress());
+            zoneManagementStrategy.alreadyRegisteredNode(csc);
+        } else if (regEntry!=null) {
+            log.warn("New node is refused connection because an active connection from the same IP address already exists: {} @ {}", csc.getId(), csc.getClientIpAddress());
+            csc.setCloseConnection(true);
+            return;
+        }
+
+        // Register client
+        _do_register(csc);
+    }
+
+    @Override
+    public synchronized void unregister(ClientShellCommand csc) {
+        if (!_logInvocation("unregister", csc, true)) return;
+        _do_unregister(csc);
+    }
+
+    protected synchronized void _do_register(ClientShellCommand csc) {
+        // Add registered node in topology map
+        addNodeInTopology(csc);
+
+        // collect client configuration
+        ClientConfiguration clientConfig = csc.getClientZone().getClientConfiguration();
+
+        // prepare configuration
+        Map<String,BrokerConnectionConfig> connCfgMap = new LinkedHashMap<>();
+        BrokerConnectionConfig groupingConn = getUpperwareBrokerConfig(server);
+        connCfgMap.put(server.getUpperwareGrouping(), groupingConn);
+        log.trace("ClusteringCoordinator: GLOBAL broker config.: {}", groupingConn);
+
+        // collect client configurations per grouping
+        for (String groupingName : server.getGroupingNames()) {
+            groupingConn = getGroupingBrokerConfig(groupingName, csc);
+            connCfgMap.put(groupingName, groupingConn);
+            log.trace("ClusteringCoordinator: {} broker config.: {}", groupingName, groupingConn);
+        }
+
+        // send client configuration to client
+        log.info("ClusteringCoordinator: --------------------------------------------------");
+        log.info("ClusteringCoordinator: Sending client configuration to client {}...\n{}", csc.getId(), clientConfig);
+        csc.getClientZone().sendClientConfigurationToZoneClients();
+        log.info("ClusteringCoordinator: Sending client configuration to client {}... done", csc.getId());
+        sleep(500);
+
+        // send grouping configurations to client
+        log.info("ClusteringCoordinator: --------------------------------------------------");
+        log.info("ClusteringCoordinator: Sending grouping configurations to client {}...\n{}", csc.getId(), connCfgMap);
+        sendGroupingConfigurations(connCfgMap, csc, server);
+        log.info("ClusteringCoordinator: Sending grouping configurations to client {}... done", csc.getId());
+        sleep(500);
+
+        // Set active grouping
+        String grouping = lastLevelGrouping.name();
+        log.info("ClusteringCoordinator: --------------------------------------------------");
+        log.info("ClusteringCoordinator: Setting active grouping of client {}: {}", csc.getId(), grouping);
+        csc.setActiveGrouping(grouping);
+        log.info("ClusteringCoordinator: --------------------------------------------------");
+        sleep(500);
+
+        // Registered node added in topology map - Notify ZoneManagementStrategy
+        addedNodeInTopology(csc);
+    }
+
+    private synchronized void addNodeInTopology(ClientShellCommand csc) {
+        // Assign client in a zone
+        String zoneId = clusterZoneDetector.getZoneIdFor(csc);
+        log.debug("addNodeInTopology: New client: id={}, address={}, zone-id={}", csc.getId(), csc.getClientIpAddress(), zoneId);
+        ClusterZone zone = topologyMap.computeIfAbsent(zoneId, this::createClusterZone);
+        log.trace("addNodeInTopology: Zone members: BEFORE: {}", zone.getNodes());
+        zone.addNode(csc);
+        log.trace("addNodeInTopology: Zone members:  AFTER: {}", zone.getNodes());
+
+        // Initialize new client's cluster node address/hostname, port and certificate
+        String nodeAddress = csc.getClientIpAddress();
+        String nodeHostname = csc.getClientHostname();
+        String nodeCanonical = csc.getClientCanonicalHostname();
+        int nodePort = zone.getPortForAddress(nodeAddress);
+        csc.setClientClusterNodePort(nodePort);
+        csc.setClientClusterNodeAddress(nodeAddress);
+        csc.setClientClusterNodeHostname(nodeHostname);
+        //csc.setClientClusterNodeHostname(nodeCanonical);
+        log.debug("addNodeInTopology: New client: Cluster node: address={}, hostname={} // {}, port={}",
+                nodeAddress, nodeHostname, nodeCanonical, nodePort);
+    }
+
+    private synchronized void addedNodeInTopology(ClientShellCommand csc) {
+        // Signal Zone Management Strategy for new client registration
+        zoneManagementStrategy.nodeAdded(csc, this, csc.getClientZone());
+        log.info("addNodeInTopology: Client added in topology: client={}, address={}", csc.getId(), csc.getClientIpAddress());
+
+        if (csc.getClientZone()!=null) {
+            IClusterZone zone = csc.getClientZone();
+            log.trace("addNodeInTopology: CSC is in zone: client={}, address={}, zone={}", csc.getId(), csc.getClientIpAddress(), zone.getId());
+
+            // Self-healing-related actions
+            List<NodeRegistryEntry> aggregatorCapableNodes = clusterSelfHealing.getAggregatorCapableNodesInZone(zone);
+            clusterSelfHealing.updateNodesSelfHealingMonitoring(zone, aggregatorCapableNodes);
+            clusterSelfHealing.removeResourceLimitedNodeSelfHealingMonitoring(zone, aggregatorCapableNodes);
+        }
+    }
+
+    protected synchronized void _do_unregister(ClientShellCommand csc) {
+        // Remove node from topology map
+        removeNodeFromTopology(csc);
+    }
+
+    private synchronized void removeNodeFromTopology(ClientShellCommand csc) {
+        // Assign client in a zone
+        String zoneId = csc.getNodeRegistryEntry()!=null ? clusterZoneDetector.getZoneIdFor(csc) : null;
+        ClusterZone zone = StringUtils.isNotBlank(zoneId) ? topologyMap.get(zoneId) : null;
+        if (zone == null) {
+            log.warn("removeNodeFromTopology: Non-registered client removed: client={}, address={}, zone-id={}", csc.getId(), csc.getClientIpAddress(), zoneId);
+            log.debug("removeNodeFromTopology: Non-registered client removed: entry={}", csc.getNodeRegistryEntry());
+        } else {
+            log.trace("removeNodeFromTopology: Zone members: BEFORE: {}", zone.getNodes());
+            zone.removeNode(csc);
+            log.trace("removeNodeFromTopology: Zone members:  AFTER: {}", zone.getNodes());
+            zoneManagementStrategy.nodeRemoved(csc, this, zone);
+            log.info("removeNodeFromTopology: Client removed from topology: client={}, address={}", csc.getId(), csc.getClientIpAddress());
+
+            ClientShellCommand aggregator = zone.getAggregator();
+            if (aggregator==csc || aggregator==null) {
+                if (aggregator==csc) zone.setAggregator(null);
+                log.warn("removeNodeFromTopology: Zone without aggregator: zone-id={}, old-aggregator-id={}, address={}", zone.getId(), csc.getId(), csc.getClientIpAddress());
+
+                // Nothing to do. Client-side self-healing must elect a new Aggregator
+                // Optionally, we can start a timer so that if no Aggregator is elected within a period, then we can appoint one or trigger Server-side self-healing
+            }
+
+            // Self-healing-related actions
+            List<NodeRegistryEntry> aggregatorCapableNodes = clusterSelfHealing.getAggregatorCapableNodesInZone(zone);
+            clusterSelfHealing.updateNodesSelfHealingMonitoring(zone, aggregatorCapableNodes);
+            if (aggregatorCapableNodes.isEmpty())
+                ; //XXX: TODO: ??Reconfigure non-candidate nodes to forward their events to EMS server??
+            clusterSelfHealing.addResourceLimitedNodeSelfHealingMonitoring(zone, aggregatorCapableNodes);
+        }
+    }
+
+    // ------------------------------------------------------------------------
+    // Methods to be used by Zone Management Strategies
+    // ------------------------------------------------------------------------
+
+    void sendClusterKey(ClientShellCommand csc, IClusterZone zoneInfo) {
+        csc.sendCommand(String.format("CLUSTER-KEY %s %s %s %s",
+                zoneInfo.getClusterKeystoreFile().getName(), zoneInfo.getClusterKeystoreType(),
+                zoneInfo.getClusterKeystorePassword(), zoneInfo.getClusterKeystoreBase64()));
+    }
+
+    void sendCommandToZone(String command, List<ClientShellCommand> zoneNodes) {
+        log.info("sendCommandToZone: Sending command: \"{}\" to zone nodes: {}", command,
+                zoneNodes.stream().map(ClientShellCommand::toStringCluster).collect(Collectors.toList()));
+        zoneNodes.forEach(c -> c.sendCommand(command));
+    }
+
+    void instructClusterJoin(ClientShellCommand csc, IClusterZone zone, boolean startElection) {
+        List<ClientShellCommand> zoneNodes = zone.getNodes();
+        log.debug("instructClusterJoin: Zone members: {}", zoneNodes);
+
+        // Build zone members list
+        final List<String> addresses = new ArrayList<>();
+        final List<String> hostnames = new ArrayList<>();
+        zoneNodes.forEach(c -> {
+            if (c!=csc) {
+                addresses.add(c.getClientClusterNodeAddress()+":"+c.getClientClusterNodePort());
+                hostnames.add(c.getClientClusterNodeHostname()+":"+c.getClientClusterNodePort());
+            }
+        });
+        log.debug("instructClusterJoin: New cluster node nearby members: addresses={}, hostnames={}", addresses, hostnames);
+
+        // Prepare cluster join commands
+        String command = String.format("%s  %s:%s:%s  start-election=%b  %s:%d  %s",
+                zone.getId(),
+                topLevelGrouping, aggregatorGrouping, lastLevelGrouping,
+                startElection,
+                csc.getClientClusterNodeAddress(),
+                csc.getClientClusterNodePort(),
+                String.join(" ", addresses));
+        /*String command =
+                zone.getId()+" "
+                +topLevelGrouping+":"+aggregatorGrouping+":"+lastLevelGrouping+" "
+                +startElection+" "
+                +csc.getClientClusterNodeHostname()+":"+csc.getClientClusterNodePort()+" "
+                +String.join(" ", hostnames);*/
+
+        // Send cluster join command
+        log.debug("instructClusterJoin: Client {} @ {} joins cluster: CLUSTER-JOIN {}", csc.getId(), csc.getClientIpAddress(), command);
+        csc.sendCommand("CLUSTER-JOIN "+command);
+    }
+
+    void instructClusterLeave(ClientShellCommand csc, IClusterZone zone) {
+        // Send cluster leave command
+        log.debug("instructClusterLeave: Client {} @ {} leaves cluster: CLUSTER-LEAVE", csc.getId(), csc.getClientIpAddress());
+        try {
+            csc.sendCommand("CLUSTER-LEAVE");
+        } catch (Exception e) {
+            // Channel has probably already been closed
+            log.warn("instructClusterLeave: EXCEPTION: ", e);
+        }
+    }
+
+    void electAggregator(IClusterZone zone) {
+        sendCommandToZone("CLUSTER-EXEC broker elect", zone.getNodes());
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/cluster/DefaultZoneManagementStrategy.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator.cluster;
+
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * The default Zone Management Strategy used when 'zone-management-strategy-class' property is not set.
+ * It groups clients based on domain name, or last byte of IP Address. If neither is available it assigns client
+ * in a new zone identified by a random UUID.
+ * The first client to join a zone will be instructed to start cluster and become aggregator.
+ * Subsequent clients will just join the cluster.
+ */
+@Slf4j
+public class DefaultZoneManagementStrategy implements IZoneManagementStrategy {
+    @Override
+    public void notPreregisteredNode(ClientShellCommand csc) {
+        log.warn("DefaultZoneManagementStrategy: Unexpected node connected: {} @ {}", csc.getId(), csc.getClientIpAddress());
+    }
+
+    @Override
+    public void alreadyRegisteredNode(ClientShellCommand csc) {
+        log.warn("DefaultZoneManagementStrategy: Node connection from an already registered IP address: {} @ {}", csc.getId(), csc.getClientIpAddress());
+    }
+
+    @Override
+    public synchronized void nodeAdded(ClientShellCommand csc, ClusteringCoordinator coordinator, IClusterZone zone) {
+        // Instruct new node to join cluster
+        log.info("DefaultZoneManagementStrategy: Node to join cluster: client={}, zone={}", csc.getId(), zone.getId());
+        joinToCluster(csc, coordinator, zone);
+    }
+
+    private void joinToCluster(ClientShellCommand csc, ClusteringCoordinator coordinator, IClusterZone zone) {
+        coordinator.sendClusterKey(csc, zone);
+        coordinator.instructClusterJoin(csc, zone, true);
+
+        coordinator.sleep(1000);
+        csc.sendCommand("CLUSTER-EXEC broker list");
+        //coordinator.sleep(1000);
+        //csc.sendCommand("CLUSTER-TEST");
+    }
+
+    @Override
+    public synchronized void nodeRemoved(ClientShellCommand csc, ClusteringCoordinator coordinator, IClusterZone zone) {
+        // Instruct node to leave cluster
+        log.info("DefaultZoneManagementStrategy: Node to leave cluster: client={}, zone={}", csc.getId(), zone.getId());
+        coordinator.instructClusterLeave(csc, zone);
+    }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/cluster/IClusterZone.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator.cluster;
+
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+import gr.iccs.imu.ems.util.ClientConfiguration;
+import lombok.NonNull;
+
+import java.io.File;
+import java.util.List;
+import java.util.Set;
+
+public interface IClusterZone {
+    String getId();
+    void addNode(@NonNull ClientShellCommand csc);
+    void removeNode(@NonNull ClientShellCommand csc);
+    Set<String> getNodeAddresses();
+    List<ClientShellCommand> getNodes();
+    ClientShellCommand getNodeByAddress(String address);
+
+    List<NodeRegistryEntry> findAggregatorCapableNodes();
+
+    void addNodeWithoutClient(@NonNull NodeRegistryEntry entry);
+    void removeNodeWithoutClient(@NonNull NodeRegistryEntry entry);
+    Set<String> getNodeWithoutClientAddresses();
+    List<NodeRegistryEntry> getNodesWithoutClient();
+    NodeRegistryEntry getNodeWithoutClientByAddress(String address);
+
+    ClientConfiguration getClientConfiguration();
+    ClientConfiguration sendClientConfigurationToZoneClients();
+
+    File getClusterKeystoreFile();
+    String getClusterKeystoreType();
+    String getClusterKeystorePassword();
+    String getClusterKeystoreBase64();
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/cluster/IClusterZoneDetector.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator.cluster;
+
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+
+import java.util.Map;
+
+public interface IClusterZoneDetector {
+    String getZoneIdFor(ClientShellCommand csc);
+    String getZoneIdFor(NodeRegistryEntry entry);
+    void setProperties(Map<String,String> zoneConfig);
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/coordinator/cluster/IZoneManagementStrategy.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.coordinator.cluster;
+
+import gr.iccs.imu.ems.baguette.server.ClientShellCommand;
+import gr.iccs.imu.ems.baguette.server.NodeRegistryEntry;
+
+import java.util.Map;
+
+public interface IZoneManagementStrategy {
+    default boolean allowAlreadyPreregisteredNode(Map<String,Object> nodeInfo) { return true; }
+    default boolean allowAlreadyPreregisteredNode(NodeRegistryEntry entry) { return true; }
+    default boolean allowAlreadyRegisteredNode(ClientShellCommand csc) { return true; }
+    default boolean allowAlreadyRegisteredNode(NodeRegistryEntry entry) { return true; }
+    default boolean allowNotPreregisteredNode(ClientShellCommand csc) { return true; }
+    default boolean allowNotPreregisteredNode(NodeRegistryEntry entry) { return true; }
+    default void notPreregisteredNode(ClientShellCommand csc) { }
+    default void notPreregisteredNode(NodeRegistryEntry entry) { }
+    default void alreadyRegisteredNode(ClientShellCommand csc) { }
+    default void alreadyRegisteredNode(NodeRegistryEntry entry) { }
+
+    default void nodeAdded(ClientShellCommand csc, ClusteringCoordinator coordinator, IClusterZone zoneInfo) { }
+    default void nodeRemoved(ClientShellCommand csc, ClusteringCoordinator coordinator, IClusterZone zoneInfo) { }
+}

ems-core/baguette-server/src/main/java/gr/iccs/imu/ems/baguette/server/properties/BaguetteServerProperties.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+ * Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+ * If a copy of the MPL was not distributed with this file, you can obtain one at
+ * https://www.mozilla.org/en-US/MPL/2.0/
+ */
+
+package gr.iccs.imu.ems.baguette.server.properties;
+
+import gr.iccs.imu.ems.baguette.server.ServerCoordinator;
+import gr.iccs.imu.ems.util.CredentialsMap;
+import gr.iccs.imu.ems.util.EmsConstant;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.validation.annotation.Validated;
+
+import javax.validation.constraints.Max;
+import javax.validation.constraints.Min;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Slf4j
+@Data
+@Validated
+@Configuration
+@ConfigurationProperties(prefix = EmsConstant.EMS_PROPERTIES_PREFIX + "baguette.server")
+public class BaguetteServerProperties implements InitializingBean {
+    public void afterPropertiesSet() {
+        log.debug("BaguetteServerProperties: {}", this);
+        checkConfig();
+    }
+
+    private void checkConfig() {
+        // Check that either coordinator class or id is provided
+        if (coordinatorClass==null && (coordinatorId==null || coordinatorId.size()==0))
+            throw new IllegalArgumentException("Either coordinator class or coordinator id must be provided");
+        if (coordinatorId!=null && coordinatorId.size()>0) {
+            coordinatorId.forEach(id -> {
+                CoordinatorConfig cc = getCoordinatorConfig().get(id);
+                if (cc==null)
+                    throw new IllegalArgumentException("Not found coordinator configuration with id: "+id);
+                if (cc.getCoordinatorClass()==null)
+                    throw new IllegalArgumentException("No coordinator class in configuration with id: "+id);
+            });
+        }
+    }
+
+    private Class<ServerCoordinator> coordinatorClass;
+    private Map<String,String> coordinatorParameters = new HashMap<>();
+
+    private List<String> coordinatorId;
+    private Map<String, CoordinatorConfig> coordinatorConfig = new HashMap<>();
+
+    @Min(-1)
+    private long registrationWindow = 30000;
+    @Min(-1)
+    private int numberOfInstances = -1;
+    @Min(-1)
+    private int NumberOfSegments = -1;
+
+    private String address;
+    public String getServerAddress() { return address; }
+    private boolean resolveHostname = true;
+
+    @Min(value = 1, message = "Valid server ports are between 1 and 65535. Please prefer ports higher than 1023.")
+    @Max(value = 65535, message = "Valid server ports are between 1 and 65535. Please prefer ports higher than 1023.")
+    private int port = 2222;
+    public int getServerPort() { return port; }
+
+    private String keyFile = "hostkey.pem";
+    public String getServerKeyFile() { return keyFile; }
+
+    private boolean heartbeatEnabled;
+    @Min(-1)
+    private long heartbeatPeriod = 60000;
+
+    private boolean clientAddressOverrideAllowed;
+    private String clientIdFormat;
+    private String clientIdFormatEscape = "~";
+
+    private final CredentialsMap credentials = new CredentialsMap();
+
+    @Data
+    public static class CoordinatorConfig {
+        private Class<ServerCoordinator> coordinatorClass;
+        private Map<String,String> parameters;
+    }
+}

ems-core/bin/client.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless 
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at 
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+BASEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )
+
+#JAVA_OPTS=-Djavax.net.ssl.trustStore=./broker-truststore.p12\ -Djavax.net.ssl.trustStorePassword=melodic\ -Djavax.net.ssl.trustStoreType=pkcs12
+# -Djavax.net.debug=all
+# -Djavax.net.debug=ssl,handshake,record
+
+java $JAVA_OPTS -jar ${BASEDIR}/public_resources/resources/broker-client.jar $*

ems-core/bin/cp2cdo.bat

@@ -0,0 +1,27 @@
+@echo off
+::
+:: Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+::
+:: This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+:: Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+:: If a copy of the MPL was not distributed with this file, you can obtain one at
+:: https://www.mozilla.org/en-US/MPL/2.0/
+::
+
+setlocal
+set PWD=%cd%
+cd %~dp0..
+set BASEDIR=%cd%
+IF NOT DEFINED EMS_CONFIG_DIR set EMS_CONFIG_DIR=%BASEDIR%\config-files
+IF NOT DEFINED PAASAGE_CONFIG_DIR set PAASAGE_CONFIG_DIR=%BASEDIR%\config-files
+
+:: Copy dependencies if missing
+if exist pom.xml (
+    if not exist %BASEDIR%\control-service\target\dependency cmd /C "cd control-service && mvn dependency:copy-dependencies"
+)
+
+java -classpath %BASEDIR%/control-service/target/classes;%BASEDIR%/control-service/target/dependency/* gr.iccs.imu.ems.control.util.CpModelHelper %*
+rem Usage: cp2cdo <file> <cdo-resource>
+
+cd %PWD%
+endlocal

ems-core/bin/cp2cdo.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+PREVWORKDIR=`pwd`
+BASEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )
+cd ${BASEDIR}
+if [[ -z $EMS_CONFIG_DIR ]]; then EMS_CONFIG_DIR=${BASEDIR}/config-files; export EMS_CONFIG_DIR; fi
+if [[ -z $PAASAGE_CONFIG_DIR ]]; then PAASAGE_CONFIG_DIR=${BASEDIR}/config-files; export PAASAGE_CONFIG_DIR; fi
+
+# Copy dependencies if missing
+if [[ -f ${BASEDIR}/control-service/pom.xml ]]; then
+    if [[ ! -d ${BASEDIR}/control-service/target/dependency ]]; then
+        cd ${BASEDIR}/control-service
+        mvn dependency:copy-dependencies
+        cd ${BASEDIR}
+    fi
+fi
+
+java -classpath "control-service/target/classes;control-service/target/dependency/*" gr.iccs.imu.ems.control.util.CpModelHelper $*
+# Usage: cp2cdo <file> <cdo-resource>
+
+cd ${PREVWORKDIR}

ems-core/bin/detect.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+#Required utilities: grep,uniq,tr,cat,cut,uname. For commented commands, awk and wc.
+
+BUSYBOX_PREFIX="${args[0]}"
+
+#TMP_NUM_CPUS=$($BUSYBOX_PREFIX grep 'physical id' /proc/cpuinfo | $BUSYBOX_PREFIX sort | $BUSYBOX_PREFIX uniq | $BUSYBOX_PREFIX wc -l)
+#TMP_NUM_CORES=$($BUSYBOX_PREFIX grep 'cpu cores' /proc/cpuinfo | $BUSYBOX_PREFIX sort | $BUSYBOX_PREFIX uniq | $BUSYBOX_PREFIX cut -d ' ' -f 3)
+#TMP_NUM_PROCESSORS=$($BUSYBOX_PREFIX grep -c ^processor /proc/cpuinfo)
+TMP_RAM_TOTAL_KB=$($BUSYBOX_PREFIX cat /proc/meminfo | $BUSYBOX_PREFIX grep MemTotal | $BUSYBOX_PREFIX tr -s ' ' | $BUSYBOX_PREFIX cut -d ' ' -f 2)
+TMP_RAM_AVAILABLE_KB=$($BUSYBOX_PREFIX cat /proc/meminfo | $BUSYBOX_PREFIX grep MemAvailable | $BUSYBOX_PREFIX tr -s ' ' | $BUSYBOX_PREFIX cut -d ' ' -f 2)
+TMP_RAM_FREE_KB=$($BUSYBOX_PREFIX cat /proc/meminfo | $BUSYBOX_PREFIX grep MemFree | $BUSYBOX_PREFIX tr -s ' ' | $BUSYBOX_PREFIX cut -d ' ' -f 2)
+TMP_DISK_TOTAL_KB=$($BUSYBOX_PREFIX df -k | $BUSYBOX_PREFIX grep /$ | $BUSYBOX_PREFIX tr -s ' ' | $BUSYBOX_PREFIX cut -d ' ' -f 2)
+TMP_DISK_FREE_KB=$($BUSYBOX_PREFIX df -k | $BUSYBOX_PREFIX grep /$ | $BUSYBOX_PREFIX tr -s ' ' | $BUSYBOX_PREFIX cut -d ' ' -f 4)
+TMP_ARCHITECTURE=$($BUSYBOX_PREFIX uname -m) #x86_64 GNU/Linux indicates that you've a 64bit Linux kernel running. If you see i386/i486/i586/i686 it is a 32-bit architecture. armv7l, armv8 etc. signal a 32-bit arm version of the library while aarch64 indicates a 64-bit arm version of the library
+TMP_KERNEL=$($BUSYBOX_PREFIX uname -s)
+TMP_KERNEL_RELEASE=$($BUSYBOX_PREFIX uname -r)
+
+#NUM_CORES_ALT=$BUSYBOX_PREFIX grep ^cpu\\scores /proc/cpuinfo | $BUSYBOX_PREFIX uniq |  $BUSYBOX_PREFIX awk '{print $4}'
+#CAN_RUN_x64 = grep flags /proc/cpuinfo | grep " lm" | wc | tr -s ' ' | cut -d ' ' -f 2 #1 means that it can run x64, 0 that it can't, although that possibly also depends on the kernel installed
+
+TMP_NUM_CPUS=$(lscpu -p | grep -v '#' | cut -d ',' -f 3 | sort -u | wc -l)
+TMP_NUM_CORES=$(lscpu -p | grep -v '#' | cut -d ',' -f 2 | sort -u | wc -l)
+TMP_NUM_PROCESSORS=$(lscpu -p | grep -v '#' | cut -d ',' -f 1 | sort -u | wc -l)
+TMP_RAM_USED_KB=$(echo $TMP_RAM_TOTAL_KB $TMP_RAM_FREE_KB | awk '{print $1 - $2}')
+TMP_RAM_UTILIZATION=$(echo $TMP_RAM_USED_KB $TMP_RAM_TOTAL_KB | awk '{print 100 * $1 / $2}')
+TMP_DISK_USED_KB=$(echo $TMP_DISK_TOTAL_KB $TMP_DISK_FREE_KB | awk '{print $1 - $2}')
+TMP_DISK_UTILIZATION=$(echo $TMP_DISK_USED_KB $TMP_DISK_TOTAL_KB | awk '{print 100 * $1 / $2}')
+
+
+echo CPU_SOCKETS=$TMP_NUM_CPUS
+echo CPU_CORES=$TMP_NUM_CORES
+echo CPU_PROCESSORS=$TMP_NUM_PROCESSORS
+echo RAM_TOTAL_KB=$TMP_RAM_TOTAL_KB
+echo RAM_AVAILABLE_KB=$TMP_RAM_AVAILABLE_KB
+echo RAM_FREE_KB=$TMP_RAM_FREE_KB
+echo RAM_USED_KB=$TMP_RAM_USED_KB
+echo RAM_UTILIZATION=$TMP_RAM_UTILIZATION
+echo DISK_TOTAL_KB=$TMP_DISK_TOTAL_KB
+echo DISK_FREE_KB=$TMP_DISK_FREE_KB
+echo DISK_USED_KB=$TMP_DISK_USED_KB
+echo DISK_UTILIZATION=$TMP_DISK_UTILIZATION
+echo OS_ARCHITECTURE=$TMP_ARCHITECTURE
+echo OS_KERNEL=$TMP_KERNEL
+echo OS_KERNEL_RELEASE=$TMP_KERNEL_RELEASE

ems-core/bin/initialize-MELODIC-keystores.sh

@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+PREVWORKDIR=`pwd`
+BASEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+cd ${BASEDIR}
+
+if [[ -z $EMS_CONFIG_DIR ]]; then EMS_CONFIG_DIR=${BASEDIR}/config; export EMS_CONFIG_DIR; fi
+
+# Get IP addresses
+echo Resolving Public IP addresses...
+#PUBLIC_IP=`curl http://ifconfig.me 2> /dev/null`
+#PUBLIC_IP=`curl http://www.icanhazip.com 2> /dev/null`
+#PUBLIC_IP=`curl http://ipecho.net/plain 2> /dev/null`
+#PUBLIC_IP=`curl http://bot.whatismyipaddress.com 2> /dev/null`
+PUBLIC_IP=`curl https://diagnostic.opendns.com/myip 2> /dev/null`
+#PUBLIC_IP=`curl http://checkip.amazonaws.com 2> /dev/null`
+
+# or get IP address with 'hostname'
+if [[ "${PUBLIC_IP}" == "" ]]; then
+    PUBLIC_IP=`hostname --all-ip-addresses`
+    echo "PUBLIC_IP (hostname -I): $PUBLIC_IP"
+fi
+
+# or set IP address manually
+if [[ "${PUBLIC_IP}" == "" ]]; then
+    PUBLIC_IP=1.2.3.4
+    echo "PUBLIC_IP (manually): $PUBLIC_IP"
+fi
+
+# or use loopback
+if [[ "${PUBLIC_IP}" == "" ]]; then
+    PUBLIC_IP=127.0.0.1
+    echo "PUBLIC_IP (loopback): $PUBLIC_IP"
+fi
+PUBLIC_IP=`echo ${PUBLIC_IP} | sed 's/ *$//g'`
+echo PUBLIC_IP=${PUBLIC_IP}
+
+
+# Get cached IP address from previous run (if any)
+CACHED_IP_FILE=${EMS_CONFIG_DIR}/MY_IP
+touch ${CACHED_IP_FILE}
+CACHED_IP=`cat ${CACHED_IP_FILE}`
+#echo "Cached IP address=${CACHED_IP}"
+
+# Check if "Force update flag is set in command-line" (i.e. -U flag)
+if [[ "$1" == "-U" ]]; then
+    CACHED_IP="----"
+fi
+
+# Check if current and cached IP addresses match
+if [[ "${PUBLIC_IP}" == "${CACHED_IP}" ]]; then
+    echo "Current and Cached IP addresses are identical: ${PUBLIC_IP}"
+    echo "Exit without changing keystores"
+    exit 0
+fi
+# ...else store new IP address
+echo ${PUBLIC_IP} > ${CACHED_IP_FILE}
+
+
+# Prepare keystore base directory and truststore file
+KEYSTORE_BASE_DIR=${EMS_CONFIG_DIR}/certs
+TRUSTSTORE_DIR=${EMS_CONFIG_DIR}/common
+TRUSTSTORE_FILE=${TRUSTSTORE_DIR}/melodic-truststore.p12
+
+mkdir -p ${KEYSTORE_BASE_DIR}
+mkdir -p ${TRUSTSTORE_DIR}
+rm -f ${TRUSTSTORE_FILE} &> /dev/null
+
+# Keystore initialization settings
+KEY_GEN_ALG=RSA
+KEY_SIZE=2048
+START_DATE=-1d
+VALIDITY=3650
+DN_FMT="CN=%s,OU=Information Management Unit (IMU),O=Institute of Communication and Computer Systems (ICCS),L=Athens,ST=Attika,C=GR"
+if [[ "${PUBLIC_IP}" != "" ]]; then
+  PUBLIC_IP_FOR_SAN=${PUBLIC_IP// /,ip:}
+  PUBLIC_IP_FOR_SAN="ip:${PUBLIC_IP_FOR_SAN}"
+fi
+if [[ "${EXTRA_IPS_FOR_SAN}" != "" ]]; then
+  EXTRA_IPS_FOR_SAN=",${EXTRA_IPS_FOR_SAN}"
+  EXTRA_IPS_FOR_SAN=`echo ${EXTRA_IPS_FOR_SAN} | sed -e 's/,/,ip:/g'`
+  EXTRA_IPS_FOR_SAN=`echo ${EXTRA_IPS_FOR_SAN} | sed -e 's/[ \t]//g'`
+fi
+EXT_SAN_FMT="SAN=dns:%s,dns:localhost,ip:127.0.0.1,${PUBLIC_IP_FOR_SAN}${EXTRA_IPS_FOR_SAN}"
+
+KEYSTORE_TYPE=PKCS12
+KEYSTORE_PASS=melodic
+
+# Definition of 'create_keystore_for' function for the:
+# Creation of key pair and certificate for component
+function create_keystore_for() {
+    local COMPONENT=$1
+    local KEYSTORE_DIR=${KEYSTORE_BASE_DIR}/${COMPONENT}
+    local KEYSTORE_FILE=${KEYSTORE_DIR}/keystore.p12
+    local CERT_FILE=${KEYSTORE_DIR}/${COMPONENT}.crt
+    local KEY_ALIAS=${COMPONENT}
+    local DN=`printf "${DN_FMT}" "${KEY_ALIAS}" `
+    local EXT_SAN=`printf "${EXT_SAN_FMT}" "${KEY_ALIAS}" `
+
+    echo "$COMPONENT:"
+    mkdir -p ${KEYSTORE_DIR}
+
+    echo "  Generating key pair and certificate for ${COMPONENT}..."
+    rm -f ${KEYSTORE_FILE} &> /dev/null
+    keytool -genkey -keyalg ${KEY_GEN_ALG} -keysize ${KEY_SIZE} \
+            -alias ${KEY_ALIAS} \
+            -startdate ${START_DATE} -validity ${VALIDITY} \
+            -dname "${DN}" -ext "${EXT_SAN}" \
+            -keystore ${KEYSTORE_FILE} \
+            -storetype ${KEYSTORE_TYPE} -storepass ${KEYSTORE_PASS}
+
+    echo "  Exporting certificate of ${COMPONENT}..."
+    rm -rf ${CERT_FILE} &> /dev/null
+    keytool -export \
+            -alias ${KEY_ALIAS} \
+            -file ${CERT_FILE} \
+            -keystore ${KEYSTORE_FILE} \
+            -storetype ${KEYSTORE_TYPE} -storepass ${KEYSTORE_PASS}
+
+    echo "  Importing ${COMPONENT} certificate to truststore..."
+    keytool -import -noprompt \
+            -alias ${KEY_ALIAS} \
+            -file ${CERT_FILE} \
+            -keystore ${TRUSTSTORE_FILE} \
+            -storetype ${KEYSTORE_TYPE} -storepass ${KEYSTORE_PASS}
+
+    echo ""
+}
+
+# Creation of key pairs, certificates of all components and population of common truststore
+create_keystore_for "cdoserver"
+create_keystore_for "mule"
+create_keystore_for "adapter"
+create_keystore_for "generator"
+create_keystore_for "cpsolver"
+create_keystore_for "camunda"
+create_keystore_for "memcache"
+create_keystore_for "ldap"
+create_keystore_for "metasolver"
+create_keystore_for "jwtserver"
+create_keystore_for "authdb"
+create_keystore_for "authserver"
+create_keystore_for "ems"
+create_keystore_for "gui-backend"
+create_keystore_for "gui-frontend"
+#create_keystore_for "cloudiator"
+
+echo Key stores, certificate and Melodic common truststores are ready.
+cd $PREVWORKDIR

ems-core/bin/initialize-keystores.bat

@@ -0,0 +1,85 @@
+@echo off
+::
+:: Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+::
+:: This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+:: Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+:: If a copy of the MPL was not distributed with this file, you can obtain one at
+:: https://www.mozilla.org/en-US/MPL/2.0/
+::
+
+setlocal
+set PWD=%cd%
+cd %~dp0..
+set BASEDIR=%cd%
+IF NOT DEFINED EMS_CONFIG_DIR set EMS_CONFIG_DIR=%BASEDIR%\config-files
+IF NOT DEFINED PAASAGE_CONFIG_DIR set PAASAGE_CONFIG_DIR=%BASEDIR%\config-files
+
+:: Get IP addresses
+set UTIL_FILE=util-4.0.0-SNAPSHOT-jar-with-dependencies.jar
+set UTIL_PATH_0=util\target\%UTIL_FILE%
+set UTIL_PATH_1=jars\util\%UTIL_FILE%
+set UTIL_PATH_2=..\util\target\%UTIL_FILE%
+set UTIL_PATH_3=.\%UTIL_FILE%
+if exist %UTIL_PATH_0% (
+    set UTIL_JAR=%UTIL_PATH_0%
+) else (
+	if exist %UTIL_PATH_1% (
+		set UTIL_JAR=%UTIL_PATH_1%
+	) else (
+		if exist %UTIL_PATH_2% (
+			set UTIL_JAR=%UTIL_PATH_2%
+		) else (
+			if exist %UTIL_PATH_3% (
+				set UTIL_JAR=%UTIL_PATH_3%
+			) else (
+				echo ERROR: Couldn't find 'util-4.0.0-SNAPSHOT-jar-with-dependencies.jar'
+				echo ERROR: Skipping keystore initialization
+				goto the_end
+			)
+		)
+	)
+)
+::echo UTIL_JAR location: %UTIL_JAR%
+
+echo Resolving Public and Default IP addresses...
+for /f %%i in ('java -jar %UTIL_JAR% -nolog public')  do set {PUBLIC_IP}=%%i
+for /f %%i in ('java -jar %UTIL_JAR% -nolog default') do set {DEFAULT_IP}=%%i
+
+IF "%{PUBLIC_IP}%" == "null" set {PUBLIC_IP}=127.0.0.1
+IF "%{DEFAULT_IP}%" == "null" set {DEFAULT_IP}=127.0.0.1
+
+echo PUBLIC_IP=%{PUBLIC_IP}%
+echo DEFAULT_IP=%{DEFAULT_IP}%
+
+:: Keystore initialization settings
+set KEY_GEN_ALG=RSA
+set KEY_SIZE=2048
+set KEY_ALIAS=ems
+set START_DATE=-1d
+set VALIDITY=3650
+set DN=CN=ems,OU=Information Management Unit (IMU),O=Institute of Communication and Computer Systems (ICCS),L=Athens,ST=Attika,C=GR
+set EXT_SAN=SAN=dns:localhost,ip:127.0.0.1,ip:%{DEFAULT_IP}%,ip:%{PUBLIC_IP}%
+set KEYSTORE=%EMS_CONFIG_DIR%\broker-keystore.p12
+set TRUSTSTORE=%EMS_CONFIG_DIR%\broker-truststore.p12
+set CERTIFICATE=%EMS_CONFIG_DIR%\broker.crt
+set KEYSTORE_TYPE=PKCS12
+set KEYSTORE_PASS=melodic
+
+:: Keystores initialization
+echo Generating key pair and certificate...
+keytool -delete -alias %KEY_ALIAS% -keystore %KEYSTORE% -storetype %KEYSTORE_TYPE% -storepass %KEYSTORE_PASS% > nul 2>&1
+keytool -genkey -keyalg %KEY_GEN_ALG% -keysize %KEY_SIZE% -alias %KEY_ALIAS% -startdate %START_DATE% -validity %VALIDITY% -dname "%DN%" -ext "%EXT_SAN%" -keystore %KEYSTORE% -storetype %KEYSTORE_TYPE% -storepass %KEYSTORE_PASS%
+
+echo Exporting certificate to file...
+del /Q %CERTIFICATE% > nul 2>&1
+keytool -export -alias %KEY_ALIAS% -file %CERTIFICATE% -keystore %KEYSTORE% -storetype %KEYSTORE_TYPE% -storepass %KEYSTORE_PASS%
+
+echo Importing certificate to trust store...
+keytool -delete -alias %KEY_ALIAS% -keystore %TRUSTSTORE% -storetype %KEYSTORE_TYPE% -storepass %KEYSTORE_PASS% > nul 2>&1
+keytool -import -noprompt -file %CERTIFICATE% -alias %KEY_ALIAS% -keystore %TRUSTSTORE% -storetype %KEYSTORE_TYPE% -storepass %KEYSTORE_PASS%
+
+echo Key store, trust stores and certificate are ready.
+:the_end
+cd %PWD%
+endlocal

ems-core/bin/initialize-keystores.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+# Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+# If a copy of the MPL was not distributed with this file, you can obtain one at
+# https://www.mozilla.org/en-US/MPL/2.0/
+#
+
+PREVWORKDIR=`pwd`
+BASEDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )
+cd ${BASEDIR}
+if [[ -z $EMS_CONFIG_DIR ]]; then EMS_CONFIG_DIR=$BASEDIR/config-files; export EMS_CONFIG_DIR; fi
+if [[ -z $PAASAGE_CONFIG_DIR ]]; then PAASAGE_CONFIG_DIR=$BASEDIR/config-files; export PAASAGE_CONFIG_DIR; fi
+
+# Get IP addresses
+UTIL_FILE=util-4.0.0-SNAPSHOT-jar-with-dependencies.jar
+UTIL_PATH_0=util/target/${UTIL_FILE}
+UTIL_PATH_1=jars/util/${UTIL_FILE}
+UTIL_PATH_2=../util/target/${UTIL_FILE}
+UTIL_PATH_3=./${UTIL_FILE}
+if [ -f ${UTIL_PATH_0} ]; then
+	UTIL_JAR=${UTIL_PATH_0}
+elif [ -f ${UTIL_PATH_1} ]; then
+	UTIL_JAR=${UTIL_PATH_1}
+elif [ -f ${UTIL_PATH_2} ]; then
+	UTIL_JAR=${UTIL_PATH_2}
+elif [ -f ${UTIL_PATH_3} ]; then
+	UTIL_JAR=${UTIL_PATH_3}
+else
+	echo "ERROR: Couldn't find 'util-4.0.0-SNAPSHOT-jar-with-dependencies.jar'"
+	echo "ERROR: Skipping keystore initialization"
+	cd ${PREVWORKDIR}
+	exit 1
+fi
+#echo UTIL_JAR location: ${UTIL_JAR}
+
+echo Resolving Public and Default IP addresses...
+PUBLIC_IP=`java -jar ${UTIL_JAR} -nolog public`
+DEFAULT_IP=`java -jar ${UTIL_JAR} -nolog default`
+
+if [[ "${PUBLIC_IP}" == "" || "${PUBLIC_IP}" == "null" ]]; then
+    PUBLIC_IP=127.0.0.1
+fi
+if [[ "${DEFAULT_IP}" == "" || "${DEFAULT_IP}" == "null" ]]; then
+    DEFAULT_IP=127.0.0.1
+fi
+
+echo PUBLIC_IP=${PUBLIC_IP}
+echo DEFAULT_IP=${DEFAULT_IP}
+
+# Keystore initialization settings
+KEY_GEN_ALG=RSA
+KEY_SIZE=2048
+KEY_ALIAS=ems
+START_DATE=-1d
+VALIDITY=3650
+DN="CN=ems,OU=Information Management Unit (IMU),O=Institute of Communication and Computer Systems (ICCS),L=Athens,ST=Attika,C=GR"
+EXT_SAN="SAN=dns:localhost,ip:127.0.0.1,ip:${DEFAULT_IP},ip:${PUBLIC_IP}"
+KEYSTORE=${EMS_CONFIG_DIR}/broker-keystore.p12
+TRUSTSTORE=${EMS_CONFIG_DIR}/broker-truststore.p12
+CERTIFICATE=${EMS_CONFIG_DIR}/broker.crt
+KEYSTORE_TYPE=PKCS12
+KEYSTORE_PASS=melodic
+
+# Keystores initialization
+echo Generating key pair and certificate...
+keytool -delete -alias ${KEY_ALIAS} -keystore ${KEYSTORE} -storetype ${KEYSTORE_TYPE} -storepass ${KEYSTORE_PASS} &> /dev/null
+keytool -genkey -keyalg ${KEY_GEN_ALG} -keysize ${KEY_SIZE} -alias ${KEY_ALIAS} -startdate ${START_DATE} -validity ${VALIDITY} -dname "${DN}" -ext "${EXT_SAN}" -keystore ${KEYSTORE} -storetype ${KEYSTORE_TYPE} -storepass ${KEYSTORE_PASS}
+
+echo Exporting certificate to file...
+rm -rf ${CERTIFICATE} &> /dev/null
+keytool -export -alias ${KEY_ALIAS} -file ${CERTIFICATE} -keystore ${KEYSTORE} -storetype ${KEYSTORE_TYPE} -storepass ${KEYSTORE_PASS}
+
+echo Importing certificate to trust store...
+keytool -delete -alias ${KEY_ALIAS} -keystore ${TRUSTSTORE} -storetype ${KEYSTORE_TYPE} -storepass ${KEYSTORE_PASS} &> /dev/null
+keytool -import -noprompt -file ${CERTIFICATE} -alias ${KEY_ALIAS} -keystore ${TRUSTSTORE} -storetype ${KEYSTORE_TYPE} -storepass ${KEYSTORE_PASS}
+
+echo Key store, trust stores and certificate are ready.
+cd $PREVWORKDIR

ems-core/bin/jwtutil.bat

@@ -0,0 +1,33 @@
+@echo off
+::
+:: Copyright (C) 2017-2023 Institute of Communication and Computer Systems (imu.iccs.gr)
+::
+:: This Source Code Form is subject to the terms of the Mozilla Public License, v2.0, unless
+:: Esper library is used, in which case it is subject to the terms of General Public License v2.0.
+:: If a copy of the MPL was not distributed with this file, you can obtain one at
+:: https://www.mozilla.org/en-US/MPL/2.0/
+::
+
+setlocal
+set PWD=%~dp0
+cd %PWD%..
+set BASEDIR=%cd%
+IF NOT DEFINED EMS_CONFIG_DIR set EMS_CONFIG_DIR=%BASEDIR%\config-files
+IF NOT DEFINED PAASAGE_CONFIG_DIR set PAASAGE_CONFIG_DIR=%BASEDIR%\config-files
+IF NOT DEFINED JARS_DIR set JARS_DIR=%BASEDIR%\control-service\target
+
+if NOT DEFINED EMS_SECRETS_FILE set EMS_SECRETS_FILE=%EMS_CONFIG_DIR%\secrets.properties
+if NOT DEFINED EMS_CONFIG_LOCATION set EMS_CONFIG_LOCATION=optional:file:%EMS_CONFIG_DIR%\ems-server.yml,optional:file:%EMS_CONFIG_DIR%\ems-server.properties,optional:file:%EMS_CONFIG_DIR%\ems.yml,optional:file:%EMS_CONFIG_DIR%\ems.properties,optional:file:%EMS_SECRETS_FILE%
+
+:: Read JASYPT password (decrypts encrypted configuration settings)
+::set JASYPT_PASSWORD=password
+if "%JASYPT_PASSWORD%"=="" (
+    set /p JASYPT_PASSWORD="Configuration Password: "
+)
+
+java -Djasypt.encryptor.password=%JASYPT_PASSWORD% -cp %JARS_DIR%\control-service.jar -Dloader.main=jwt.util.gr.iccs.imu.ems.control.JwtTokenUtil -Dlogging.level.ROOT=WARN -Dlogging.level.gr.iccs.imu.ems.util=ERROR "-Dspring.config.location=%EMS_CONFIG_LOCATION%" org.springframework.boot.loader.PropertiesLauncher %*
+set exitcode=%ERRORLEVEL%
+
+cd %PWD%
+endlocal && SET exitcode=%exitcode%
+exit /B %exitcode%