starlingx
/
integ
Code Issues Proposed changes

Merge "Code enhancements for luks-fs-mgr"

This commit is contained in:
Zuul 2024-01-15 14:01:27 +00:00 committed by Gerrit Code Review
parent dd59d24235 151539e64b
commit 60e8c113f7
1 changed files with 223 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

356
filesystem/luks/src/encryption/luks-fs-mgr.cpp
View File

@ -1,5 +1,5 @@
/* /*
* Copyright (c) 2023 Wind River Systems, Inc. * Copyright (c) 2023-2024 Wind River Systems, Inc.
* *
* SPDX-License-Identifier: Apache-2.0 * SPDX-License-Identifier: Apache-2.0
* *
@ -13,6 +13,8 @@
#include <signal.h> #include <signal.h>
#include <sys/inotify.h> #include <sys/inotify.h>
#include <syslog.h> #include <syslog.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h> #include <unistd.h>
#include <libdaemon/daemon.h> #include <libdaemon/daemon.h>
#include <iostream> #include <iostream>
@ -20,13 +22,12 @@
#include <cstdlib> #include <cstdlib>
#include <cstring> #include <cstring>
#include <type_traits> #include <type_traits>
#include <thread> #include <atomic>
#include "PassphraseGenerator.h" #include "PassphraseGenerator.h"
#define INOTIFY_THREAD_SLEEP 15
#define CONTROLLER_0 "controller-0" #define CONTROLLER_0 "controller-0"
#define CONTROLLER_1 "controller-1" #define CONTROLLER_1 "controller-1"
#define SLEEP_DURATION 60
#define BUFFER 1024 #define BUFFER 1024
#define MAX_BUF_SIZE 4096
#define FAIL_FILE_WRITE (11) #define FAIL_FILE_WRITE (11)
#define FAIL_PID_OPEN (9) #define FAIL_PID_OPEN (9)
#define K8_PROVIDER_FILE "encryption-provider.yaml" #define K8_PROVIDER_FILE "encryption-provider.yaml"
@ -39,6 +40,7 @@ const char *defaultMountPath = "/var/luks/stx/luks_fs";
const char *createdConfigFile = "/etc/luks-fs-mgr.d/created_luks.json"; const char *createdConfigFile = "/etc/luks-fs-mgr.d/created_luks.json";
const char *luksControllerDataPath = "/var/luks/stx/luks_fs/controller/"; const char *luksControllerDataPath = "/var/luks/stx/luks_fs/controller/";
const char *pidFileName = "/var/run/luks-fs-mgr.pid"; const char *pidFileName = "/var/run/luks-fs-mgr.pid";
atomic<bool> exitFlag(false);
// Define a struct to hold configuration variables // Define a struct to hold configuration variables
struct LuksConfig { struct LuksConfig {
const char *vaultFile; const char *vaultFile;
@ -654,25 +656,17 @@ bool resizeVault(const char* vaultFile,
} }
/* *********************************************************************** /* ***********************************************************************
* *
* Name : monitorLUKSVolume * Name : isSymlink
* *
* Description: This function monitors the LUKS volume status and runs * Description: This function checks if file is symlink or not.
* in loop until there's any issue with the LUKS volume.
* *
* ************************************************************************/ * ************************************************************************/
void monitorLUKSVolume(const string& volumeName) { bool isSymlink(const char* path) {
while (1) { struct stat buf;
string statusCommand = "cryptsetup status " + volumeName + if (lstat(path, &buf) != -1) {
" 2>/dev/null"; return S_ISLNK(buf.st_mode);
int status = system(statusCommand.c_str()); }
if (status != 0) { return false;
string errorMessage = "LUKS volume is not in use. "
"Error code: " + to_string(status);
log(errorMessage, LOG_ERR);
break;
}
sleep(SLEEP_DURATION);
}
} }
/* *********************************************************************** /* ***********************************************************************
* *
@ -681,16 +675,19 @@ void monitorLUKSVolume(const string& volumeName) {
* Description: This function execute command on shell and collect output. * Description: This function execute command on shell and collect output.
* *
* ************************************************************************/ * ************************************************************************/
bool execCmd(const string &cmd, string &result) { int execCmd(const string &cmd, string &result) {
const int MAX_BUF = 256; const int MAX_BUF = 256;
char buf[MAX_BUF]; char buf[MAX_BUF];
result = ""; result = "";
int cmdStatus; int cmdStatus;
bool retVal = false; int errorCode = 0;
FILE *fstream = popen(cmd.c_str(), "r"); FILE *fstream = popen(cmd.c_str(), "r");
if (!fstream) if (!fstream) {
return retVal; log("popen error for " + cmd + " OS err no: "
+ to_string(errno), LOG_ERR);
return 1;
}
if (fstream) { if (fstream) {
while (!feof(fstream)) { while (!feof(fstream)) {
@ -702,11 +699,12 @@ bool execCmd(const string &cmd, string &result) {
if (!result.empty()) if (!result.empty())
result = result.substr(0, result.size() - 1); result = result.substr(0, result.size() - 1);
if (WIFEXITED(cmdStatus) && WEXITSTATUS(cmdStatus) == 0) { errorCode = WEXITSTATUS(cmdStatus);
retVal = true;
}
return retVal; if (WIFEXITED(cmdStatus) && errorCode == 0) {
return 0;
}
return errorCode;
} }
/* *********************************************************************** /* ***********************************************************************
* *
@ -730,16 +728,21 @@ void syncLuksVolume() {
int maxRetries = 3; int maxRetries = 3;
int retryDelay = 5; int retryDelay = 5;
size_t strFound = 0; size_t strFound = 0;
int rc = 0;
try { try {
// Get the hostname // Get the hostname
if (!execCmd("/usr/bin/hostname", hostname)) { rc = execCmd("/usr/bin/hostname", hostname);
logMsg = "Command failed: Unable to retrieve hostname: "; if (rc != 0) {
logMsg = "Command failed: Unable to retrieve hostname: Error code: "
+to_string(rc);
log(logMsg, LOG_ERR); log(logMsg, LOG_ERR);
throw std::runtime_error("Command Hostname failed."); throw std::runtime_error("Command Hostname failed.");
} }
// Check if controller is not standalone/simplex // Check if controller is not standalone/simplex
if (!execCmd(facterStandAloneCmd, output)) { rc = execCmd(facterStandAloneCmd, output);
logMsg = "Command failed: Unable to fetch FACTER standalone"; if (rc != 0) {
logMsg = "Command failed: Unable to fetch FACTER standalone. "
" Error code: "+to_string(rc);
log(logMsg, LOG_ERR); log(logMsg, LOG_ERR);
throw std::runtime_error("Command: FACTER standalone failed."); throw std::runtime_error("Command: FACTER standalone failed.");
} }
@ -749,8 +752,10 @@ void syncLuksVolume() {
} }
// Check if the controller is active // Check if the controller is active
// Use the FACTER to get the active status of controller // Use the FACTER to get the active status of controller
if (!execCmd(facterActiveCmd, output)) { rc = execCmd(facterActiveCmd, output);
logMsg = "Command failed: Unable to fetch FACTER."; if (rc != 0) {
logMsg = "Command failed: Unable to fetch active FACTER."
" Error code: " + to_string(rc);
log(logMsg, LOG_ERR); log(logMsg, LOG_ERR);
throw std::runtime_error("Command: FACTER active failed."); throw std::runtime_error("Command: FACTER active failed.");
} }
@ -775,9 +780,10 @@ void syncLuksVolume() {
"/var/luks/stx/luks_fs/controller/ rsync://"; "/var/luks/stx/luks_fs/controller/ rsync://";
rsyncCmd.append(standbyHostname); rsyncCmd.append(standbyHostname);
rsyncCmd += "/luksdata/"; rsyncCmd += "/luksdata/";
if (!execCmd(rsyncCmd, output)) { rc = execCmd(rsyncCmd, output);
if (rc != 0) {
logMsg = "rysnc failed: Command execution " logMsg = "rysnc failed: Command execution "
"failed: " + rsyncCmd; "failed: " + rsyncCmd + " Error code:"+to_string(rc);
log(logMsg, LOG_ERR); log(logMsg, LOG_ERR);
if (attempt < maxRetries) { if (attempt < maxRetries) {
log("Retrying...", LOG_INFO); log("Retrying...", LOG_INFO);
@ -790,9 +796,9 @@ void syncLuksVolume() {
} else { } else {
logMsg = "rysnc successful: " + rsyncCmd; logMsg = "rysnc successful: " + rsyncCmd;
log(logMsg, LOG_INFO); log(logMsg, LOG_INFO);
break; // exit on rysnc success break; // Exit on rysnc success
} }
} // loop ends } // Loop ends
} }
} catch (const exception &ex) { } catch (const exception &ex) {
string errorStr = "rsync failed, error: " + string(ex.what()); string errorStr = "rsync failed, error: " + string(ex.what());
@ -802,58 +808,35 @@ void syncLuksVolume() {
} }
/* *********************************************************************** /* ***********************************************************************
* *
* Name : notifyLuksVolumeChange * Name : syncLuksVolumeChange
* *
* Description: This function watches LUKS volume for any changes such as * Description: This function watches LUKS volume for any changes such as
* create/modify/delete and accordingly generates notification * create/modify/delete and accordingly generates notification
* event. * event and sync the luks volume with standby controller.
* *
* ************************************************************************/ * ************************************************************************/
void notifyLuksVolumeChange(const char* luksPath) { int syncLuksVolumeChange(const char* luksPath) {
char buffer[4096]; FILE *fp;
ssize_t totalBufferBytes; char notifyWaitCmd[MAX_BUF_SIZE] = {0};
// Sleep for 15 secs while the luks_volume mounted. char output[BUFFER] = {0};
// This is to avoid the race condition between inotifyThread
// and luksVolume creation. This sleep will give that extra bit (void)snprintf(notifyWaitCmd, (MAX_BUF_SIZE - 1), "timeout 5s "
// of time for luksVolume to be created and mounted. "inotifywait -m -r -e "
sleep(INOTIFY_THREAD_SLEEP); "create,modify,delete,attrib,move --format "
// Initialize iNotify "'%%e %%w%%f' %s 2>/dev/null", luksPath);
int inotify_fd = inotify_init(); fp = popen(notifyWaitCmd, "r");
if (inotify_fd < 0) { if (fp == NULL) {
log("inotify_init failed to initialize", LOG_ERR); log("Error opening inotifywait pipe", LOG_ERR);
close(inotify_fd); return 1;
return;
} }
// Adding luksVolumeDirectory to watch // Read inotifywait output
int wd = inotify_add_watch(inotify_fd, luksPath, IN_MODIFY while (fgets(output, sizeof(output), fp) != NULL) {
| IN_CREATE | IN_DELETE); // Initiate rsync on change detected
if (wd < 0) { syncLuksVolume();
string errMsg = "inotify_add_watch failed: Failed to add " break;
"watch for dir: " + string(luksPath);
log(errMsg, LOG_ERR);
close(inotify_fd);
return;
} }
while (true) { pclose(fp);
// Listen for notification event return 0;
// The read() method blocks until one or more alerts arrive
totalBufferBytes = read(inotify_fd, buffer, sizeof(buffer));
if (totalBufferBytes < 0) {
log("Failed to read event.", LOG_ERR);
continue;
}
for (char* ptr = buffer; ptr < buffer + totalBufferBytes;) {
struct inotify_event* event =
reinterpret_cast<struct inotify_event*>(ptr);
if (event->mask & (IN_MODIFY | IN_CREATE | IN_DELETE)) {
log("Change detected: " + string(event->name), LOG_INFO);
// Initiate rsync when change detected
syncLuksVolume();
}
ptr += sizeof(struct inotify_event) + event->len;
} // ending for loop
} // While
close(inotify_fd);
} }
/* *********************************************************************** /* ***********************************************************************
* *
@ -922,50 +905,115 @@ void luksMgrSignalHandler(int signo) {
if (signo == SIGTERM) { if (signo == SIGTERM) {
// Cleanup tasks and exit the daemon // Cleanup tasks and exit the daemon
log("luks daemon: Received SIGTERM. Exiting", LOG_INFO); log("luks daemon: Received SIGTERM. Exiting", LOG_INFO);
exit(EXIT_SUCCESS); exitFlag.store(true);
} }
} }
/* *********************************************************************** /* ***********************************************************************
* *
* Name : copyKubeProviderFile * Name : copyKubeProviderFile
* *
* Description: This function creates "/controller/etc/kubernetes/" directory * Description: This function creates "/controller/etc/kubernetes/" directory
* on the LUKS volume. Then it copies encryption-provider.yaml * on the LUKS volume. Then it copies encryption-provider.yaml
* in the directory and creates symlink for * in the directory and creates symlink for
* /etc/kubernetes/encryption-provider.yaml file. * /etc/kubernetes/encryption-provider.yaml file.
* encryption-provider.yaml is generated during bootstrap and
* copied to LUKS volume.
* This function is written specifically for the patch-back
* and upgrade case to move encryption-provider.yaml to LUKS
* volume.
* *
* ************************************************************************/ * ************************************************************************/
void copyKubeProviderFile(void) { int copyKubeProviderFile(void) {
// Create /etc/kubernetes directory int rc = 0;
string luksKubernetesDirPath = string(luksControllerDataPath) string luksKubernetesDirPath = string(luksControllerDataPath)
+ "etc/kubernetes/"; + "etc/kubernetes/";
string sourceFilePath = luksKubernetesDirPath + K8_PROVIDER_FILE; string sourceFilePath = luksKubernetesDirPath + K8_PROVIDER_FILE;
// Check if the encryption-provider.yaml file already exists // Check if the encryption-provider.yaml file already exists on LUKS
if (access(sourceFilePath.c_str(), F_OK) == 0) { if (access(sourceFilePath.c_str(), F_OK) == 0) {
return; log("encryption-provider.yaml file is already present on "
"LUKS filesystem.", LOG_INFO);
return 0;
} }
// Create controller directory on luks volume if doesnt exist.
// This directory is needed for syncing files between active-standby
// controllers.
// Note: Do not delete this directory in failure path of this function.
// This is to avoid rsync failure in standby controller.
string kubernetesDirCmd = "/usr/bin/mkdir -p " + luksKubernetesDirPath; string kubernetesDirCmd = "/usr/bin/mkdir -p " + luksKubernetesDirPath;
string outResult; string outResult;
if (!execCmd(kubernetesDirCmd, outResult)) { log("Creating dir: "+kubernetesDirCmd, LOG_INFO);
log("Command failed: unable to create /controller/etc/kubernetes/ " rc = execCmd(kubernetesDirCmd, outResult);
"directory path", LOG_ERR); if (rc != 0) {
log("Command failed: " + kubernetesDirCmd+" Error code: "
+to_string(rc), LOG_ERR);
return rc;
} }
// Copy the encryption-provider.yaml file to kubernetesFilePath
// Get the SW_Version from build.info
string swVersionCmd = "grep 'SW_VERSION' /etc/build.info | "
"cut -d'=' -f2 | tr -d '\"'";
rc = execCmd(swVersionCmd, outResult);
if (rc != 0) {
log("Command failed: "+ swVersionCmd + " Error code: "
+to_string(rc), LOG_ERR);
return rc;
}
if (outResult.empty()) {
log(swVersionCmd +
": Could not get software version from /etc/build.info", LOG_ERR);
return 1;
}
// Verify if encryption-provider.yaml file exists.
// If exists, then move to luks volume.
string platformConfigPath = "/opt/platform/config/" +outResult+
"/kubernetes/encryption-provider.yaml";
if (access(platformConfigPath.c_str(), F_OK) == 0) {
log("File: "+platformConfigPath+" exists.", LOG_INFO);
// Copy the encryption-provider.yaml file to kubernetesFilePath
string moveEncryptionFileCmd = "/usr/bin/mv " +
platformConfigPath + " " + luksKubernetesDirPath;
log("Move File: "+moveEncryptionFileCmd, LOG_INFO);
rc = execCmd(moveEncryptionFileCmd, outResult);
if (rc != 0) {
log("Command failed: " + moveEncryptionFileCmd +
" Error code: " + to_string(rc), LOG_ERR);
return rc;
}
}
// Verify if /etc/kubernetes/encryption-provider.yaml file exists.
// Note: access() does not detect symlink file.
string encryptionFilePath = "/etc/kubernetes/encryption-provider.yaml"; string encryptionFilePath = "/etc/kubernetes/encryption-provider.yaml";
string copyEncryptionFileCmd = "/usr/bin/mv " + if (access(encryptionFilePath.c_str(), F_OK) == 0) {
encryptionFilePath + " " + luksKubernetesDirPath; string delEncryptionFileCmd = "/usr/bin/rm -f " +
if (!execCmd(copyEncryptionFileCmd, outResult)) { encryptionFilePath;
log("Command failed: unable to copy " log("Delete File: "+delEncryptionFileCmd, LOG_INFO);
"encryption-provider.yaml file to luksVolume", LOG_ERR); rc = execCmd(delEncryptionFileCmd, outResult);
} if (rc != 0) {
// Create symlink for encryption-provider.yaml file log("Command failed: " + delEncryptionFileCmd +
string symLinkCmd = "/usr/bin/ln -s " + sourceFilePath + " " + " Error code: " + to_string(rc), LOG_ERR);
encryptionFilePath; return rc;
if (!execCmd(symLinkCmd, outResult)) { } // Check if symlink exists at /etc/kubernetes/
log("Command failed: unable to create " } else if (isSymlink(encryptionFilePath.c_str())) {
"symlink for encryption-provider.yaml file", LOG_ERR); log(encryptionFilePath + " already exists", LOG_INFO);
} return 0;
return; }
// Create symlink for encryption-provider.yaml file
string symLinkCmd = "/usr/bin/ln -s " + sourceFilePath + " " +
encryptionFilePath;
log("Creating symlink: "+symLinkCmd, LOG_INFO);
rc = execCmd(symLinkCmd, outResult);
if (rc != 0) {
log("Command failed: " + symLinkCmd + " Error code: "
+ to_string(rc), LOG_ERR);
return rc;
}
return rc;
} }
/* *********************************************************************** /* ***********************************************************************
* *
@ -1096,18 +1144,34 @@ int handleResize(string &passphrase, string &volName) {
log("Encrypted vault is mounted.", LOG_INFO); log("Encrypted vault is mounted.", LOG_INFO);
} else { } else {
log("Encrypted vault is already mounted.", LOG_INFO); log("Encrypted vault is already mounted.", LOG_INFO);
}
} else {
// Mount path does not exist, create filesystem and then mount
if (!createFilesystem(luksConfig.volName)) {
log("Error creating filesystem", LOG_ERR);
rc = 1;
goto cleanup;
} }
if (!mountFilesystem(luksConfig.volName, luksConfig.mountPath, } else {
defaultDirectoryPath)) { statusCommand = "cryptsetup status " +
rc = 1; string(createdLuksConfig.volName) + " 2>/dev/null";
goto cleanup; status = system(statusCommand.c_str());
if (status == 0) {
log("LUKS device is already open and in use", LOG_INFO);
if (!mountFilesystem(luksConfig.volName, luksConfig.mountPath,
defaultDirectoryPath)) {
rc = 1;
goto cleanup;
}
} else {
log("LUKS device is not open. Try opening", LOG_INFO);
if (openLUKSVolume(createdLuksConfig.vaultFile,
createdLuksConfig.volName,
passphrase.c_str())) {
if (!mountFilesystem(luksConfig.volName,
luksConfig.mountPath,
defaultDirectoryPath)) {
rc = 1;
goto cleanup;
}
} else {
log("Unable to open LUKS device.", LOG_ERR);
rc = 1;
goto cleanup;
}
} }
log("Encrypted vault is mounted.", LOG_INFO); log("Encrypted vault is mounted.", LOG_INFO);
} }
@ -1115,8 +1179,6 @@ int handleResize(string &passphrase, string &volName) {
cleanup: cleanup:
json_object_put(jsonConfig); json_object_put(jsonConfig);
json_object_put(usedAttributes); json_object_put(usedAttributes);
json_object_put(luksvolumesArray);
json_object_put(attributesObj);
return (rc); return (rc);
} }
/* *********************************************************************** /* ***********************************************************************
@ -1301,10 +1363,36 @@ int initialVolCreate(string &passphrase, string &volName) {
cleanup: cleanup:
json_object_put(jsonConfig); json_object_put(jsonConfig);
json_object_put(usedAttributes); json_object_put(usedAttributes);
json_object_put(luksvolumesArray);
json_object_put(attributesObj);
return (rc); return (rc);
} }
/* ***********************************************************************
*
* Name : monitorLUKSVolume
*
* Description: This function monitors the LUKS volume status and runs
* in loop until there's any issue with the LUKS volume.
*
* ************************************************************************/
void monitorLUKSVolume(const string& volumeName) {
log("Monitoring LUKS volume: " + volumeName, LOG_INFO);
while (!exitFlag.load()) {
string statusCommand = "cryptsetup status " + volumeName +
" 2>/dev/null";
int status = system(statusCommand.c_str());
if (status != 0) {
string errorMessage = "LUKS volume is not in use. "
"Error code: " + to_string(status);
log(errorMessage, LOG_ERR);
break;
}
int rc = syncLuksVolumeChange(luksControllerDataPath);
if (rc != 0) {
log("Sync failed. Error code: " + to_string(rc), LOG_ERR);
break;
}
}
}
int main() { int main() {
int rc = 0; int rc = 0;
int ret = daemon(0, 0); int ret = daemon(0, 0);
@ -1338,25 +1426,27 @@ int main() {
" returned an empty passphrase.", LOG_ERR); " returned an empty passphrase.", LOG_ERR);
return 1; return 1;
} }
// Create a thread for luks volume monitoring
std::thread notifyThread(notifyLuksVolumeChange, luksControllerDataPath);
if (access(createdConfigFile, F_OK) == 0) { if (access(createdConfigFile, F_OK) == 0) {
// Volume exists, check resize required or not and handle // Volume exists, check resize required or not and handle
rc = handleResize(passphrase, volName); rc = handleResize(passphrase, volName);
if (rc != 0) { if (rc != 0) {
goto cleanup; log("Volume resize failed. Error code: " + to_string(rc), LOG_ERR);
return rc;
} }
} else { } else {
rc = initialVolCreate(passphrase, volName); rc = initialVolCreate(passphrase, volName);
if (rc != 0) { if (rc != 0) {
goto cleanup; log("Initial volume creation failed. Error code: " +
to_string(rc), LOG_ERR);
return rc;
} }
copyKubeProviderFile(); }
rc = copyKubeProviderFile();
if (rc != 0) {
log("copyKubeProviderFile() failed. Error code: "
+to_string(rc), LOG_ERR);
return rc;
} }
monitorLUKSVolume(volName); monitorLUKSVolume(volName);
cleanup: return rc;
if (notifyThread.joinable()) {
notifyThread.join();
}
return (rc);
} }