shells_path is now a directory of .shell files

Matches the existing units_path pattern — shells_path points to a
directory that is scanned for .shell files rather than a single
monolithic definitions file.

.gitignore

@@ -7,3 +7,4 @@ cmake-build-release
 CMakeFiles
 CMakeLists.txt
 Makefile
+sample/logs/*

CMakeLists.txt

@@ -3,4 +3,4 @@ project(rex)
 
 set(CMAKE_CXX_STANDARD 14)
 
-add_executable(rex Rex.cpp src/json_support/jsoncpp/json.h src/json_support/jsoncpp/json-forwards.h src/json_support/jsoncpp/jsoncpp.cpp src/logger/Logger.cpp src/logger/Logger.h src/json_support/JSON.cpp src/json_support/JSON.h src/misc/helpers.cpp src/misc/helpers.h src/config/Config.cpp src/config/Config.h src/suite/Suite.cpp src/suite/Suite.h src/suite/Unit.cpp src/suite/Unit.h src/shells/shells.cpp src/shells/shells.h src/plan/Plan.cpp src/plan/Plan.h src/plan/Task.cpp src/plan/Task.h src/lcpex/liblcpex.h src/lcpex/liblcpex.cpp src/lcpex/vpty/libclpex_tty.h src/lcpex/vpty/libclpex_tty.cpp src/lcpex/Contexts.h src/lcpex/Contexts.cpp src/lcpex/helpers.h src/lcpex/string_expansion/string_expansion.h src/lcpex/string_expansion/string_expansion.cpp src/lcpex/vpty/pty_fork_mod/pty_fork.h src/lcpex/vpty/pty_fork_mod/pty_fork.cpp src/lcpex/vpty/pty_fork_mod/pty_master_open.h src/lcpex/vpty/pty_fork_mod/pty_master_open.cpp src/lcpex/vpty/pty_fork_mod/tty_functions.h src/lcpex/vpty/pty_fork_mod/tty_functions.cpp )
+add_executable(rex Rex.cpp src/json_support/jsoncpp/json.h src/json_support/jsoncpp/json-forwards.h src/json_support/jsoncpp/jsoncpp.cpp src/logger/Logger.cpp src/logger/Logger.h src/json_support/JSON.cpp src/json_support/JSON.h src/misc/helpers.cpp src/misc/helpers.h src/config/Config.cpp src/config/Config.h src/suite/Suite.cpp src/suite/Suite.h src/suite/Unit.cpp src/suite/Unit.h src/shells/shells.cpp src/shells/shells.h src/plan/Plan.cpp src/plan/Plan.h src/plan/Task.cpp src/plan/Task.h src/lcpex/helpers.h src/lcpex/helpers.cpp src/lcpex/liblcpex.h src/lcpex/liblcpex.cpp src/lcpex/vpty/libclpex_tty.h src/lcpex/vpty/libclpex_tty.cpp src/lcpex/Contexts.h src/lcpex/Contexts.cpp src/lcpex/helpers.h src/lcpex/string_expansion/string_expansion.h src/lcpex/string_expansion/string_expansion.cpp src/lcpex/vpty/pty_fork_mod/pty_fork.h src/lcpex/vpty/pty_fork_mod/pty_fork.cpp src/lcpex/vpty/pty_fork_mod/pty_master_open.h src/lcpex/vpty/pty_fork_mod/pty_master_open.cpp src/lcpex/vpty/pty_fork_mod/tty_functions.h src/lcpex/vpty/pty_fork_mod/tty_functions.cpp )

Rex.cpp

@@ -15,12 +15,15 @@
 */
 
 #include <iostream>
+#include <string>
+#include <cstring>
 #include <unistd.h>
 #include <getopt.h>
 #include "src/logger/Logger.h"
 #include "src/config/Config.h"
 #include "src/suite/Suite.h"
 #include "src/plan/Plan.h"
+#include "src/misc/helpers.h"
 
 void version_info()
 {
@@ -162,6 +165,8 @@ int main( int argc, char * argv[] )
     interpolate( config_path );
     interpolate( plan_path );
 
+    plan_path = get_absolute_path( plan_path );
+
     // default logging level
     int L_LEVEL = E_INFO;
 

sample/components/independent_test_1.bash

@@ -1,5 +1,8 @@
-#!/bin/bash
+#!/usr/bin/bash
-echo "This is an independent test.  It does not depend on other tests."
+#
-echo "Environment file check: TEST_VAR from environment file is set to: $TEST_VAR"
+echo test from script
+#/usr/bin/dialog --title "This should be one argument" --inputbox "Enter your name:" 0 0
 
-exit $?
+
+
+env

sample/environments/rex.variables

@@ -1,11 +1,8 @@
 #!/usr/bin/bash
 
-
-
-
-
 set -a
 
 echo "variables file says hello and set a variable named TEST_VAR"
 TEST_VAR="999"
+TERM=xterm
 

sample/logs/2023-02-15_18:27:40.stderr.log

@@ -1,2 +0,0 @@
-lcpex: setgid failed: Operation not permitted
-lcpex: Aborting: Setting GID failed: bagira/bagira

sample/logs/2023-02-15_18:29:35.stderr.log

@@ -1 +0,0 @@
-/usr/bin/bash: line 1: /home/phanes/development/internal/rex-rewrite/sample/environments/rex.variables: Permission denied

sample/logs/2023-02-15_18:30:48.stderr.log

@@ -1 +0,0 @@
-/usr/bin/bash: line 1: /home/phanes/development/internal/rex-rewrite/sample/environments/rex.variables: Permission denied

sample/plans/test.plan

@@ -1,5 +1,6 @@
 {
   "plan": [
-    { "name": "independent test 1", "dependencies":  [ null ] }
+    { "name": "independent test 1", "dependencies":  [ null ] },
+    { "name": "independent test 2", "dependencies":  [ null ] }
   ]
 }

sample/rex.config

@@ -1,9 +1,9 @@
 {
   "config": {
-	"project_root": 	"/home/phanes/development/internal/rex-rewrite/sample",
+	"project_root": 	"$HOME/development/internal/rex/sample",
   	"units_path":		"units/",
   	"logs_path": 		"logs/",
-	"shells_path":		"shells/shells.definitions",
+	"shells_path":		"shells/",
   	"config_version": "5"
   }
 }

sample/shells/bash.shell

@@ -5,12 +5,6 @@
 			"path": 		"/usr/bin/bash",
 			"execution_arg": 	"-c",
 			"source_cmd":		"source"
-		},
-		{
-			"name":			"ksh",
-			"path":			"/usr/bin/ksh",
-			"execution_arg":	"-c",
-			"source_cmd":		"source"
 		}
 	]
 }

sample/shells/ksh.shell

@@ -0,0 +1,10 @@
+{
+	"shells":	[
+		{
+			"name":			"ksh",
+			"path":			"/usr/bin/ksh",
+			"execution_arg":	"-c",
+			"source_cmd":		"source"
+		}
+	]
+}

sample/units/new.units

@@ -13,8 +13,26 @@
         "active":       		true,
         "required":     		true,
 	"set_user_context": 		true,
-        "user":         		"bagira",
+        "user":         		"$USER",
-        "group":        		"bagira",
+        "group":        		"$USER",
+	"supply_environment":		true,
+        "environment":  		"environments/rex.variables"
+    },
+    {
+        "name":         		"independent test 2",
+        "target":       		"components/independent_test_1.bash --ls -s --arg",
+	"is_shell_command":		true,
+	"shell_definition":		"bash",
+	"force_pty":			true,
+	"set_working_directory":	false,
+	"working_directory":		"",
+        "rectify":      		false,
+        "rectifier":    		"echo rectifier executed",
+        "active":       		true,
+        "required":     		true,
+	"set_user_context": 		true,
+        "user":         		"$USER",
+        "group":        		"$USER",
 	"supply_environment":		true,
         "environment":  		"environments/rex.variables"
     }

src/config/Config.cpp

@@ -72,7 +72,7 @@ void Conf::set_object_s(std::string keyname, std::string & object_member, std::s
     if ( this->get_string(jval_s, keyname) != 0) {
         throw ConfigLoadException( "'" + keyname + "' string is not set in the config file supplied: " + filename );
     } else {
-        interpolate(jval_s);
+        //interpolate(jval_s);
         object_member = jval_s;
     }
     this->slog.log_task( E_DEBUG, "SET_PROPERTY", "'" + keyname + "': " + object_member );
@@ -101,7 +101,7 @@ void Conf::set_object_s_derivedpath(std::string keyname, std::string & object_me
     if ( this->get_string(jval_s, keyname) != 0) {
         throw ConfigLoadException( "'" + keyname + "' string is not set in the config file supplied: " + filename );
     } else {
-        interpolate(jval_s);
+        //interpolate(jval_s);
         object_member = prepend_project_root( jval_s );
     }
     this->slog.log_task( E_DEBUG, "SET_PROPERTY", "'" + keyname + "': " + object_member );
@@ -134,6 +134,12 @@ void Conf::set_object_b(std::string keyname, bool & object_member, std::string f
     this->slog.log_task( E_DEBUG, "SET_PROPERTY", "'" + keyname + "' " + std::to_string(object_member));
 }
 
+void removeTrailingSlash(std::string &str) {
+    if (!str.empty() && str.back() == '/') {
+        str.pop_back();
+    }
+}
+
 /**
  * @brief Prepend the project root to a relative path
  *
@@ -147,32 +153,11 @@ void Conf::set_object_b(std::string keyname, bool & object_member, std::string f
  */
 std::string Conf::prepend_project_root( std::string relative_path)
 {
+    removeTrailingSlash(relative_path);
     return this->project_root + "/" + relative_path;
 }
 
-/**
- * @brief Get the absolute path from a relative path
- *
- * This function takes a relative path and returns the corresponding absolute path.
- * The absolute path is obtained by calling the `realpath` function.
- * If the `realpath` function returns a null pointer, an error message is printed to the standard error stream and an empty string is returned.
- *
- * @param relative_path The relative path to be converted to an absolute path
- *
- * @return The absolute path corresponding to the relative path
- */
-std::string get_absolute_path(const std::string &relative_path)
-{
-    char resolved_path[1024];
-    memset(resolved_path, 0, sizeof(resolved_path));
 
-    if( realpath( relative_path.c_str(), resolved_path) == nullptr ) {
-        std::cerr << "Error resolving path: " << relative_path << std::endl;
-        return "";
-    }
-
-    return std::string(resolved_path);
-}
 
 /**
  * @brief Check if a path exists
@@ -212,26 +197,44 @@ void Conf::checkPathExists( std::string keyname, const std::string &path ) {
  */
 void Conf::load_shells() {
     this->slog.log_task( E_DEBUG, "SHELLS", "Loading shells..." );
-    try {
-        // load the test file.
-        this->load_json_file( this->shell_definitions_path );
-    } catch (std::exception& e) {
-        this->slog.log_task( E_FATAL, "SHELLS", "Unable to load shell definition file: '" + this->shell_definitions_path + "'. Error: " + e.what());
-        throw ConfigLoadException("Parsing error in shell definitions file.");
-    }
-    Json::Value jbuff;
 
-    if ( this-> get_serialized( jbuff, "shells" ) != 0 ) {
+    std::vector<std::string> shell_files;
-        this->slog.log_task( E_FATAL, "SHELLS", "Parsing error: '" + this->shell_definitions_path + "'. Error: 'shells' key not found." );
-        throw ConfigLoadException("Parsing error in shell definitions file.");
-    }
 
-    Shell tmp_S = Shell( this->LOG_LEVEL );
+    if ( is_dir( this->shell_definitions_path ) )
-    for ( int index = 0; index < jbuff.size(); index++ )
     {
-        tmp_S.load_root( jbuff[index] );
+        get_shells_from_dir( &shell_files, this->shell_definitions_path );
-        this->shells.push_back( tmp_S );
+    }
-        this->slog.log_task( E_DEBUG, "SHELLS", "Loaded shell: '" + tmp_S.name + "'" );
+
+    if ( is_file( this->shell_definitions_path ) )
+    {
+        shell_files.push_back( this->shell_definitions_path );
+    }
+
+    this->slog.log_task( E_INFO, "SHELLS", "Shell files found: " + std::to_string( shell_files.size() ) );
+
+    for ( int i = 0; i < shell_files.size(); i++ )
+    {
+        try {
+            this->load_json_file( shell_files[i] );
+        } catch (std::exception& e) {
+            this->slog.log_task( E_FATAL, "SHELLS", "Unable to load shell definition file: '" + shell_files[i] + "'. Error: " + e.what());
+            throw ConfigLoadException("Parsing error in shell definitions file.");
+        }
+
+        Json::Value jbuff;
+
+        if ( this->get_serialized( jbuff, "shells" ) != 0 ) {
+            this->slog.log_task( E_FATAL, "SHELLS", "Parsing error: '" + shell_files[i] + "'. Error: 'shells' key not found." );
+            throw ConfigLoadException("Parsing error in shell definitions file.");
+        }
+
+        Shell tmp_S = Shell( this->LOG_LEVEL );
+        for ( int index = 0; index < jbuff.size(); index++ )
+        {
+            tmp_S.load_root( jbuff[index] );
+            this->shells.push_back( tmp_S );
+            this->slog.log_task( E_DEBUG, "SHELLS", "Loaded shell: '" + tmp_S.name + "' (" + tmp_S.path + ")" );
+        }
     }
 }
 
@@ -258,7 +261,6 @@ Shell Conf::get_shell_by_name( std::string name ) {
     throw ConfigLoadException("The shell specified ('" + name + "') is not defined in the shell definitions file.");
 }
 
-
 /**
  * @class Conf
  * @brief Loads the configuration for the application
@@ -276,9 +278,9 @@ Shell Conf::get_shell_by_name( std::string name ) {
 Conf::Conf(std::string filename, int LOG_LEVEL ): JSON_Loader(LOG_LEVEL ), slog(LOG_LEVEL, "_conf_" )
 {
     this->LOG_LEVEL = LOG_LEVEL;
-    this->slog.log_task( E_DEBUG, "LOAD", "Loading configuration file: " + filename );
 
     interpolate( filename );
+    this->slog.log_task( E_DEBUG, "LOAD", "Loading configuration file: " + filename );
 
     try {
         // load the test file.
@@ -287,6 +289,7 @@ Conf::Conf(std::string filename, int LOG_LEVEL ): JSON_Loader(LOG_LEVEL ), slog(
         this->slog.log( E_FATAL, "Unable to load configuration file: '" + filename + "'. Error: " + e.what());
         throw ConfigLoadException("Parsing error in configuration file.");
     }
+
     Json::Value jbuff;
 
     if ( this->get_serialized( jbuff, "config" ) != 0) {
@@ -297,20 +300,27 @@ Conf::Conf(std::string filename, int LOG_LEVEL ): JSON_Loader(LOG_LEVEL ), slog(
         this->json_root = jbuff;
     }
 
-    set_object_s(               "project_root",     this->project_root,           filename );
+    set_object_s( "project_root",     this->project_root,           filename );
+    interpolate( this->project_root );
+
     // convert to an absolute path after all the interpolation is done.
     this->project_root = get_absolute_path( this->project_root );
 
+    set_object_s( "logs_path",        this->logs_path,              filename );
+    interpolate( this->logs_path );
+
     // all other paths are relative to project_root
-    set_object_s_derivedpath(   "units_path",       this->units_path,             filename );
+    set_object_s_derivedpath( "units_path",       this->units_path,             filename );
-    set_object_s_derivedpath(   "logs_path",        this->logs_path,              filename );
+    interpolate( this->units_path );
-    set_object_s_derivedpath(   "shells_path",      this->shell_definitions_path, filename );
+
+    set_object_s_derivedpath( "shells_path",      this->shell_definitions_path, filename );
+    interpolate( this->shell_definitions_path );
 
     // ensure these paths exists, with exception to the logs_path, which will be created at runtime
     this->slog.log_task( E_DEBUG, "SANITY_CHECKS", "Checking for sanity..." );
-    checkPathExists(            "project_root",     this->project_root );
+    checkPathExists( "project_root",     this->project_root );
-    checkPathExists(            "units_path",       this->units_path );
+    checkPathExists( "units_path",       this->units_path );
-    checkPathExists(            "shells_path",      this->shell_definitions_path );
+    checkPathExists( "shells_path",      this->shell_definitions_path );
 
     // shells are scoped beyond plan so they need to be considered part of config
     load_shells();

src/lcpex/helpers.cpp

@@ -0,0 +1,15 @@
+#include "helpers.h"
+
+ssize_t write_all(int fd, const void *buf, size_t count) {
+    const char *p = (const char *)buf;
+    while (count > 0) {
+        ssize_t written = write(fd, p, count);
+        if (written == -1) {
+            if (errno == EINTR || errno == EAGAIN) continue; // Retry
+            return -1; // Other errors
+        }
+        count -= written;
+        p += written;
+    }
+    return 0;
+}

src/lcpex/helpers.h

@@ -1,6 +1,9 @@
 #ifndef LCPEX_HELPERS_H
 #define LCPEX_HELPERS_H
 
+#include <unistd.h>
+#include "errno.h"
+
 // helper for sanity
 enum PIPE_ENDS {
     READ_END = 0,
@@ -16,6 +19,6 @@ enum CHILD_PIPE_NAMES {
 
 #define BUFFER_SIZE 1024
 
-
+ssize_t write_all(int fd, const void *buf, size_t count);
 
 #endif //LCPEX_HELPERS_H

src/lcpex/liblcpex.cpp

@@ -1,5 +1,6 @@
 #include "liblcpex.h"
 
+
 std::string prefix_generator(
         std::string command,
         bool is_shell_command,
@@ -16,7 +17,8 @@ std::string prefix_generator(
         prefix = shell_path;
 
         // if the shell takes an argument to execute a command, add it enclosed in quotes
-        if (shell_execution_arg != "") {
+        if ( shell_execution_arg != "" )
+        {
             // add the execution arg
             prefix += " " + shell_execution_arg + " ";
         } else {
@@ -35,15 +37,15 @@ std::string prefix_generator(
         // it's not a shell command, so we can just execute it directly
         prefix = command;
     }
-    std::cout << "prefix: " << prefix << std::endl;
+    std::cout << "LAUNCHER: " << prefix << std::endl;
     return prefix;
 }
 
 
 int lcpex(
         std::string command,
-        std::string stdout_log_file,
+        FILE * stdout_log_fh,
-        std::string stderr_log_file,
+        FILE * stderr_log_fh,
         bool context_override,
         std::string context_user,
         std::string context_group,
@@ -71,11 +73,11 @@ int lcpex(
     // if we are forcing a pty, then we will use the vpty library
     if( force_pty )
     {
-        return exec_pty( command, stdout_log_file, stderr_log_file, context_override, context_user, context_group, supply_environment );
+        return exec_pty( command, stdout_log_fh, stderr_log_fh, context_override, context_user, context_group, supply_environment );
     }
 
     // otherwise, we will use the execute function
-    return execute( command, stdout_log_file, stderr_log_file, context_override, context_user, context_group, supply_environment );
+    return execute( command, stdout_log_fh, stderr_log_fh, context_override, context_user, context_group, supply_environment );
 }
 
 /**
@@ -130,23 +132,23 @@ void run_child_process(bool context_override, const char* context_user, const ch
             case IDENTITY_CONTEXT_ERRORS::ERROR_NONE:
                 break;
             case IDENTITY_CONTEXT_ERRORS::ERROR_NO_SUCH_USER:
-                std::cerr << "lcpex: Aborting: context user not found: " << context_user << std::endl;
+                std::cerr << "REX: Aborting: context user not found: " << context_user << std::endl;
                 exit(1);
                 break;
             case IDENTITY_CONTEXT_ERRORS::ERROR_NO_SUCH_GROUP:
-                std::cerr << "lcpex: Aborting: context group not found: " << context_group << std::endl;
+                std::cerr << "REX: Aborting: context group not found: " << context_group << std::endl;
                 exit(1);
                 break;
             case IDENTITY_CONTEXT_ERRORS::ERROR_SETGID_FAILED:
-                std::cerr << "lcpex: Aborting: Setting GID failed: " << context_user << "/" << context_group << std::endl;
+                std::cerr << "REX: Aborting: Setting GID failed: " << context_user << "/" << context_group << std::endl;
                 exit(1);
                 break;
             case IDENTITY_CONTEXT_ERRORS::ERROR_SETUID_FAILED:
-                std::cerr << "lcpex: Aborting: Setting UID failed: " << context_user << "/" << context_group << std::endl;
+                std::cerr << "REX: Aborting: Setting UID failed: " << context_user << "/" << context_group << std::endl;
                 exit(1);
                 break;
             default:
-                std::cerr << "lcpex: Aborting: Unknown error while setting identity context." << std::endl;
+                std::cerr << "REX: Aborting: Unknown error while setting identity context." << std::endl;
                 exit(1);
                 break;
         }
@@ -160,8 +162,8 @@ void run_child_process(bool context_override, const char* context_user, const ch
 
 int execute(
         std::string command,
-        std::string stdout_log_file,
+        FILE * stdout_log_fh,
-        std::string stderr_log_file,
+        FILE * stderr_log_fh,
         bool context_override,
         std::string context_user,
         std::string context_group,
@@ -179,13 +181,10 @@ int execute(
     // before we fork the process, so that the child process will inherit the environment
     // from the parent process
     if ( environment_supplied ) {
-        clearenv();
+        // this breaks reuse of env variables in between executions
+        //clearenv();
     }
 
-    // open file handles to the two log files we need to create for each execution
-    FILE * stdout_log_fh = fopen( stdout_log_file.c_str(), "a+" );
-    FILE * stderr_log_fh = fopen( stderr_log_file.c_str(), "a+" );
-
     // create the pipes for the child process to write and read from using its stdin/stdout/stderr
     int fd_child_stdout_pipe[2];
     int fd_child_stderr_pipe[2];
@@ -268,6 +267,12 @@ int execute(
             // loop until we've read all the data from the child process
             while ( ! break_out ) {
                 num_files_readable = poll(watched_fds, sizeof(watched_fds) / sizeof(watched_fds[0]), -1);
+                // after the poll() call, add a check to see if both pipes are closed
+                if (!(watched_fds[CHILD_PIPE_NAMES::STDOUT_READ].events & POLLIN) &&
+                    !(watched_fds[CHILD_PIPE_NAMES::STDERR_READ].events & POLLIN)) {
+                    break_out = true;
+                }
+
                 if (num_files_readable == -1) {
                     // error occurred in poll()
                     perror("poll");
@@ -284,9 +289,12 @@ int execute(
                         byte_count = read(watched_fds[this_fd].fd, buf, BUFFER_SIZE);
 
                         if (byte_count == -1) {
-                            // error reading from pipe
+                            if (errno == EAGAIN) { continue; } else {
-                            perror("read");
+                                // error reading from pipe
-                            exit(EXIT_FAILURE);
+                                perror("read");
+                                exit(EXIT_FAILURE);
+                            }
+
                         } else if (byte_count == 0) {
                             // reached EOF on one of the streams but not a HUP
                             // we've read all we can this cycle, so go to the next fd in the for loop
@@ -296,12 +304,12 @@ int execute(
                             // write to stdout,stderr
                             if (this_fd == CHILD_PIPE_NAMES::STDOUT_READ) {
                                 // the child's stdout pipe is readable
-                                write(stdout_log_fh->_fileno, buf, byte_count);
+                                write_all(stdout_log_fh->_fileno, buf, byte_count);
-                                write(STDOUT_FILENO, buf, byte_count);
+                                write_all(STDOUT_FILENO, buf, byte_count);
                             } else if (this_fd == CHILD_PIPE_NAMES::STDERR_READ) {
                                 // the child's stderr pipe is readable
-                                write(stderr_log_fh->_fileno, buf, byte_count);
+                                write_all(stderr_log_fh->_fileno, buf, byte_count);
-                                write(STDERR_FILENO, buf, byte_count);
+                                write_all(STDERR_FILENO, buf, byte_count);
                             } else {
                                 // this should never happen
                                 perror("Logic error!");
@@ -315,18 +323,25 @@ int execute(
                     }
                     if (watched_fds[this_fd].revents & POLLHUP) {
                         // this pipe has hung up
-                        close(watched_fds[this_fd].fd);
+                        // don't close the file descriptor yet, there might still be data to read
-                        break_out = true;
+                        // instead, remove the POLLIN event to avoid getting a POLLHUP event in the next poll() call
+                        watched_fds[this_fd].events &= ~POLLIN;
                     }
-
                 }
             }
             // wait for child to exit, capture status
             waitpid(pid, &status, 0);
 
-            // close the log file handles
+            // Drain the pipes before exiting
-            fclose(stdout_log_fh);
+            while ((byte_count = read(fd_child_stdout_pipe[READ_END], buf, BUFFER_SIZE)) > 0) {
-            fclose(stderr_log_fh);
+                write_all(stdout_log_fh->_fileno, buf, byte_count);
+                write_all(STDOUT_FILENO, buf, byte_count);
+            }
+            while ((byte_count = read(fd_child_stderr_pipe[READ_END], buf, BUFFER_SIZE)) > 0) {
+                write_all(stderr_log_fh->_fileno, buf, byte_count);
+                write_all(STDERR_FILENO, buf, byte_count);
+            }
+
             if WIFEXITED(status) {
                 return WEXITSTATUS(status);
             } else {

src/lcpex/liblcpex.h

@@ -39,8 +39,8 @@
  */
 int execute(
         std::string command,
-        std::string stdout_log_file,
+        FILE * stdout_log_fh,
-        std::string stderr_log_file,
+        FILE * stderr_log_fh,
         bool context_override,
         std::string context_user,
         std::string context_group,
@@ -76,8 +76,8 @@ int execute(
  */
 int lcpex(
         std::string command,
-        std::string stdout_log_file,
+        FILE * stdout_log_fh,
-        std::string stderr_log_file,
+        FILE * stderr_log_fh,
         bool context_override,
         std::string context_user,
         std::string context_group,

src/lcpex/string_expansion/string_expansion.cpp

@@ -1,5 +1,7 @@
 #include "string_expansion.h"
 
+
+
 // convert a string to a char** representing our artificial argv to be consumed by execvp
 char ** expand_env(const std::string& var, int flags )
 {

src/lcpex/vpty/libclpex_tty.cpp

@@ -83,7 +83,7 @@ void run_child_process( int fd_child_stderr_pipe[2], char * processed_command[],
 
     // execute the dang command, print to stdout, stderr (of parent), and dump to file for each!!!!
     // (and capture exit code in parent)
-    int exit_code = execvp(processed_command[0], processed_command);
+    int exit_code = execvp( processed_command[0], processed_command );
     safe_perror("failed on execvp in child", ttyOrig );
     exit(exit_code);
 }
@@ -94,8 +94,8 @@ void run_child_process( int fd_child_stderr_pipe[2], char * processed_command[],
 //  - TEE child stdout/stderr to parent stdout/stderr
 int exec_pty(
         std::string command,
-        std::string stdout_log_file,
+        FILE * stdout_log_fh,
-        std::string stderr_log_file,
+        FILE * stderr_log_fh,
         bool context_override,
         std::string context_user,
         std::string context_group,
@@ -108,15 +108,22 @@ int exec_pty(
     // before we fork the process, so that the child process will inherit the environment
     // from the parent process
     if ( environment_supplied ) {
-        clearenv();
+        //clearenv();
     }
 
     // turn our command string into something execvp can consume
-    char ** processed_command = expand_env(command );
+    char ** processed_command = expand_env( command );
+
+    if ( stdout_log_fh == NULL ) {
+        safe_perror( "Error opening STDOUT log file.  Aborting.", &ttyOrig );
+        exit( 1 );
+    }
+
+    if ( stderr_log_fh == NULL ) {
+        safe_perror( "Error opening STDERR log file.  Aborting.", &ttyOrig );
+        exit( 1 );
+    }
 
-    // open file handles to the two log files we need to create for each execution
-    FILE * stdout_log_fh = fopen( stdout_log_file.c_str(), "a+" );
-    FILE * stderr_log_fh = fopen( stderr_log_file.c_str(), "a+" );
 
     // create the pipes for the child process to write and read from using its stderr
     int fd_child_stderr_pipe[2];
@@ -204,6 +211,11 @@ int exec_pty(
             // loop until we've read all the data from the child process
             while ( ! break_out ) {
                 num_files_readable = poll(watched_fds, sizeof(watched_fds) / sizeof(watched_fds[0]), -1);
+                // after the poll() call, add a check to see if both pipes are closed
+                if (!(watched_fds[1].events & POLLIN) &&
+                    !(watched_fds[2].events & POLLIN)) {
+                    break_out = true;
+                }
 
                 if (num_files_readable == -1) {
                     // error occurred in poll()
@@ -223,9 +235,11 @@ int exec_pty(
                         byte_count = read(watched_fds[this_fd].fd, buf, BUFFER_SIZE);
 
                         if (byte_count == -1) {
-                            // error reading from pipe
+                            if (errno == EAGAIN) { continue; } else {
-                            safe_perror("read", &ttyOrig );
+                                // error reading from pipe
-                            exit(EXIT_FAILURE);
+                                safe_perror("read", &ttyOrig );
+                                exit(EXIT_FAILURE);
+                            }
                         } else if (byte_count == 0) {
                             // reached EOF on one of the streams but not a HUP
                             // we've read all we can this cycle, so go to the next fd in the for loop
@@ -233,17 +247,18 @@ int exec_pty(
                         } else {
                             // byte count was sane
                             // write to stdout,stderr
+
                             if (this_fd == 0) {
                                 // parent stdin received, write to child pty (stdin)
-                                write(masterFd, buf, byte_count);
+                                write_all(masterFd, buf, byte_count);
                             } else if (this_fd == 1 ) {
                                 // child pty sent some stuff, write to parent stdout and log
-                                write(stdout_log_fh->_fileno, buf, byte_count);
+                                write_all(stdout_log_fh->_fileno, buf, byte_count);
-                                write(STDOUT_FILENO, buf, byte_count);
+                                write_all(STDOUT_FILENO, buf, byte_count);
                             } else if ( this_fd == 2 ){
                                 //the child's stderr pipe sent some stuff, write to parent stderr and log
-                                write(stderr_log_fh->_fileno, buf, byte_count);
+                                write_all(stderr_log_fh->_fileno, buf, byte_count);
-                                write(STDERR_FILENO, buf, byte_count);
+                                write_all(STDERR_FILENO, buf, byte_count);
                             } else {
                                 // this should never happen
                                 perror("Logic error!");
@@ -252,24 +267,37 @@ int exec_pty(
                         }
                     }
                     if (watched_fds[this_fd].revents & POLLERR) {
-                        close(watched_fds[this_fd].fd);
+                        //close(watched_fds[this_fd].fd);
-                        //break_out = true;
+                        break_out = true;
-                        continue;
+                        //continue;
                     }
+//                    if (watched_fds[this_fd].revents & POLLHUP) {
+//                        // this pipe has hung up
+//                        close(watched_fds[this_fd].fd);
+//                        break_out = true;
+//                        break;
+//                    }
                     if (watched_fds[this_fd].revents & POLLHUP) {
                         // this pipe has hung up
-                        close(watched_fds[this_fd].fd);
+                        // don't close the file descriptor yet, there might still be data to read
-                        break_out = true;
+                        // instead, remove the POLLIN event to avoid getting a POLLHUP event in the next poll() call
-                        break;
+                        watched_fds[this_fd].events &= ~POLLIN;
                     }
                 }
             }
             // wait for child to exit, capture status
             waitpid(pid, &status, 0);
 
-            // close the log file handles
+            while ((byte_count = read(watched_fds[1].fd, buf, BUFFER_SIZE)) > 0) {
-            fclose(stdout_log_fh);
+                write_all(stdout_log_fh->_fileno, buf, byte_count);
-            fclose(stderr_log_fh);
+                write_all(STDOUT_FILENO, buf, byte_count);
+            }
+
+            while ((byte_count = read(fd_child_stderr_pipe[READ_END], buf, BUFFER_SIZE)) > 0) {
+                write_all(stderr_log_fh->_fileno, buf, byte_count);
+                write_all(STDERR_FILENO, buf, byte_count);
+            }
+
             ttyResetExit( &ttyOrig);
             if WIFEXITED(status) {
                 return WEXITSTATUS(status);
@@ -279,4 +307,3 @@ int exec_pty(
         }
     }
 }
-

src/lcpex/vpty/libclpex_tty.h

@@ -38,8 +38,8 @@
  */
 int exec_pty(
         std::string command,
-        std::string stdout_log_file,
+        FILE * stdout_log_fh,
-        std::string stderr_log_file,
+        FILE * stderr_log_fh,
         bool context_override,
         std::string context_user,
         std::string context_group,

src/misc/helpers.cpp

@@ -96,19 +96,57 @@ std::string get_8601()
  * @brief Interpolates the environment variables in the input text
  *
  * This function takes a string reference as input and replaces all occurrences of
- * environment variables in the format `${VAR_NAME}` with their corresponding values.
+ * environment variables in the format `${VAR_NAME}` or `$VAR_NAME` with their corresponding values.
  * If an environment variable is not set, it is replaced with an empty string.
  *
  * @param text The input text to be processed
  */
+//void interpolate( std::string & text )
+//{
+//    static std::regex env1( "\\$\\{([^}]+)\\}" );
+//    static std::regex env2( "\\$([^/]+)" ); // matches $VAR_NAME until a / is found
+//    std::smatch match;
+//    while ( std::regex_search( text, match, env1 ) || std::regex_search( text, match, env2 ) )
+//    {
+//        const char * s = getenv( match[1].str().c_str() );
+//        const std::string var( s == NULL ? "" : s );
+//        text.replace( match[0].first, match[0].second, var );
+//    }
+//}
 void interpolate( std::string & text )
 {
-    static std::regex env( "\\$\\{([^}]+)\\}" );
+    std::regex env1( "\\$\\{([^}]+)\\}" );
+    std::regex env2( "\\$([^/]+)" ); // matches $VAR_NAME until a / is found
     std::smatch match;
-    while ( std::regex_search( text, match, env ) )
+    while ( std::regex_search( text, match, env1 ) || std::regex_search( text, match, env2 ) )
     {
         const char * s = getenv( match[1].str().c_str() );
         const std::string var( s == NULL ? "" : s );
         text.replace( match[0].first, match[0].second, var );
     }
 }
+
+/**
+ * @brief Get the absolute path from a relative path
+ *
+ * This function takes a relative path and returns the corresponding absolute path.
+ * The absolute path is obtained by calling the `realpath` function.
+ * If the `realpath` function returns a null pointer, an error message is printed to the standard error stream and an empty string is returned.
+ *
+ * @param relative_path The relative path to be converted to an absolute path
+ *
+ * @return The absolute path corresponding to the relative path
+ */
+std::string get_absolute_path(const std::string &relative_path)
+{
+    char resolved_path[1024];
+    memset(resolved_path, 0, sizeof(resolved_path));
+
+    if( realpath( relative_path.c_str(), resolved_path) == nullptr ) {
+        std::cerr << "Error resolving path: " << relative_path << std::endl;
+        return "";
+    }
+
+    return std::string(resolved_path);
+}
+

src/misc/helpers.h

@@ -23,6 +23,9 @@
 #define REX_HELPERS_H
 
 #include <string>
+#include <cstring>
+#include <string.h>
+
 #include <sys/stat.h>
 #include <sys/param.h>
 #include <unistd.h>
@@ -51,4 +54,17 @@ std::string get_8601();
 
 const char * command2args( std::string input_string );
 
+/**
+ * @brief Get the absolute path from a relative path
+ *
+ * This function takes a relative path and returns the corresponding absolute path.
+ * The absolute path is obtained by calling the `realpath` function.
+ * If the `realpath` function returns a null pointer, an error message is printed to the standard error stream and an empty string is returned.
+ *
+ * @param relative_path The relative path to be converted to an absolute path
+ *
+ * @return The absolute path corresponding to the relative path
+ */
+std::string get_absolute_path(const std::string &relative_path);
+
 #endif //REX_HELPERS_H

src/plan/Task.cpp

@@ -307,7 +307,7 @@ bool createDirectory(const std::string& path) {
     return true;
 }
 
-bool Task::prepare_logs( std::string task_name, std::string logs_root, std::string timestamp )
+bool Task::prepare_logs( std::string task_name, std::string logs_root )
 {
     std::string full_path = logs_root + "/" + task_name;
     bool ret = false;
@@ -354,8 +354,19 @@ void Task::execute( Conf * configuration )
     std::string environment_file = this->definition.get_environment_file();
     std::string logs_root = configuration->get_logs_path();
 
+    interpolate(task_name);
     this->slog.log_task( E_DEBUG, task_name, "Using unit definition: \"" + task_name + "\"." );
 
+    interpolate(command);
+    interpolate(shell_name);
+    interpolate(user);
+    interpolate(group);
+    interpolate(environment_file);
+    interpolate(new_working_dir);
+    interpolate(rectifier);
+    interpolate(logs_root);
+
+
     // sanitize all path inputs from unit definition to be either absolute paths or relative to
     // project_root
 
@@ -407,19 +418,24 @@ void Task::execute( Conf * configuration )
         logs_root = configuration->get_project_root() + "/" + logs_root;
     }
 
-    std::string timestamp = get_8601();
+
     // set these first so the pre-execution logs get there.
     /*
      * create the logs dir here
      */
 
-    if (! this->prepare_logs( task_name, logs_root, timestamp ) )
+    if (! this->prepare_logs( task_name, logs_root ) )
     {
         throw TaskException("Could not prepare logs for task execution at '" + logs_root + "'.");
     }
 
-    std::string stdout_log_file = logs_root + "/" + timestamp + ".stdout.log";
+    std::string timestamp = get_8601();
-    std::string stderr_log_file = logs_root + "/" + timestamp + ".stderr.log";
+    std::string stdout_log_file = logs_root + "/" + task_name + "/" + timestamp + ".stdout.log";
+    std::string stderr_log_file = logs_root + "/" + task_name + "/" + timestamp + ".stderr.log";
+
+    // open file handles to the two log files we need to create for each execution
+    FILE * stdout_log_fh = fopen( stdout_log_file.c_str(), "a+" );
+    FILE * stderr_log_fh = fopen( stderr_log_file.c_str(), "a+" );
 
     // check if working directory is to be set
     if ( override_working_dir )
@@ -443,8 +459,8 @@ void Task::execute( Conf * configuration )
 
     int return_code = lcpex(
             command,
-            stdout_log_file,
+            stdout_log_fh,
-            stderr_log_file,
+            stderr_log_fh,
             set_user_context,
             user,
             group,
@@ -457,7 +473,6 @@ void Task::execute( Conf * configuration )
             environment_file
     );
 
-
     // **********************************************
     // d[0] Error Code Check
     // **********************************************
@@ -513,8 +528,8 @@ void Task::execute( Conf * configuration )
             this->slog.log_task( E_INFO, task_name, "Executing rectification: " + rectifier + "." );
             int rectifier_error = lcpex(
                     rectifier,
-                    stdout_log_file,
+                    stdout_log_fh,
-                    stderr_log_file,
+                    stderr_log_fh,
                     set_user_context,
                     user,
                     group,
@@ -565,8 +580,8 @@ void Task::execute( Conf * configuration )
 
                 int retry_code = lcpex(
                         command,
-                        stdout_log_file,
+                        stdout_log_fh,
-                        stderr_log_file,
+                        stderr_log_fh,
                         set_user_context,
                         user,
                         group,
@@ -618,4 +633,7 @@ void Task::execute( Conf * configuration )
         // end d[1] Rectify Check
         // **********************************************
     }
+    // close the log file handles
+    fclose(stdout_log_fh);
+    fclose(stderr_log_fh);
 }

src/plan/Task.h

@@ -47,7 +47,7 @@ class Task
         // the readiness of this task to execute
         bool defined;
 
-        bool prepare_logs( std::string task_name, std::string logs_root, std::string timestamp );
+        bool prepare_logs( std::string task_name, std::string logs_root );
 
 
 public:

src/shells/shells.cpp

@@ -54,11 +54,46 @@ int Shell::load_root( Json::Value loader_root )
         throw ShellException("No execution_arg attribute specified when loading a shell definition.");
     }
 
-    if ( loader_root.isMember("execution_arg") )
+    if ( loader_root.isMember("source_cmd") )
     {
-        this->execution_arg = loader_root.get( "execution_arg", errmsg ).asString();
+        this->source_cmd = loader_root.get( "source_cmd", errmsg ).asString();
     } else {
-        throw ShellException("No execution_arg attribute specified when loading a shell definition.");
+        throw ShellException("No source_cmd attribute specified when loading a shell definition.");
     }
     return 0;
 }
+
+
+void get_shells_from_dir( std::vector<std::string> * files, std::string path )
+{
+    DIR* dirFile = opendir( path.c_str() );
+    if ( dirFile )
+    {
+        struct dirent* hFile;
+        errno = 0;
+        while (( hFile = readdir( dirFile )) != NULL )
+        {
+            if ( !strcmp( hFile->d_name, "."  ))
+            {
+                continue;
+            }
+            if ( !strcmp( hFile->d_name, ".." ))
+            {
+                continue;
+            }
+
+            // hidden files
+            if ( hFile->d_name[0] == '.' )
+            {
+                continue;
+            }
+
+            if ( strstr( hFile->d_name, ".shell" ))
+            {
+                std::string full_path = path + "/" + hFile->d_name;
+                files->push_back( full_path );
+            }
+        }
+        closedir( dirFile );
+    }
+}

src/shells/shells.h

@@ -2,7 +2,11 @@
 #define REX_SHELLS_H
 
 #include "../json_support/JSON.h"
+#include "../misc/helpers.h"
 #include <string>
+#include <string.h>
+#include <vector>
+#include <dirent.h>
 
 class Shell: public JSON_Loader {
     public:
@@ -23,4 +27,6 @@ class Shell: public JSON_Loader {
 
 };
 
+void get_shells_from_dir( std::vector<std::string> * files, std::string path );
+
 #endif //REX_SHELLS_H

src/suite/Suite.cpp

@@ -129,7 +129,7 @@ void Suite::get_units_from_dir( std::vector<std::string> * files, std::string pa
             // you want here. Something like:
             if ( strstr( hFile->d_name, ".units" ))
             {
-                std::string full_path = path + hFile->d_name;
+                std::string full_path = path + "/" + hFile->d_name;
                 files->push_back( full_path );
             }
         }