shells_path is now a directory of .shell files

Matches the existing units_path pattern — shells_path points to a
directory that is scanned for .shell files rather than a single
monolithic definitions file.

sample/environments/rex.variables

@@ -1,9 +1,5 @@
 #!/usr/bin/bash
 
-
-
-
-
 set -a
 
 echo "variables file says hello and set a variable named TEST_VAR"

sample/plans/test.plan

@@ -1,5 +1,6 @@
 {
   "plan": [
-    { "name": "independent test 1", "dependencies":  [ null ] }
+    { "name": "independent test 1", "dependencies":  [ null ] },
+    { "name": "independent test 2", "dependencies":  [ null ] }
   ]
 }

sample/rex.config

@@ -3,7 +3,7 @@
 	"project_root": 	"$HOME/development/internal/rex/sample",
   	"units_path":		"units/",
   	"logs_path": 		"logs/",
-	"shells_path":		"shells/shells.definitions",
+	"shells_path":		"shells/",
   	"config_version": "5"
   }
 }

sample/shells/bash.shell

@@ -5,12 +5,6 @@
 			"path": 		"/usr/bin/bash",
 			"execution_arg": 	"-c",
 			"source_cmd":		"source"
-		},
-		{
-			"name":			"ksh",
-			"path":			"/usr/bin/ksh",
-			"execution_arg":	"-c",
-			"source_cmd":		"source"
 		}
 	]
 }

sample/shells/ksh.shell

@@ -0,0 +1,10 @@
+{
+	"shells":	[
+		{
+			"name":			"ksh",
+			"path":			"/usr/bin/ksh",
+			"execution_arg":	"-c",
+			"source_cmd":		"source"
+		}
+	]
+}

sample/units/new.units

@@ -17,6 +17,24 @@
         "group":        		"$USER",
 	"supply_environment":		true,
         "environment":  		"environments/rex.variables"
+    },
+    {
+        "name":         		"independent test 2",
+        "target":       		"components/independent_test_1.bash --ls -s --arg",
+	"is_shell_command":		true,
+	"shell_definition":		"bash",
+	"force_pty":			true,
+	"set_working_directory":	false,
+	"working_directory":		"",
+        "rectify":      		false,
+        "rectifier":    		"echo rectifier executed",
+        "active":       		true,
+        "required":     		true,
+	"set_user_context": 		true,
+        "user":         		"$USER",
+        "group":        		"$USER",
+	"supply_environment":		true,
+        "environment":  		"environments/rex.variables"
     }
  ]
 }

src/config/Config.cpp

@@ -72,7 +72,7 @@ void Conf::set_object_s(std::string keyname, std::string & object_member, std::s
     if ( this->get_string(jval_s, keyname) != 0) {
         throw ConfigLoadException( "'" + keyname + "' string is not set in the config file supplied: " + filename );
     } else {
-        interpolate(jval_s);
+        //interpolate(jval_s);
         object_member = jval_s;
     }
     this->slog.log_task( E_DEBUG, "SET_PROPERTY", "'" + keyname + "': " + object_member );
@@ -101,7 +101,7 @@ void Conf::set_object_s_derivedpath(std::string keyname, std::string & object_me
     if ( this->get_string(jval_s, keyname) != 0) {
         throw ConfigLoadException( "'" + keyname + "' string is not set in the config file supplied: " + filename );
     } else {
-        interpolate(jval_s);
+        //interpolate(jval_s);
         object_member = prepend_project_root( jval_s );
     }
     this->slog.log_task( E_DEBUG, "SET_PROPERTY", "'" + keyname + "': " + object_member );
@@ -197,26 +197,44 @@ void Conf::checkPathExists( std::string keyname, const std::string &path ) {
  */
 void Conf::load_shells() {
     this->slog.log_task( E_DEBUG, "SHELLS", "Loading shells..." );
-    try {
-        // load the test file.
-        this->load_json_file( this->shell_definitions_path );
-    } catch (std::exception& e) {
-        this->slog.log_task( E_FATAL, "SHELLS", "Unable to load shell definition file: '" + this->shell_definitions_path + "'. Error: " + e.what());
-        throw ConfigLoadException("Parsing error in shell definitions file.");
-    }
-    Json::Value jbuff;
 
-    if ( this-> get_serialized( jbuff, "shells" ) != 0 ) {
+    std::vector<std::string> shell_files;
-        this->slog.log_task( E_FATAL, "SHELLS", "Parsing error: '" + this->shell_definitions_path + "'. Error: 'shells' key not found." );
-        throw ConfigLoadException("Parsing error in shell definitions file.");
-    }
 
-    Shell tmp_S = Shell( this->LOG_LEVEL );
+    if ( is_dir( this->shell_definitions_path ) )
-    for ( int index = 0; index < jbuff.size(); index++ )
     {
-        tmp_S.load_root( jbuff[index] );
+        get_shells_from_dir( &shell_files, this->shell_definitions_path );
-        this->shells.push_back( tmp_S );
+    }
-        this->slog.log_task( E_DEBUG, "SHELLS", "Loaded shell: '" + tmp_S.name + "' (" + tmp_S.path + ")" );
+
+    if ( is_file( this->shell_definitions_path ) )
+    {
+        shell_files.push_back( this->shell_definitions_path );
+    }
+
+    this->slog.log_task( E_INFO, "SHELLS", "Shell files found: " + std::to_string( shell_files.size() ) );
+
+    for ( int i = 0; i < shell_files.size(); i++ )
+    {
+        try {
+            this->load_json_file( shell_files[i] );
+        } catch (std::exception& e) {
+            this->slog.log_task( E_FATAL, "SHELLS", "Unable to load shell definition file: '" + shell_files[i] + "'. Error: " + e.what());
+            throw ConfigLoadException("Parsing error in shell definitions file.");
+        }
+
+        Json::Value jbuff;
+
+        if ( this->get_serialized( jbuff, "shells" ) != 0 ) {
+            this->slog.log_task( E_FATAL, "SHELLS", "Parsing error: '" + shell_files[i] + "'. Error: 'shells' key not found." );
+            throw ConfigLoadException("Parsing error in shell definitions file.");
+        }
+
+        Shell tmp_S = Shell( this->LOG_LEVEL );
+        for ( int index = 0; index < jbuff.size(); index++ )
+        {
+            tmp_S.load_root( jbuff[index] );
+            this->shells.push_back( tmp_S );
+            this->slog.log_task( E_DEBUG, "SHELLS", "Loaded shell: '" + tmp_S.name + "' (" + tmp_S.path + ")" );
+        }
     }
 }
 
@@ -243,8 +261,6 @@ Shell Conf::get_shell_by_name( std::string name ) {
     throw ConfigLoadException("The shell specified ('" + name + "') is not defined in the shell definitions file.");
 }
 
-
-
 /**
  * @class Conf
  * @brief Loads the configuration for the application
@@ -262,9 +278,9 @@ Shell Conf::get_shell_by_name( std::string name ) {
 Conf::Conf(std::string filename, int LOG_LEVEL ): JSON_Loader(LOG_LEVEL ), slog(LOG_LEVEL, "_conf_" )
 {
     this->LOG_LEVEL = LOG_LEVEL;
-    this->slog.log_task( E_DEBUG, "LOAD", "Loading configuration file: " + filename );
 
     interpolate( filename );
+    this->slog.log_task( E_DEBUG, "LOAD", "Loading configuration file: " + filename );
 
     try {
         // load the test file.
@@ -273,6 +289,7 @@ Conf::Conf(std::string filename, int LOG_LEVEL ): JSON_Loader(LOG_LEVEL ), slog(
         this->slog.log( E_FATAL, "Unable to load configuration file: '" + filename + "'. Error: " + e.what());
         throw ConfigLoadException("Parsing error in configuration file.");
     }
+
     Json::Value jbuff;
 
     if ( this->get_serialized( jbuff, "config" ) != 0) {
@@ -283,26 +300,27 @@ Conf::Conf(std::string filename, int LOG_LEVEL ): JSON_Loader(LOG_LEVEL ), slog(
         this->json_root = jbuff;
     }
 
-    set_object_s(   "project_root",     this->project_root,           filename );
+    set_object_s( "project_root",     this->project_root,           filename );
-    interpolate( project_root );
+    interpolate( this->project_root );
+
     // convert to an absolute path after all the interpolation is done.
     this->project_root = get_absolute_path( this->project_root );
 
-    set_object_s(   "logs_path",        this->logs_path,              filename );
+    set_object_s( "logs_path",        this->logs_path,              filename );
     interpolate( this->logs_path );
 
     // all other paths are relative to project_root
-    set_object_s_derivedpath(   "units_path",       this->units_path,             filename );
+    set_object_s_derivedpath( "units_path",       this->units_path,             filename );
     interpolate( this->units_path );
 
-    set_object_s_derivedpath(   "shells_path",      this->shell_definitions_path, filename );
+    set_object_s_derivedpath( "shells_path",      this->shell_definitions_path, filename );
     interpolate( this->shell_definitions_path );
 
     // ensure these paths exists, with exception to the logs_path, which will be created at runtime
     this->slog.log_task( E_DEBUG, "SANITY_CHECKS", "Checking for sanity..." );
-    checkPathExists(            "project_root",     this->project_root );
+    checkPathExists( "project_root",     this->project_root );
-    checkPathExists(            "units_path",       this->units_path );
+    checkPathExists( "units_path",       this->units_path );
-    checkPathExists(            "shells_path",      this->shell_definitions_path );
+    checkPathExists( "shells_path",      this->shell_definitions_path );
 
     // shells are scoped beyond plan so they need to be considered part of config
     load_shells();

src/lcpex/liblcpex.cpp

@@ -181,7 +181,8 @@ int execute(
     // before we fork the process, so that the child process will inherit the environment
     // from the parent process
     if ( environment_supplied ) {
-        clearenv();
+        // this breaks reuse of env variables in between executions
+        //clearenv();
     }
 
     // create the pipes for the child process to write and read from using its stdin/stdout/stderr
@@ -320,11 +321,6 @@ int execute(
                         close(watched_fds[this_fd].fd);
                         break_out = true;
                     }
-//                    if (watched_fds[this_fd].revents & POLLHUP) {
-//                        // this pipe has hung up
-//                        close(watched_fds[this_fd].fd);
-//                        break_out = true;
-//                    }
                     if (watched_fds[this_fd].revents & POLLHUP) {
                         // this pipe has hung up
                         // don't close the file descriptor yet, there might still be data to read

src/lcpex/vpty/libclpex_tty.cpp

@@ -108,7 +108,7 @@ int exec_pty(
     // before we fork the process, so that the child process will inherit the environment
     // from the parent process
     if ( environment_supplied ) {
-        clearenv();
+        //clearenv();
     }
 
     // turn our command string into something execvp can consume
@@ -235,13 +235,11 @@ int exec_pty(
                         byte_count = read(watched_fds[this_fd].fd, buf, BUFFER_SIZE);
 
                         if (byte_count == -1) {
-                            if (errno == EAGAIN) {
+                            if (errno == EAGAIN) { continue; } else {
-                                // no data to read
+                                // error reading from pipe
-                                continue;
+                                safe_perror("read", &ttyOrig );
+                                exit(EXIT_FAILURE);
                             }
-                            // error reading from pipe
-                            safe_perror("read", &ttyOrig );
-                            exit(EXIT_FAILURE);
                         } else if (byte_count == 0) {
                             // reached EOF on one of the streams but not a HUP
                             // we've read all we can this cycle, so go to the next fd in the for loop
@@ -249,6 +247,7 @@ int exec_pty(
                         } else {
                             // byte count was sane
                             // write to stdout,stderr
+
                             if (this_fd == 0) {
                                 // parent stdin received, write to child pty (stdin)
                                 write_all(masterFd, buf, byte_count);
@@ -268,7 +267,7 @@ int exec_pty(
                         }
                     }
                     if (watched_fds[this_fd].revents & POLLERR) {
-                        close(watched_fds[this_fd].fd);
+                        //close(watched_fds[this_fd].fd);
                         break_out = true;
                         //continue;
                     }
@@ -289,6 +288,11 @@ int exec_pty(
             // wait for child to exit, capture status
             waitpid(pid, &status, 0);
 
+            while ((byte_count = read(watched_fds[1].fd, buf, BUFFER_SIZE)) > 0) {
+                write_all(stdout_log_fh->_fileno, buf, byte_count);
+                write_all(STDOUT_FILENO, buf, byte_count);
+            }
+
             while ((byte_count = read(fd_child_stderr_pipe[READ_END], buf, BUFFER_SIZE)) > 0) {
                 write_all(stderr_log_fh->_fileno, buf, byte_count);
                 write_all(STDERR_FILENO, buf, byte_count);

src/misc/helpers.cpp

@@ -101,10 +101,22 @@ std::string get_8601()
  *
  * @param text The input text to be processed
  */
+//void interpolate( std::string & text )
+//{
+//    static std::regex env1( "\\$\\{([^}]+)\\}" );
+//    static std::regex env2( "\\$([^/]+)" ); // matches $VAR_NAME until a / is found
+//    std::smatch match;
+//    while ( std::regex_search( text, match, env1 ) || std::regex_search( text, match, env2 ) )
+//    {
+//        const char * s = getenv( match[1].str().c_str() );
+//        const std::string var( s == NULL ? "" : s );
+//        text.replace( match[0].first, match[0].second, var );
+//    }
+//}
 void interpolate( std::string & text )
 {
-    static std::regex env1( "\\$\\{([^}]+)\\}" );
+    std::regex env1( "\\$\\{([^}]+)\\}" );
-    static std::regex env2( "\\$([^/]+)" ); // matches $VAR_NAME until a / is found
+    std::regex env2( "\\$([^/]+)" ); // matches $VAR_NAME until a / is found
     std::smatch match;
     while ( std::regex_search( text, match, env1 ) || std::regex_search( text, match, env2 ) )
     {

src/plan/Task.cpp

@@ -354,9 +354,9 @@ void Task::execute( Conf * configuration )
     std::string environment_file = this->definition.get_environment_file();
     std::string logs_root = configuration->get_logs_path();
 
+    interpolate(task_name);
     this->slog.log_task( E_DEBUG, task_name, "Using unit definition: \"" + task_name + "\"." );
 
-    interpolate(task_name);
     interpolate(command);
     interpolate(shell_name);
     interpolate(user);

src/shells/shells.cpp

@@ -62,3 +62,38 @@ int Shell::load_root( Json::Value loader_root )
     }
     return 0;
 }
+
+
+void get_shells_from_dir( std::vector<std::string> * files, std::string path )
+{
+    DIR* dirFile = opendir( path.c_str() );
+    if ( dirFile )
+    {
+        struct dirent* hFile;
+        errno = 0;
+        while (( hFile = readdir( dirFile )) != NULL )
+        {
+            if ( !strcmp( hFile->d_name, "."  ))
+            {
+                continue;
+            }
+            if ( !strcmp( hFile->d_name, ".." ))
+            {
+                continue;
+            }
+
+            // hidden files
+            if ( hFile->d_name[0] == '.' )
+            {
+                continue;
+            }
+
+            if ( strstr( hFile->d_name, ".shell" ))
+            {
+                std::string full_path = path + "/" + hFile->d_name;
+                files->push_back( full_path );
+            }
+        }
+        closedir( dirFile );
+    }
+}

src/shells/shells.h

@@ -2,7 +2,11 @@
 #define REX_SHELLS_H
 
 #include "../json_support/JSON.h"
+#include "../misc/helpers.h"
 #include <string>
+#include <string.h>
+#include <vector>
+#include <dirent.h>
 
 class Shell: public JSON_Loader {
     public:
@@ -23,4 +27,6 @@ class Shell: public JSON_Loader {
 
 };
 
+void get_shells_from_dir( std::vector<std::string> * files, std::string path );
+
 #endif //REX_SHELLS_H