Commits

Ralph Castain committed c455ee3

Per conversation with Josh H., remove the stale rsh component from the filem framework. Let the "raw" component do the job.

Comments (0)

Files changed (7)

orte/mca/filem/base/base.h

 #define ORTE_FILEM_GET_REMOTE_PATH_CMD     2
 
 /**
- * FileM request object maintenance functions
- */
-ORTE_DECLSPEC void orte_filem_base_process_set_construct(orte_filem_base_process_set_t *obj);
-ORTE_DECLSPEC void orte_filem_base_process_set_destruct( orte_filem_base_process_set_t *obj);
-
-ORTE_DECLSPEC void orte_filem_base_file_set_construct(orte_filem_base_file_set_t *obj);
-ORTE_DECLSPEC void orte_filem_base_file_set_destruct( orte_filem_base_file_set_t *obj);
-
-ORTE_DECLSPEC void orte_filem_base_construct(orte_filem_base_request_t *obj);
-ORTE_DECLSPEC void orte_filem_base_destruct( orte_filem_base_request_t *obj);
-
-
-/**
  * Globals
  */
 ORTE_DECLSPEC extern orte_filem_base_module_t orte_filem;
  * These are to be used when no component is selected.
  * They just return success, and empty strings as necessary.
  */
-ORTE_DECLSPEC int orte_filem_base_none_open(void);
-ORTE_DECLSPEC int orte_filem_base_none_close(void);
-ORTE_DECLSPEC int orte_filem_base_none_query(mca_base_module_t **module, int *priority);
-
 int orte_filem_base_module_init(void);
 int orte_filem_base_module_finalize(void);
 
                                         void* cbdata);
 
 
-/**
- * Get Node Name for an ORTE process
- */
-ORTE_DECLSPEC int orte_filem_base_get_proc_node_name(orte_process_name_t *proc, char **machine_name);
-ORTE_DECLSPEC int orte_filem_base_get_remote_path(char **remote_ref, orte_process_name_t *peer, int *flag);
-
-/**
- * Setup request structure
- */
-ORTE_DECLSPEC int orte_filem_base_prepare_request(orte_filem_base_request_t *request, int move_type);
-
 END_C_DECLS
 
 #endif /* ORTE_FILEM_BASE_H */

orte/mca/filem/base/filem_base_fns.c

 /******************
  * Object Stuff
  ******************/
-ORTE_DECLSPEC OBJ_CLASS_INSTANCE(orte_filem_base_process_set_t,
-                                 opal_list_item_t,
-                                 orte_filem_base_process_set_construct,
-                                 orte_filem_base_process_set_destruct);
-
-ORTE_DECLSPEC void orte_filem_base_process_set_construct(orte_filem_base_process_set_t *req) {
+static void process_set_construct(orte_filem_base_process_set_t *req) {
     req->source = *ORTE_NAME_INVALID;
     req->sink   = *ORTE_NAME_INVALID;
 }
 
-ORTE_DECLSPEC void orte_filem_base_process_set_destruct( orte_filem_base_process_set_t *req) {
+static void process_set_destruct( orte_filem_base_process_set_t *req) {
     req->source = *ORTE_NAME_INVALID;
     req->sink   = *ORTE_NAME_INVALID;
 }
 
-ORTE_DECLSPEC OBJ_CLASS_INSTANCE(orte_filem_base_file_set_t,
-                                 opal_list_item_t,
-                                 orte_filem_base_file_set_construct,
-                                 orte_filem_base_file_set_destruct);
+OBJ_CLASS_INSTANCE(orte_filem_base_process_set_t,
+                   opal_list_item_t,
+                   process_set_construct,
+                   process_set_destruct);
 
-ORTE_DECLSPEC void orte_filem_base_file_set_construct(orte_filem_base_file_set_t *req) {
+static void file_set_construct(orte_filem_base_file_set_t *req) {
     req->local_target  = NULL;
     req->local_hint    = ORTE_FILEM_HINT_NONE;
 
 
 }
 
-ORTE_DECLSPEC void orte_filem_base_file_set_destruct( orte_filem_base_file_set_t *req) {
+static void file_set_destruct( orte_filem_base_file_set_t *req) {
     if( NULL != req->local_target ) {
         free(req->local_target);
         req->local_target = NULL;
     req->target_flag   = ORTE_FILEM_TYPE_UNKNOWN;
 }
 
-ORTE_DECLSPEC OBJ_CLASS_INSTANCE(orte_filem_base_request_t,
-                                 opal_list_item_t,
-                                 orte_filem_base_construct,
-                                 orte_filem_base_destruct);
+OBJ_CLASS_INSTANCE(orte_filem_base_file_set_t,
+                   opal_list_item_t,
+                   file_set_construct,
+                   file_set_destruct);
 
-void orte_filem_base_construct(orte_filem_base_request_t *req) {
+static void req_construct(orte_filem_base_request_t *req) {
     OBJ_CONSTRUCT(&req->process_sets,  opal_list_t);
     OBJ_CONSTRUCT(&req->file_sets,     opal_list_t);
 
     req->movement_type = ORTE_FILEM_MOVE_TYPE_UNKNOWN;
 }
 
-void orte_filem_base_destruct( orte_filem_base_request_t *req) {
+static void req_destruct( orte_filem_base_request_t *req) {
     opal_list_item_t* item = NULL;
 
     while( NULL != (item = opal_list_remove_first(&req->process_sets)) ) {
     req->movement_type = ORTE_FILEM_MOVE_TYPE_UNKNOWN;
 }
 
+OBJ_CLASS_INSTANCE(orte_filem_base_request_t,
+                   opal_list_item_t,
+                   req_construct,
+                   req_destruct);
+
 /***********************
  * None component stuff
  ************************/
-int orte_filem_base_none_open(void)
-{
-    return ORTE_SUCCESS;
-}
-
-int orte_filem_base_none_close(void)
-{
-    return ORTE_SUCCESS;
-}
-
-int orte_filem_base_none_query(mca_base_module_t **module, int *priority)
-{
-    *module = NULL;
-    *priority = 0;
-
-    return OPAL_SUCCESS;
-}
-
 int orte_filem_base_module_init(void)
 {
     return ORTE_SUCCESS;
 {
     return ORTE_SUCCESS;
 }
-
-
-/********************
- * Utility functions
- ********************/
-int orte_filem_base_get_proc_node_name(orte_process_name_t *proc, char **machine_name) {
-    int ret;
-    orte_std_cntr_t count;
-    opal_buffer_t request, answer;
-    orte_filem_cmd_flag_t command=ORTE_FILEM_GET_PROC_NODE_NAME_CMD;
-
-    /* set default answer */
-    *machine_name = NULL;
-
-    if (ORTE_PROC_IS_HNP) {
-        /* if I am the HNP, then all the data structures are local to me - no
-         * need to send messages around to get the info
-         */
-        orte_job_t *jdata;
-        orte_proc_t **procs;
-
-        /* get the job data object for this proc */
-        if (NULL == (jdata = orte_get_job_data_object(proc->jobid))) {
-            ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
-            return ORTE_ERR_NOT_FOUND;
-        }
-        /* get the proc object for it */
-        procs = (orte_proc_t**)jdata->procs->addr;
-        if (NULL == procs[proc->vpid] || NULL == procs[proc->vpid]->node) {
-            ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
-            return ORTE_ERR_NOT_FOUND;
-        }
-        *machine_name = strdup(procs[proc->vpid]->node->name);
-        return ORTE_SUCCESS;
-    }
-    
-    /* if I am not the HNP, then I have to send a request to the HNP
-     * for the information
-     */
-    OBJ_CONSTRUCT(&request, opal_buffer_t);
-    OBJ_CONSTRUCT(&answer, opal_buffer_t);
- 
-    if (ORTE_SUCCESS != (ret = opal_dss.pack(&request, &command, 1, ORTE_FILEM_CMD))) {
-        ORTE_ERROR_LOG(ret);
-        goto CLEANUP;
-    }
-    if (ORTE_SUCCESS != (ret = opal_dss.pack(&request, proc, 1, ORTE_NAME))) {
-        ORTE_ERROR_LOG(ret);
-        goto CLEANUP;
-    }
-    
-    if (0 > (ret = orte_rml.send_buffer(ORTE_PROC_MY_HNP, &request, ORTE_RML_TAG_FILEM_BASE, 0))) {
-        ORTE_ERROR_LOG(ret);
-        goto CLEANUP;
-    }
-
-    /* wait for answer */
-    if (0 > (ret = orte_rml.recv_buffer(ORTE_NAME_WILDCARD, &answer, ORTE_RML_TAG_FILEM_BASE_RESP, 0))) {
-        ORTE_ERROR_LOG(ret);
-        goto CLEANUP;
-    }
-    
-    /* unpack the machine name */
-    count = 1;
-    if (ORTE_SUCCESS != (ret = opal_dss.unpack(&answer, machine_name, &count, OPAL_STRING))) {
-        ORTE_ERROR_LOG(ret);
-        goto CLEANUP;
-    }
-
- CLEANUP:
-    OBJ_DESTRUCT(&answer);
-    OBJ_DESTRUCT(&request);
-
-    return ret;
-}
-
-
-/*
- * This function is paired with the filem_base_process_get_remote_path_cmd() function on the remote machine
- */
-int orte_filem_base_get_remote_path(char **remote_ref, orte_process_name_t *peer, int *flag) {
-    int ret, exit_status = ORTE_SUCCESS;
-    char *tmp_ref = NULL;
-    orte_std_cntr_t n;
-    opal_buffer_t request, answer;
-    int tmp_flag;
-    orte_filem_cmd_flag_t command=ORTE_FILEM_GET_REMOTE_PATH_CMD;
-
-    /*
-     * Ask for remote file information from the HNP
-     */
-    OBJ_CONSTRUCT(&request, opal_buffer_t);
-    OBJ_CONSTRUCT(&answer, opal_buffer_t);
-
-    if (ORTE_SUCCESS != (ret = opal_dss.pack(&request, &command, 1, ORTE_FILEM_CMD))) {
-        ORTE_ERROR_LOG(ret);
-        goto cleanup;
-    }
-
-    if (ORTE_SUCCESS != (ret = opal_dss.pack(&request, remote_ref, 1, OPAL_STRING))) {
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if (0 > (ret = orte_rml.send_buffer(peer, &request, ORTE_RML_TAG_FILEM_BASE, 0))) {
-        exit_status = ret;
-        goto cleanup;
-    }
-    
-    /*
-     * Get the response
-     */
-    if( 0 > (ret = orte_rml.recv_buffer(peer, &answer, ORTE_RML_TAG_FILEM_BASE_RESP, 0)) ) {
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    /*
-     * The absolute path for the remote file
-     */
-    n = 1;
-    if ( ORTE_SUCCESS != (ret = opal_dss.unpack(&answer, &tmp_ref, &n, OPAL_STRING)) ) {
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    /*
-     * The file type on the remote machine
-     */
-    n = 1;
-    if ( ORTE_SUCCESS != (ret = opal_dss.unpack(&answer, &tmp_flag, &n, OPAL_INT)) ) {
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if(NULL != *remote_ref)
-        free(*remote_ref);
-    
-    *remote_ref = strdup(tmp_ref);
-    *flag = tmp_flag;
-
- cleanup:
-    OBJ_DESTRUCT(&answer);
-    OBJ_DESTRUCT(&request);
-
-    if( NULL != tmp_ref)
-        free(tmp_ref);
-
-    return exit_status;
-}
-
-int orte_filem_base_prepare_request(orte_filem_base_request_t *request, int move_type)
-{
-    int num_reqs = 0, i = 0;
-
-    if( ORTE_FILEM_MOVE_TYPE_RM == move_type ) {
-        num_reqs = opal_list_get_size(&request->process_sets);
-    }
-    else {
-        num_reqs = opal_list_get_size(&request->process_sets) * opal_list_get_size(&request->file_sets);
-    }
-
-    if( 0 >= num_reqs ) {
-        return ORTE_ERROR;
-    }
-    else {
-        if( NULL != request->is_done ) {
-            free(request->is_done);
-            request->is_done = NULL;
-        }
-
-        if( NULL != request->is_active ) {
-            free(request->is_active);
-            request->is_active = NULL;
-        }
-
-        if( NULL != request->exit_status ) {
-            free(request->exit_status);
-            request->exit_status = NULL;
-        }
-
-        request->num_mv      = num_reqs;
-        request->is_done     = (bool*) malloc(sizeof(bool) * num_reqs);
-        request->is_active   = (bool*) malloc(sizeof(bool) * num_reqs);
-        request->exit_status = (int32_t*) malloc(sizeof(int32_t) * num_reqs);
-        for( i = 0; i < num_reqs; ++i) {
-            request->is_done[i]     = false;
-            request->is_active[i]   = false;
-            request->exit_status[i] = 0;
-        }
-    }
-
-    request->movement_type = move_type;
-
-    return ORTE_SUCCESS;
-}

orte/mca/filem/rsh/Makefile.am

-#
-# Copyright (c) 2004-2007 The Trustees of Indiana University.
-#                         All rights reserved.
-# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
-#                         All rights reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
-#                         University of Stuttgart.  All rights reserved.
-# Copyright (c) 2004-2005 The Regents of the University of California.
-#                         All rights reserved.
-# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
-# $COPYRIGHT$
-# 
-# Additional copyrights may follow
-# 
-# $HEADER$
-#
-
-dist_pkgdata_DATA = help-orte-filem-rsh.txt
-
-sources = \
-        filem_rsh.h \
-        filem_rsh_component.c \
-        filem_rsh_module.c
-
-# Make the output library in this directory, and name it either
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
-# (for static builds).
-
-if MCA_BUILD_orte_filem_rsh_DSO
-component_noinst =
-component_install = mca_filem_rsh.la
-else
-component_noinst = libmca_filem_rsh.la
-component_install =
-endif
-
-mcacomponentdir = $(pkglibdir)
-mcacomponent_LTLIBRARIES = $(component_install)
-mca_filem_rsh_la_SOURCES = $(sources)
-mca_filem_rsh_la_LDFLAGS = -module -avoid-version
-
-noinst_LTLIBRARIES = $(component_noinst)
-libmca_filem_rsh_la_SOURCES = $(sources)
-libmca_filem_rsh_la_LDFLAGS = -module -avoid-version

orte/mca/filem/rsh/filem_rsh.h

-/*
- * Copyright (c) 2004-2010 The Trustees of Indiana University.
- *                         All rights reserved.
- * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
- *                         All rights reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-/**
- * @file
- * 
- * RSH FILEM component
- *
- */
-
-#ifndef MCA_FILEM_RSH_EXPORT_H
-#define MCA_FILEM_RSH_EXPORT_H
-
-#include "orte_config.h"
-
-#include "opal/mca/mca.h"
-#include "orte/mca/filem/filem.h"
-
-BEGIN_C_DECLS
-
-#define ORTE_FILEM_RSH_ASK   0
-#define ORTE_FILEM_RSH_ALLOW 1
-#define ORTE_FILEM_RSH_DONE  2
-
-/*
- * Local Component structures
- */
-struct orte_filem_rsh_component_t {
-    /** Base FILEM component */
-    orte_filem_base_component_t super;
-
-    /** RSH cp command: rsh = rcp, ssh = scp */
-    char * cp_command;
-
-    /** Unix cp command */
-    char * cp_local_command;
-
-    /** SSH remote login command */	
-    char * remote_sh_command;
-};
-typedef struct orte_filem_rsh_component_t orte_filem_rsh_component_t;
-ORTE_MODULE_DECLSPEC extern orte_filem_rsh_component_t mca_filem_rsh_component;
-
-extern int orte_filem_rsh_max_incomming;
-extern int orte_filem_rsh_max_outgoing;
-extern int orte_filem_rsh_progress_meter;
-
-int orte_filem_rsh_component_query(mca_base_module_t **module, int *priority);
-
-/*
- * Module functions
- */
-int orte_filem_rsh_module_init(void);
-int orte_filem_rsh_module_finalize(void);
-
-int orte_filem_rsh_put(orte_filem_base_request_t *request);
-int orte_filem_rsh_put_nb(orte_filem_base_request_t *request);
-
-int orte_filem_rsh_get(orte_filem_base_request_t *request);
-int orte_filem_rsh_get_nb(orte_filem_base_request_t *request);
-
-int orte_filem_rsh_rm( orte_filem_base_request_t *request);
-int orte_filem_rsh_rm_nb( orte_filem_base_request_t *request);
-
-int orte_filem_rsh_wait( orte_filem_base_request_t *request);
-int orte_filem_rsh_wait_all( opal_list_t *request_list);
-
-END_C_DECLS
-
-#endif /* MCA_FILEM_RSH_EXPORT_H */

orte/mca/filem/rsh/filem_rsh_component.c

-/*
- * Copyright (c) 2004-2010 The Trustees of Indiana University.
- *                         All rights reserved.
- * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
- *                         All rights reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-#include "orte_config.h"
-#include "opal/util/output.h"
-#include "orte/constants.h"
-
-
-
-#include "orte/mca/filem/filem.h"
-#include "orte/mca/filem/base/base.h"
-#include "filem_rsh.h"
-
-/*
- * Public string for version number
- */
-const char *orte_filem_rsh_component_version_string = 
-"ORTE FILEM rsh MCA component version " ORTE_VERSION;
-
-/*
- * Local functionality
- */
-static int filem_rsh_register(void);
-static int filem_rsh_open(void);
-static int filem_rsh_close(void);
-
-int orte_filem_rsh_max_incomming = 10;
-int orte_filem_rsh_max_outgoing = 10;
-int orte_filem_rsh_progress_meter = 0;
-
-/*
- * Instantiate the public struct with all of our public information
- * and pointer to our public functions in it
- */
-orte_filem_rsh_component_t mca_filem_rsh_component = {
-    /* First do the base component stuff */
-    {
-        /* Handle the general mca_component_t struct containing 
-         *  meta information about the component itrsh
-         */
-        {
-            ORTE_FILEM_BASE_VERSION_2_0_0,
-            /* Component name and version */
-            "rsh",
-            ORTE_MAJOR_VERSION,
-            ORTE_MINOR_VERSION,
-            ORTE_RELEASE_VERSION,
-            
-            /* Component open and close functions */
-            filem_rsh_open,
-            filem_rsh_close,
-            orte_filem_rsh_component_query,
-            filem_rsh_register
-        },
-        {
-            /* The component is checkpoint ready */
-            MCA_BASE_METADATA_PARAM_CHECKPOINT
-        },
-    },
-
-    /* cp_command */
-    NULL,
-
-    /* cp_local_command */
-    NULL,
-
-    /* remote_sh_command */
-    NULL
-};
-
-static int filem_rsh_register(void)
-{
-    mca_base_component_t *component = &mca_filem_rsh_component.super.base_version;
-    mca_filem_rsh_component.cp_command = "scp";
-    (void) mca_base_component_var_register(component, "rcp",
-                                           "The rsh cp command for the FILEM rsh component",
-                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_filem_rsh_component.cp_command);
-
-    mca_filem_rsh_component.cp_local_command = "cp";
-    (void) mca_base_component_var_register(component, "cp",
-                                           "The Unix cp command for the FILEM rsh component",
-                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_filem_rsh_component.cp_local_command);
-
-    mca_filem_rsh_component.remote_sh_command = "ssh";
-    (void) mca_base_component_var_register(component, "rsh",
-                                           "The remote shell command for the FILEM rsh component",
-                                           MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_filem_rsh_component.remote_sh_command);
-
-    orte_filem_rsh_max_incomming = 10;
-    (void) mca_base_component_var_register(component, "max_incomming",
-                                           "Maximum number of incomming connections (0 = any)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &orte_filem_rsh_max_incomming);
-
-    orte_filem_rsh_max_outgoing = 10;
-    (void) mca_base_component_var_register(component, "max_outgoing",
-                                           "Maximum number of out going connections (0 = any)",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &orte_filem_rsh_max_outgoing);
-
-    orte_filem_rsh_progress_meter = 0;
-    (void) mca_base_component_var_register(component, "progress_meter",
-                                           "Display Progress every X percentage done. [Default = 0/off]",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
-                                           OPAL_INFO_LVL_9,
-                                           MCA_BASE_VAR_SCOPE_READONLY,
-                                           &orte_filem_rsh_progress_meter);
-
-    return ORTE_SUCCESS;
-}
-
-static int filem_rsh_open(void) 
-{
-
-    if( orte_filem_rsh_max_incomming < 0 ) {
-        orte_filem_rsh_max_incomming = 1;
-    }
-
-    if( orte_filem_rsh_max_outgoing < 0 ) {
-        orte_filem_rsh_max_outgoing = 1;
-    }
-
-    orte_filem_rsh_progress_meter = (orte_filem_rsh_progress_meter % 101);
-
-    /*
-     * Debug Output
-     */
-    opal_output_verbose(10, orte_filem_base_framework.framework_output,
-                        "filem:rsh: open()");
-    opal_output_verbose(20, orte_filem_base_framework.framework_output,
-                        "filem:rsh: open: cp command  = %s", 
-                        mca_filem_rsh_component.cp_command);
-    opal_output_verbose(20, orte_filem_base_framework.framework_output,
-                        "filem:rsh: open: cp local command  = %s", 
-                        mca_filem_rsh_component.cp_local_command);
-    opal_output_verbose(20, orte_filem_base_framework.framework_output,
-                        "filem:rsh: open: rsh command  = %s", 
-                        mca_filem_rsh_component.remote_sh_command);
-
-    return ORTE_SUCCESS;
-}
-
-static int filem_rsh_close(void)
-{
-    opal_output_verbose(10, orte_filem_base_framework.framework_output,
-                        "filem:rsh: close()");
-
-    return ORTE_SUCCESS;
-}

orte/mca/filem/rsh/filem_rsh_module.c

-/*
- * Copyright (c) 2004-2010 The Trustees of Indiana University.
- *                         All rights reserved.
- * Copyright (c) 2004-2011 The Trustees of the University of Tennessee.
- *                         All rights reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-/*
- *
- */
-
-#include "orte_config.h"
-
-#ifdef HAVE_STRING_H
-#include <string.h>
-#endif
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/wait.h>
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif  /* HAVE_UNISTD_H */
-#if HAVE_TIME_H
-#include <time.h>
-#endif
-#if HAVE_SYS_TIME_H
-#include <sys/time.h>
-#endif
-
-#include "opal/mca/mca.h"
-#include "opal/mca/base/base.h"
-
-#include "opal/mca/event/event.h"
-
-#include "orte/constants.h"
-#include "orte/util/show_help.h"
-#include "opal/util/argv.h"
-#include "opal/util/output.h"
-#include "opal/util/opal_environ.h"
-#include "opal/util/basename.h"
-
-#include "opal/threads/mutex.h"
-#include "opal/threads/condition.h"
-
-#include "orte/util/name_fns.h"
-#include "orte/util/proc_info.h"
-#include "orte/runtime/orte_globals.h"
-#include "orte/runtime/orte_wait.h"
-#include "orte/mca/errmgr/errmgr.h"
-#include "orte/mca/rml/rml_types.h"
-
-#include "orte/mca/filem/filem.h"
-#include "orte/mca/filem/base/base.h"
-
-#include "filem_rsh.h"
-
-/**********
- * Local Function and Variable Declarations
- **********/
-static int  orte_filem_rsh_start_copy(orte_filem_base_request_t *request);
-static int  orte_filem_rsh_start_rm(orte_filem_base_request_t *request);
-
-static int  orte_filem_rsh_start_command(orte_filem_base_process_set_t *proc_set,
-                                         orte_filem_base_file_set_t    *file_set,
-                                         char * command,
-                                         orte_filem_base_request_t *request,
-                                         int index);
-static int start_child(char * command,
-                       orte_filem_base_request_t *request,
-                       int index);
-
-static int  orte_filem_rsh_query_remote_path(char **remote_ref,
-                                             orte_process_name_t *proc,
-                                             int *flag);
-
-static void filem_rsh_waitpid_cb(pid_t pid, int status, void* cbdata);
-
-/* Permission to send functionality */
-static int orte_filem_rsh_permission_listener_init(void);
-static int orte_filem_rsh_permission_listener_cancel(void);
-static void orte_filem_rsh_permission_callback(int status,
-                                               orte_process_name_t* sender,
-                                               opal_buffer_t *buffer,
-                                               orte_rml_tag_t tag,
-                                               void* cbdata);
-static int orte_filem_rsh_permission_ask(orte_process_name_t* sender, int num_sends);
-static int permission_send_done(orte_process_name_t* sender, int num_avail,
-                                int32_t exit_status,
-                                char * local_target,
-                                char * remote_target,
-                                char * command);
-static int permission_send_num_allowed(orte_process_name_t* sender, int num_allowed);
-
-
-/*************
- * Local work pool structure
- *************/
-int cur_num_incomming = 0;
-int cur_num_outgoing  = 0;
-
-static bool work_pool_all_done = false;
-static opal_mutex_t     work_pool_lock;
-static opal_condition_t work_pool_cond;
-
-/*
- * work_pool_waiting:
- *  - processes that are waiting for my permission to put() to me
- */
-opal_list_t work_pool_waiting;
-/*
- * work_pool_held:
- *  - requests that are held before asking permission to reduce load
- */
-opal_list_t work_pool_held;
-/*
- * work_pool_pending:
- *  - put requests waiting on permission to send to peer
- */
-opal_list_t work_pool_pending;
-/*
- * work_pool_active:
- * - put requests currently sending
- */
-opal_list_t work_pool_active;
-
-struct orte_filem_rsh_work_pool_item_t {
-    /** This is an object, so must have a super */
-    opal_list_item_t super;
-
-    /** Command to exec */
-    char * command;
-
-    /** Pointer to FileM Request */
-    orte_filem_base_request_t *request;
-
-    /** Index into the request */
-    int index;
-
-    /** Process Set */
-    orte_filem_base_process_set_t proc_set;
-
-    /** File Set */
-    orte_filem_base_file_set_t file_set;
-
-    /** If this item is active */
-    bool active;
-};
-typedef struct orte_filem_rsh_work_pool_item_t orte_filem_rsh_work_pool_item_t;
-OBJ_CLASS_DECLARATION(orte_filem_rsh_work_pool_item_t);
-
-void orte_filem_rsh_work_pool_construct(orte_filem_rsh_work_pool_item_t *obj);
-void orte_filem_rsh_work_pool_destruct( orte_filem_rsh_work_pool_item_t *obj);
-
-OBJ_CLASS_INSTANCE(orte_filem_rsh_work_pool_item_t,
-                   opal_list_item_t,
-                   orte_filem_rsh_work_pool_construct,
-                   orte_filem_rsh_work_pool_destruct);
-
-void orte_filem_rsh_work_pool_construct(orte_filem_rsh_work_pool_item_t *obj) {
-    obj->command = NULL;
-    obj->request = NULL;
-    obj->index = 0;
-
-    OBJ_CONSTRUCT(&(obj->proc_set), orte_filem_base_process_set_t);
-    OBJ_CONSTRUCT(&(obj->file_set), orte_filem_base_file_set_t);
-
-    obj->active = false;
-}
-
-void orte_filem_rsh_work_pool_destruct( orte_filem_rsh_work_pool_item_t *obj) {
-    if( NULL != obj->command ) {
-        free(obj->command);
-        obj->command = NULL;
-    }
-
-    if( NULL != obj->request ) {
-        OBJ_RELEASE(obj->request);
-        obj->request = NULL;
-    }
-
-    obj->index = 0;
-
-    OBJ_DESTRUCT(&(obj->proc_set));
-    OBJ_DESTRUCT(&(obj->file_set));
-
-    obj->active = false;
-}
-
-/* placeholders */
-static int rsh_preposition_files(orte_job_t *jdata,
-                                 orte_filem_completion_cbfunc_t cbfunc,
-                                 void *cbdata)
-{
-    /* nothing to do here, so fire the callback and return */
-    if (NULL != cbfunc) {
-        cbfunc(ORTE_SUCCESS, cbdata);
-    }
-    return ORTE_SUCCESS;
-}
-static int rsh_link_local_files(orte_job_t *jdata,
-                                orte_app_context_t *app)
-{
-    return ORTE_SUCCESS;
-}
-
-/*
- * Rsh module
- */
-static orte_filem_base_module_t loc_module = {
-    /** Initialization Function */
-    orte_filem_rsh_module_init,
-    /** Finalization Function */
-    orte_filem_rsh_module_finalize,
-
-    orte_filem_rsh_put,
-    orte_filem_rsh_put_nb,
-
-    orte_filem_rsh_get,
-    orte_filem_rsh_get_nb,
-
-    orte_filem_rsh_rm,
-    orte_filem_rsh_rm_nb,
-
-    orte_filem_rsh_wait,
-    orte_filem_rsh_wait_all,
-
-    rsh_preposition_files,
-    rsh_link_local_files
-};
-
-/*
- * MCA Functions
- */
-int orte_filem_rsh_component_query(mca_base_module_t **module, int *priority)
-{
-    OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                         "filem:rsh: component_query()"));
-
-    *priority = 20;
-    *module = (mca_base_module_t *)&loc_module;
-
-    return ORTE_SUCCESS;
-}
-
-int orte_filem_rsh_module_init(void)
-{
-    int ret;
-
-    orte_filem_base_is_active = false;
-
-    OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                         "filem:rsh: module_init()"));
-
-    /*
-     * Allocate the work pools
-     */
-    OBJ_CONSTRUCT(&work_pool_waiting, opal_list_t);
-    OBJ_CONSTRUCT(&work_pool_held,    opal_list_t);
-    OBJ_CONSTRUCT(&work_pool_pending, opal_list_t);
-    OBJ_CONSTRUCT(&work_pool_active,  opal_list_t);
-
-    OBJ_CONSTRUCT(&work_pool_lock, opal_mutex_t);
-    OBJ_CONSTRUCT(&work_pool_cond, opal_condition_t);
-
-    work_pool_all_done = false;
-
-    /*
-     * Start the listener for permission
-     */
-    if( ORTE_SUCCESS != (ret = orte_filem_rsh_permission_listener_init())) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh:init Failed to start listener\n");
-        return ret;
-    }
-    
-    /* start the base receive */
-    if (ORTE_SUCCESS != (ret = orte_filem_base_comm_start())) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh:init Failed to start base receive\n");
-        return ret;
-    }
-    return ORTE_SUCCESS;
-}
-
-int orte_filem_rsh_module_finalize(void)
-{
-    opal_list_item_t *item = NULL;
-
-    OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                         "filem:rsh: module_finalize()"));
-
-    /*
-     * Make sure all active requests are completed
-     */
-    if( orte_filem_base_is_active ) {
-        while(0 < opal_list_get_size(&work_pool_active) ) {
-#if ORTE_ENABLE_PROGRESS_THREADS
-            {
-                /* provide a very short quiet period so we
-                 * don't hammer the cpu while we wait
-                 */
-                struct timespec tp = {0, 100};
-                nanosleep(&tp, NULL);
-            }
-#else
-            opal_progress();
-#endif
-        }
-    }
-
-    orte_filem_base_is_active = false;
-
-    /*
-     * Stop the listeners
-     */
-    orte_filem_rsh_permission_listener_cancel();
-    
-    /*
-     * Stop the base receive
-     */
-    orte_filem_base_comm_stop();
-
-    /*
-     * Deallocate the work pools
-     */
-    while( NULL != (item = opal_list_remove_first(&work_pool_waiting)) ) {
-        OBJ_RELEASE(item);
-    }
-    OBJ_DESTRUCT(&work_pool_waiting);
-
-    while( NULL != (item = opal_list_remove_first(&work_pool_held)) ) {
-        OBJ_RELEASE(item);
-    }
-    OBJ_DESTRUCT(&work_pool_held);
-
-    while( NULL != (item = opal_list_remove_first(&work_pool_pending)) ) {
-        OBJ_RELEASE(item);
-    }
-    OBJ_DESTRUCT(&work_pool_pending);
-
-    while( NULL != (item = opal_list_remove_first(&work_pool_active)) ) {
-        OBJ_RELEASE(item);
-    }
-    OBJ_DESTRUCT(&work_pool_active);
-
-    OBJ_DESTRUCT(&work_pool_lock);
-    OBJ_DESTRUCT(&work_pool_cond);
-
-    return ORTE_SUCCESS;
-}
-
-/******************
- * Local functions
- ******************/
-int orte_filem_rsh_put(orte_filem_base_request_t *request)
-{
-    int ret, exit_status = ORTE_SUCCESS;
-
-    orte_filem_base_is_active = true;
-
-    if( ORTE_SUCCESS != (ret = orte_filem_base_prepare_request(request, ORTE_FILEM_MOVE_TYPE_PUT) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: put(): Failed to prepare the request structure (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if( ORTE_SUCCESS != (ret = orte_filem_rsh_start_copy(request) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: put(): Failed to post the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if( ORTE_SUCCESS != (ret = orte_filem_rsh_wait(request)) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: put(): Failed to wait on the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
- cleanup:
-    if( 0 < opal_list_get_size(&work_pool_active) ) {
-        orte_filem_base_is_active = true;
-    } else {
-        orte_filem_base_is_active = false;
-    }
-
-    return exit_status;
-}
-
-int orte_filem_rsh_put_nb(orte_filem_base_request_t *request)
-{
-    int ret, exit_status = ORTE_SUCCESS;
-
-    orte_filem_base_is_active = true;
-
-    if( ORTE_SUCCESS != (ret = orte_filem_base_prepare_request(request, ORTE_FILEM_MOVE_TYPE_PUT) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: put(): Failed to prepare the request structure (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if( ORTE_SUCCESS != (ret = orte_filem_rsh_start_copy(request) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: put(): Failed to post the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
- cleanup:
-    if( 0 < opal_list_get_size(&work_pool_active) ) {
-        orte_filem_base_is_active = true;
-    } else {
-        orte_filem_base_is_active = false;
-    }
-
-    return exit_status;
-}
-
-int orte_filem_rsh_get(orte_filem_base_request_t *request)
-{
-    int ret, exit_status = ORTE_SUCCESS;
-
-    orte_filem_base_is_active = true;
-
-    if( ORTE_SUCCESS != (ret = orte_filem_base_prepare_request(request, ORTE_FILEM_MOVE_TYPE_GET) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: get(): Failed to prepare the request structure (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if( ORTE_SUCCESS != (ret = orte_filem_rsh_start_copy(request) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: get(): Failed to post the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if( ORTE_SUCCESS != (ret = orte_filem_rsh_wait(request)) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: get(): Failed to wait on the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
- cleanup:
-    if( 0 < opal_list_get_size(&work_pool_active) ) {
-        orte_filem_base_is_active = true;
-    } else {
-        orte_filem_base_is_active = false;
-    }
-
-    return exit_status;
-}
-
-int orte_filem_rsh_get_nb(orte_filem_base_request_t *request)
-{
-    int ret, exit_status = ORTE_SUCCESS;
-
-    orte_filem_base_is_active = true;
-
-    if( ORTE_SUCCESS != (ret = orte_filem_base_prepare_request(request, ORTE_FILEM_MOVE_TYPE_GET) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: get(): Failed to prepare the request structure (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if( ORTE_SUCCESS != (ret = orte_filem_rsh_start_copy(request) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: get(): Failed to post the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
- cleanup:
-    if( 0 < opal_list_get_size(&work_pool_active) ) {
-        orte_filem_base_is_active = true;
-    } else {
-        orte_filem_base_is_active = false;
-    }
-
-    return exit_status;
-}
- 
-int orte_filem_rsh_rm(orte_filem_base_request_t *request)
-{
-    int ret = ORTE_SUCCESS, exit_status = ORTE_SUCCESS;
-
-    orte_filem_base_is_active = true;
-
-    if( ORTE_SUCCESS != (ret = orte_filem_base_prepare_request(request, ORTE_FILEM_MOVE_TYPE_RM) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: rm(): Failed to prepare on the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if( ORTE_SUCCESS != (ret = orte_filem_rsh_start_rm(request) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: rm(): Failed to start the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if( ORTE_SUCCESS != (ret = orte_filem_rsh_wait(request)) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: rm(): Failed to wait on the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
- cleanup:
-    if( 0 < opal_list_get_size(&work_pool_active) ) {
-        orte_filem_base_is_active = true;
-    } else {
-        orte_filem_base_is_active = false;
-    }
-
-    return exit_status;
-}
-
-int orte_filem_rsh_rm_nb(orte_filem_base_request_t *request)
-{
-    int ret = ORTE_SUCCESS, exit_status = ORTE_SUCCESS;
-
-    orte_filem_base_is_active = true;
-
-    if( ORTE_SUCCESS != (ret = orte_filem_base_prepare_request(request, ORTE_FILEM_MOVE_TYPE_RM) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: rm_nb(): Failed to prepare on the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
-    if( ORTE_SUCCESS != (ret = orte_filem_rsh_start_rm(request) ) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: rm_nb(): Failed to start on the request (%d)", ret);
-        exit_status = ret;
-        goto cleanup;
-    }
-
- cleanup:
-    if( 0 < opal_list_get_size(&work_pool_active) ) {
-        orte_filem_base_is_active = true;
-    } else {
-        orte_filem_base_is_active = false;
-    }
-
-    return exit_status;
-}
-
-int orte_filem_rsh_wait(orte_filem_base_request_t *request)
-{
-    int exit_status = ORTE_SUCCESS;
-    orte_filem_rsh_work_pool_item_t *wp_item = NULL;
-    opal_list_item_t *item = NULL;
-    int i;
-    int num_finished = 0;
-    bool found_match = false;
-
-    /*
-     * Stage 0: A pass to see if the request is already done
-     */
-    for(i = 0; i < request->num_mv; ++i) {
-        if( request->is_done[i]   == true &&
-            request->is_active[i] == true) {
-            ++num_finished;
-        }
-    }
-
-    while(num_finished < request->num_mv ) {
-        /*
-         * Stage 1: Complete all active requests
-         */
-        for(i = 0; i < request->num_mv; ++i) {
-            /* If the child is still executing (active) then continue
-             * checking other children
-             */
-            if( request->is_done[i]   == false && 
-                request->is_active[i] == true) {
-                continue;
-            }
-            /* If the child is done executing (!active), but has not been
-             * collected then do so
-             */
-            else if( request->is_done[i]   == true && 
-                     request->is_active[i] == false) {
-                /* 
-                 * Find the reference in the active pool
-                 */
-                found_match = false;
-                for (item  = opal_list_get_first( &work_pool_active);
-                     item != opal_list_get_end(   &work_pool_active);
-                     item  = opal_list_get_next(   item) ) {
-                    wp_item = (orte_filem_rsh_work_pool_item_t *)item;
-                    if(request == wp_item->request &&
-                       i       == wp_item->index) {
-                        found_match = true;
-                        break;
-                    }
-                }
-
-                /* If no match then assume on the pending list, and continue */
-                if( !found_match ) {
-                    continue;
-                }
-
-                OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                     "filem:rsh: wait(): Transfer complete. Cleanup\n"));
-
-                opal_list_remove_item(&work_pool_active, item);
-
-                /* Mark as fully done [active = true, done = true]
-                 * It does not make complete sense to call this 'active' when
-                 * it is obviously not, but this is a state [true, true] that
-                 * should only be reached if the transfer has finished
-                 * completely.
-                 */
-                request->is_done[i]   = true;
-                request->is_active[i] = true;
-
-                /* Tell peer we are finished with a send */
-                permission_send_done(&(wp_item->proc_set.source), 1, request->exit_status[i],
-                                     wp_item->file_set.local_target,
-                                     wp_item->file_set.remote_target,
-                                     wp_item->command);
-
-                OBJ_RELEASE(wp_item);
-                wp_item = NULL;
-
-                ++num_finished;
-            }
-        }
-
-        /*
-         * Wait for a child to complete
-         */
-        if( num_finished < request->num_mv ) {
-            OPAL_THREAD_LOCK(&work_pool_lock);
-            opal_condition_wait(&work_pool_cond,
-                                &work_pool_lock);
-            OPAL_THREAD_UNLOCK(&work_pool_lock);
-        }
-    }
-
-    /*
-     * Stage 2: Determine the return value
-     */
-    for(i = 0; i < request->num_mv; ++i) {
-        if( request->exit_status[i] < 0 ) {
-            exit_status = request->exit_status[i];
-        }
-    }
-
-    if( 0 < opal_list_get_size(&work_pool_active) ) {
-        orte_filem_base_is_active = true;
-    } else {
-        orte_filem_base_is_active = false;
-    }
-
-    return exit_status;
-}
-
-int orte_filem_rsh_wait_all(opal_list_t * request_list)
-{
-    int ret = ORTE_SUCCESS, exit_status = ORTE_SUCCESS;
-    opal_list_item_t *item = NULL;
-    double perc_done, last_reported = 0.0;
-    int total, done;
-
-    total = opal_list_get_size(request_list);
-    done  = 0;
-
-    for (item  = opal_list_get_first( request_list);
-         item != opal_list_get_end(   request_list);
-         item  = opal_list_get_next(  item) ) {
-        orte_filem_base_request_t *request = (orte_filem_base_request_t *) item;
-
-        if( ORTE_SUCCESS != (ret = orte_filem_rsh_wait(request)) ) {
-            opal_output(orte_filem_base_framework.framework_output,
-                        "filem:rsh: wait_all(): Wait failed (%d)", ret);
-            exit_status = ret;
-            goto cleanup;
-        }
-
-        /* Progress Meter */
-        if( OPAL_UNLIKELY(orte_filem_rsh_progress_meter > 0) ) {
-            ++done;
-            perc_done = (total - done) / (1.0 * total);
-            perc_done = (perc_done-1)*(-100.0);
-            if( perc_done >= (last_reported + orte_filem_rsh_progress_meter) || last_reported == 0.0 ) {
-                last_reported = perc_done;
-                opal_output(0, "filem:rsh: progress:    %10.2f %c Finished\n",
-                            perc_done, '%');
-            }
-        }
-    }
-
- cleanup:
-    if( 0 < opal_list_get_size(&work_pool_active) ) {
-        orte_filem_base_is_active = true;
-    } else {
-        orte_filem_base_is_active = false;
-    }
-
-    return exit_status;
-}
-
-/**************************
- * Support functions
- **************************/
-static int orte_filem_rsh_start_copy(orte_filem_base_request_t *request) {
-    int ret = ORTE_SUCCESS, exit_status = ORTE_SUCCESS;
-    opal_list_item_t *f_item = NULL;
-    opal_list_item_t *p_item = NULL;
-    char *remote_machine = NULL;
-    char *remote_file    = NULL;
-    char *command        = NULL;
-    char *dir_arg        = NULL;
-    int cur_index = 0;
-
-    /* For each file pair */
-    for (f_item  = opal_list_get_first( &request->file_sets);
-         f_item != opal_list_get_end(   &request->file_sets);
-         f_item  = opal_list_get_next(   f_item) ) {
-        orte_filem_base_file_set_t * f_set = (orte_filem_base_file_set_t*)f_item;
-
-        /* For each process set */
-        for (p_item  = opal_list_get_first( &request->process_sets);
-             p_item != opal_list_get_end(   &request->process_sets);
-             p_item  = opal_list_get_next(   p_item) ) {
-            orte_filem_base_process_set_t * p_set = (orte_filem_base_process_set_t*)p_item;
-
-
-            /*
-             * If the source and sink are the same, then this is a local operation
-             * Further if the files are the same, then nothing to do
-             */
-            if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, &p_set->source, &p_set->sink) ) {
-                if( 0 == strncmp(f_set->local_target, f_set->remote_target, strlen(f_set->remote_target) ) ) {
-                    request->is_done[cur_index]     = true;
-                    request->is_active[cur_index]   = true;
-                    request->exit_status[cur_index] = 0;
-                    goto continue_set;
-                }
-            }
-
-            if( request->movement_type == ORTE_FILEM_MOVE_TYPE_PUT ) {
-                /*
-                 * The file should exist if we are going to put it somewhere else
-                 */
-                if( 0 != access(f_set->local_target, R_OK) ) {
-                    OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                         "filem:rsh: copy(): %s -> %s: Error: Cannot move file %s to %s. Does not exist at source\n",
-                                         ORTE_NAME_PRINT(&p_set->source),
-                                         ORTE_NAME_PRINT(&p_set->sink),
-                                         f_set->local_target,
-                                         f_set->remote_target));
-                    orte_show_help("help-orte-filem-rsh.txt",
-                                   "orte-filem-rsh:get-file-not-exist",
-                                   true, f_set->local_target, orte_process_info.nodename);
-                    request->is_done[cur_index]     = true;
-                    request->is_active[cur_index]   = true;
-                    request->exit_status[cur_index] = -1;
-                    goto continue_set;
-                }
-            }
-            /* Do not check a local get() operation, to help supress the warnings from the HNP */
-            else if (OPAL_EQUAL != orte_util_compare_name_fields(ORTE_NS_CMP_ALL, &p_set->source, &p_set->sink) ) {
-                char *base = NULL;
-                asprintf(&base, "%s/%s", f_set->local_target, opal_basename(f_set->remote_target));
-                /*
-                 * The file should not exist if we are getting a file with the
-                 * same name since we do not want to overwrite the filename
-                 * without the users consent.
-                 */
-                if( 0 == access(base, R_OK) ) {
-                    OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                         "filem:rsh: copy(): %s -> %s: Error: Cannot move file %s to %s. Already exists at destination (%s)\n",
-                                         ORTE_NAME_PRINT(&p_set->source),
-                                         ORTE_NAME_PRINT(&p_set->sink),
-                                         f_set->remote_target,
-                                         f_set->local_target, base));
-                    orte_show_help("help-orte-filem-rsh.txt",
-                                   "orte-filem-rsh:get-file-exists",
-                                   true, f_set->local_target, orte_process_info.nodename);
-                    free(base);
-                    base = NULL;
-                    request->is_done[cur_index]     = true;
-                    request->is_active[cur_index]   = true;
-                    request->exit_status[cur_index] = -1;
-                    goto continue_set;
-                }
-                free(base);
-                base = NULL;
-            }
-
-            if( request->movement_type == ORTE_FILEM_MOVE_TYPE_PUT ) {
-                OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                     "filem:rsh: copy(): %s -> %s: Moving file %s %s to %s %s\n",
-                                     ORTE_NAME_PRINT(&p_set->source),
-                                     ORTE_NAME_PRINT(&p_set->sink),
-                                     (f_set->local_hint == ORTE_FILEM_HINT_SHARED ? "(S)" : ""),
-                                     f_set->local_target,
-                                     (f_set->remote_hint == ORTE_FILEM_HINT_SHARED ? "(S)" : ""),
-                                     f_set->remote_target));
-            } else {
-                OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                     "filem:rsh: copy(): %s -> %s: Moving file %s %s to %s %s\n",
-                                     ORTE_NAME_PRINT(&p_set->source),
-                                     ORTE_NAME_PRINT(&p_set->sink),
-                                     (f_set->remote_hint == ORTE_FILEM_HINT_SHARED ? "(S)" : ""),
-                                     f_set->remote_target,
-                                     (f_set->local_hint == ORTE_FILEM_HINT_SHARED ? "(S)" : ""),
-                                     f_set->local_target));
-            }
-
-            /*
-             * Get the remote machine identifier from the process_name struct
-             */
-            OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                 "filem:rsh: copy(): %s -> %s: Get node name.\n",
-                                 ORTE_NAME_PRINT(&p_set->source),
-                                 ORTE_NAME_PRINT(&p_set->sink)));
-            if( ORTE_SUCCESS != (ret = orte_filem_base_get_proc_node_name(&p_set->source, &remote_machine))) {
-                opal_output(orte_filem_base_framework.framework_output,
-                            "filem:rsh: copy(): Get Node Name failed (%d)", ret);
-                exit_status = ret;
-                goto cleanup;
-            }
-            OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                 "filem:rsh: copy(): %s -> %s: Got node name: %s\n",
-                                 ORTE_NAME_PRINT(&p_set->source),
-                                 ORTE_NAME_PRINT(&p_set->sink),
-                                 remote_machine));
-
-            /*
-             * Fix the remote_filename.
-             * If it is an absolute path, then assume it is valid for the remote server
-             * ow then we must construct the correct path.
-             */
-            OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                 "filem:rsh: copy(): %s -> %s: Query remote path (%s).\n",
-                                 ORTE_NAME_PRINT(&p_set->source),
-                                 ORTE_NAME_PRINT(&p_set->sink),
-                                 f_set->remote_target));
-            remote_file = strdup(f_set->remote_target);
-            if( ORTE_SUCCESS != (ret = orte_filem_rsh_query_remote_path(&remote_file, &p_set->source, &f_set->target_flag) ) ) {
-                opal_output(orte_filem_base_framework.framework_output,
-                            "filem:rsh: copy(): Query Remote Path failed (%d)", ret);
-                exit_status = ret;
-                goto cleanup;
-            }
-            OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                 "filem:rsh: copy(): %s -> %s: Remote path (%s) is (%s).\n",
-                                 ORTE_NAME_PRINT(&p_set->source),
-                                 ORTE_NAME_PRINT(&p_set->sink),
-                                 f_set->remote_target,
-                                 remote_file));
-
-            /*
-             * Transfer the file or directory
-             */
-            if(ORTE_FILEM_TYPE_DIR == f_set->target_flag) {
-                dir_arg = strdup(" -r ");
-            }
-            else if(ORTE_FILEM_TYPE_UNKNOWN == f_set->target_flag) {
-                opal_output(orte_filem_base_framework.framework_output,
-                            "filem:rsh: copy(): Error: File type unknown (%s)",
-                            f_set->remote_target);
-                request->is_done[cur_index]     = true;
-                request->is_active[cur_index]   = true;
-                request->exit_status[cur_index] = -1;
-                goto continue_set;
-            }
-            else {
-                dir_arg = strdup("");
-            }
-
-            /*
-             * If this is the put() routine
-             */
-            if( request->movement_type == ORTE_FILEM_MOVE_TYPE_PUT ) {
-                /* Use a local 'cp' when able */
-                if(f_set->remote_hint == ORTE_FILEM_HINT_SHARED ) {
-                    asprintf(&command, "%s %s %s %s ",
-                             mca_filem_rsh_component.cp_local_command, 
-                             dir_arg, 
-                             f_set->local_target,
-                             remote_file);
-                } else {
-                    asprintf(&command, "%s %s %s %s:%s ",
-                             mca_filem_rsh_component.cp_command, 
-                             dir_arg, 
-                             f_set->local_target,
-                             remote_machine, 
-                             remote_file);
-                }
-            }
-            /*
-             * ow it is the get() routine
-             */
-            else {
-                /* Use a local 'cp' when able */
-                if(f_set->local_hint == ORTE_FILEM_HINT_SHARED ) {
-                    asprintf(&command, "%s %s %s %s %s %s ",
-                             mca_filem_rsh_component.remote_sh_command, 
-                             remote_machine, 
-                             mca_filem_rsh_component.cp_local_command,
-                             dir_arg, 
-                             remote_file,
-                             f_set->local_target);
-                } else {
-                    asprintf(&command, "%s %s %s:%s %s ",
-                             mca_filem_rsh_component.cp_command, 
-                             dir_arg, 
-                             remote_machine, 
-                             remote_file,
-                             f_set->local_target);
-                }
-            }
-
-            /*
-             * Start the command
-             */
-            OPAL_OUTPUT_VERBOSE((17, orte_filem_base_framework.framework_output,
-                                 "filem:rsh:%s about to execute [%s]",
-                                 (request->movement_type == ORTE_FILEM_MOVE_TYPE_PUT ? "put" : "get"),
-                                 command));
-            if( ORTE_SUCCESS != (ret = orte_filem_rsh_start_command(p_set,
-                                                                    f_set,
-                                                                    command,
-                                                                    request,
-                                                                    cur_index)) ) {
-                exit_status = ret;
-                goto cleanup;
-            }
-
-        continue_set:
-            /* A small bit of cleanup */
-            if( NULL != dir_arg) {
-                free(dir_arg);
-                dir_arg = NULL;
-            }
-
-            if( NULL != remote_file) {
-                free(remote_file);
-                remote_file = NULL;
-            }
-
-            if(NULL != remote_machine) {
-                free(remote_machine);
-                remote_machine = NULL;
-            }
-
-            ++cur_index;
-        } /* For each process set */
-    } /* For each file pair */
-
- cleanup:
-    if( NULL != command )
-        free(command);
-    if( NULL != remote_machine)
-        free(remote_machine);
-    if( NULL != dir_arg) 
-        free(dir_arg);
-    if( NULL != remote_file)
-        free(remote_file);
-        
-    return exit_status;
-}
-
-static int orte_filem_rsh_start_rm(orte_filem_base_request_t *request)
-{
-    int ret = ORTE_SUCCESS, exit_status = ORTE_SUCCESS;
-    opal_list_item_t *f_item = NULL;
-    opal_list_item_t *p_item = NULL;
-    char *command        = NULL;
-    char *remote_machine = NULL;
-    char *remote_targets = NULL;
-    char *remote_file    = NULL;
-    char *dir_arg        = NULL;
-    char **remote_file_set = NULL;
-    int argc = 0;
-    int cur_index = 0;
-
-    /* For each process set */
-    for (p_item  = opal_list_get_first( &request->process_sets);
-         p_item != opal_list_get_end(   &request->process_sets);
-         p_item  = opal_list_get_next(   p_item) ) {
-        orte_filem_base_process_set_t * p_set = (orte_filem_base_process_set_t*)p_item;
-
-        /*
-         * Get the remote machine identifier from the process_name struct
-         */
-        if( ORTE_SUCCESS != (ret = orte_filem_base_get_proc_node_name(&p_set->source, &remote_machine))) {
-            exit_status = ret;
-            goto cleanup;
-        }
-
-        /* For each file pair */
-        for (f_item  = opal_list_get_first( &request->file_sets);
-             f_item != opal_list_get_end(   &request->file_sets);
-             f_item  = opal_list_get_next(   f_item) ) {
-            orte_filem_base_file_set_t * f_set = (orte_filem_base_file_set_t*)f_item;
-
-            /*
-             * Fix the remote_filename.
-             * If it is an absolute path, then assume it is valid for the remote server
-             * ow then we must construct the correct path.
-             */
-            remote_file = strdup(f_set->remote_target);
-            if( ORTE_SUCCESS != (ret = orte_filem_rsh_query_remote_path(&remote_file, &p_set->source, &f_set->target_flag) ) ) {
-                exit_status = ret;
-                goto cleanup;
-            }
-
-            if(ORTE_FILEM_TYPE_UNKNOWN == f_set->target_flag) {
-                continue;
-            }
-
-            opal_argv_append(&argc, &remote_file_set, remote_file);
-
-            /*
-             * If we are removing a directory in the mix, then we
-             * need the recursive argument.
-             */
-            if(NULL == dir_arg) {
-                if(ORTE_FILEM_TYPE_DIR == f_set->target_flag) {
-                    dir_arg = strdup(" -rf ");
-                }
-            }
-        } /* All File Pairs */
-
-        if(NULL == dir_arg) {
-            dir_arg = strdup(" -f ");
-        }
-        
-        remote_targets = opal_argv_join(remote_file_set, ' ');
-
-        asprintf(&command, "%s %s rm %s %s ",
-                 mca_filem_rsh_component.remote_sh_command, 
-                 remote_machine,
-                 dir_arg,
-                 remote_targets);
-
-        OPAL_OUTPUT_VERBOSE((15, orte_filem_base_framework.framework_output,
-                             "filem:rsh:rm about to execute [%s]", command));
-
-        if( ORTE_SUCCESS != (ret = orte_filem_rsh_start_command(p_set,
-                                                                NULL,
-                                                                command,
-                                                                request,
-                                                                cur_index)) ) {
-            exit_status = ret;
-            goto cleanup;
-        }
-
-        /* A small bit of cleanup */
-        if( NULL != dir_arg) {
-            free(dir_arg);
-            dir_arg = NULL;
-        }
-
-        if( NULL != remote_targets) {
-            free(remote_targets);
-            remote_targets = NULL;
-        }
-            
-        if( NULL != remote_file_set) {
-            opal_argv_free(remote_file_set);
-            remote_file_set = NULL;
-        }
-
-        if(NULL != remote_machine) {
-            free(remote_machine);
-            remote_machine = NULL;
-        }
-
-        ++cur_index;
-    } /* Process set */
-
- cleanup:
-    if( NULL != command )
-        free(command);
-    if( NULL != remote_machine)
-        free(remote_machine);
-    if( NULL != dir_arg) 
-        free(dir_arg);
-    if( NULL != remote_targets)
-        free(remote_targets);
-    if( NULL != remote_file_set)
-        opal_argv_free(remote_file_set);
-
-    return exit_status;
-}
-
-/******************
- * Local Functions
- ******************/
-
-/******************************
- * Work Pool functions
- ******************************/
-static int  orte_filem_rsh_start_command(orte_filem_base_process_set_t *proc_set,
-                                         orte_filem_base_file_set_t    *file_set,
-                                         char * command,
-                                         orte_filem_base_request_t *request,
-                                         int index)
-{
-    orte_filem_rsh_work_pool_item_t *wp_item = NULL;
-    int ret;
-
-    /* Construct a work pool item */
-    wp_item = OBJ_NEW(orte_filem_rsh_work_pool_item_t);
-    /* Copy the Process Set */
-    if( NULL != proc_set ) {
-        wp_item->proc_set.source.jobid = proc_set->source.jobid;
-        wp_item->proc_set.source.vpid  = proc_set->source.vpid;
-
-        wp_item->proc_set.sink.jobid = proc_set->sink.jobid;
-        wp_item->proc_set.sink.vpid  = proc_set->sink.vpid;
-    }
-    /* Copy the File Set */
-    if( NULL != file_set ) {
-        wp_item->file_set.local_target  = strdup(file_set->local_target);
-        wp_item->file_set.remote_target = strdup(file_set->remote_target);
-        wp_item->file_set.target_flag   = file_set->target_flag;
-    }
-    OBJ_RETAIN(request);
-    wp_item->command = strdup(command);
-    wp_item->request = request;
-    wp_item->index   = index;
-
-    if( orte_filem_rsh_max_outgoing > 0 && cur_num_outgoing >= orte_filem_rsh_max_outgoing ) {
-        OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                             "filem:rsh: wait(): *** Hold send from proc %s (%d of %d)",
-                             ORTE_NAME_PRINT(&(wp_item->proc_set.source)), cur_num_outgoing, orte_filem_rsh_max_outgoing));
-        /*
-         * - put the request on the held list, since we only allow 2 active filem ops at a time
-         */
-        opal_list_append(&work_pool_held, &(wp_item->super));
-    }
-    else {
-        ++cur_num_outgoing;
-
-        /*
-         * - put the request on the pending list
-         * - wait for the peer to tell us that it can receive
-         */
-        opal_list_append(&work_pool_pending, &(wp_item->super));
-
-        /*
-         * Ask for permission to send this file so we do not overwhelm the peer
-         * Allow only one file request at a time.
-         * JJH: Look into permission for multiple file permissions at a time
-         */
-        OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                             "filem:rsh: start_command(): Ask permission to send from proc %s (%d of %d)",
-                             ORTE_NAME_PRINT(&(proc_set->source)), cur_num_outgoing, orte_filem_rsh_max_outgoing));
-        if( ORTE_SUCCESS != (ret = orte_filem_rsh_permission_ask(&(proc_set->source), 1)) ) {
-            return ret;
-        }
-    }
-
-    return ORTE_SUCCESS;
-}
-
-/******************************
- * Child process start and wait functions
- ******************************/
-static int start_child(char * command,
-                       orte_filem_base_request_t *request,
-                       int index)
-{
-    char **argv = NULL;
-    int status, ret;
-
-    OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                         "filem:rsh: start_child(): Starting the command [%s]",
-                         command));
-    /* fork() -> done = false, active = true */
-    request->is_done[index]     = false;
-    request->is_active[index]   = true;
-    request->exit_status[index] = fork();
-
-    if( request->exit_status[index] == 0 ) { /* Child */
-        /* Redirect stdout to /dev/null */
-        freopen( "/dev/null", "w", stdout);
-
-        argv = opal_argv_split(command, ' ');
-
-        status = execvp(argv[0], argv);
-
-        opal_output(0, "filem:rsh:start_child Failed to exec child [%s] status = %d\n", command, status);
-        exit(ORTE_ERROR);
-    }
-    else if( request->exit_status[index] > 0 ) {
-        OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                             "filem:rsh: start_child(): Started Child %d Running command [%s]",
-                             request->exit_status[index], command));
-
-        /*
-         * Register a callback for when this process exits
-         */
-        if( ORTE_SUCCESS != (ret = orte_wait_cb(request->exit_status[index], filem_rsh_waitpid_cb, NULL) ) ) {
-            opal_output(0, "filem:rsh: start_child(): Failed to register a waitpid callback for child [%d] executing the command [%s]\n",
-                        request->exit_status[index], command);
-            return ret;
-        }
-    }
-    else {
-        return ORTE_ERROR;
-    }
-
-    return ORTE_SUCCESS;
-}
-
-
-static void filem_rsh_waitpid_cb(pid_t pid, int status, void* cbdata)
-{
-    int ret;
-    orte_filem_rsh_work_pool_item_t *wp_item = NULL;
-    orte_filem_base_request_t *request;
-    opal_list_item_t *item = NULL;
-    int index;
-
-    OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                         "filem:rsh: waitpid_cb(): Pid %d finished with status [%d].\n",
-                         pid, status));
-
-    /*
-     * Find this pid in the active queue
-     */
-    OPAL_THREAD_LOCK(&work_pool_lock);
-    for (item  = opal_list_get_first( &work_pool_active);
-         item != opal_list_get_end(   &work_pool_active);
-         item  = opal_list_get_next(   item) ) {
-        wp_item = (orte_filem_rsh_work_pool_item_t *)item;
-        request = wp_item->request;
-        index   = wp_item->index;
-        if( request->is_done[index]     == false &&
-            request->exit_status[index] == pid ) {
-            request->exit_status[index] = status;
-            /* waitpid() -> done = true, active = false */
-            request->is_done[index]     = true;
-            request->is_active[index]   = false;
-            OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                 "filem:rsh: waitpid_cb(): Marked pid %d as complete [status = %d].\n",
-                                 pid, status));
-            break;
-        }
-    }
-
-    --cur_num_outgoing;
-
-    /*
-     * If we are holding any requests, start them
-     */
-    if( opal_list_get_size(&work_pool_held) > 0 ) {
-        item = opal_list_remove_first(&work_pool_held);
-        wp_item = (orte_filem_rsh_work_pool_item_t *)item;
-
-        ++cur_num_outgoing;
-
-        /*
-         * - put the request on the pending list
-         * - wait for the peer to tell us that it can receive
-         */
-        opal_list_append(&work_pool_pending, &(wp_item->super));
-
-        /*
-         * Ask for permission to send this file so we do not overwhelm the peer
-         */
-        OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                             "filem:rsh: start_command(): Ask permission to send from proc %s (*** Activate Held)",
-                             ORTE_NAME_PRINT(&(wp_item->proc_set.source))));
-        if( ORTE_SUCCESS != (ret = orte_filem_rsh_permission_ask(&(wp_item->proc_set.source), 1)) ) {
-            opal_output(0, "ERROR: Failed to ask permission!\n");
-        }
-    }
-
-    /*
-     * Signal in case anyone is waiting for a child to finish.
-     */
-    opal_condition_signal(&work_pool_cond);
-    OPAL_THREAD_UNLOCK(&work_pool_lock);
-}
-
-/******************************
- * Path resolution functions
- ******************************/
-/*
- * This function is paired with the filem_base_process_get_remote_path_cmd() function on the remote machine
- */
-static int orte_filem_rsh_query_remote_path(char **remote_ref, orte_process_name_t *peer, int *flag) {
-    int ret;
-
-    /*
-     * If we are given an absolute path for the remote side, then there is
-     * nothing to do. If the remote directory does not exist, then scp will
-     * error out, which is caught by the filem_rsh_waitpid_cb() function.
-     *
-     * Assume the remote path is a directory, since if it is just a file then
-     * the command will still work as normal.
-     */
-    if( *remote_ref[0] == '/' ) {
-        *flag = ORTE_FILEM_TYPE_DIR;
-        return ORTE_SUCCESS;
-    }
-
-    /* Call the base function */
-    if( ORTE_SUCCESS != (ret = orte_filem_base_get_remote_path(remote_ref, peer, flag) ) ) {
-        return ret;
-    }
-
-    return ORTE_SUCCESS;
-}
-
-/******************************
- * Permission functions
- ******************************/
-static int orte_filem_rsh_permission_listener_init(void)
-{
-    int ret;
-
-    if( ORTE_SUCCESS != (ret = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD,
-                                                       ORTE_RML_TAG_FILEM_RSH,
-                                                       ORTE_RML_PERSISTENT,
-                                                       orte_filem_rsh_permission_callback,
-                                                       NULL)) ) {
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: listener_init: Failed to register the receive callback (%d)",
-                    ret);
-        return ret;
-    }
-
-    return ORTE_SUCCESS;
-}
-
-static int orte_filem_rsh_permission_listener_cancel(void)
-{
-    int ret;
-
-    if( ORTE_SUCCESS != (ret = orte_rml.recv_cancel(ORTE_NAME_WILDCARD, ORTE_RML_TAG_FILEM_RSH) ) ) {
-#if 0
-        opal_output(orte_filem_base_framework.framework_output,
-                    "filem:rsh: listener_cancel: Failed to deregister the receive callback (%d)",
-                    ret);
-#endif
-        return ret;
-    }
-
-    return ORTE_SUCCESS;
-}
-
-static void orte_filem_rsh_permission_callback(int status,
-                                               orte_process_name_t* sender,
-                                               opal_buffer_t *buffer,
-                                               orte_rml_tag_t tag,
-                                               void* cbdata)
-{
-    orte_filem_rsh_work_pool_item_t *wp_item = NULL;
-    opal_list_item_t *item = NULL;
-    int ret;
-    orte_std_cntr_t n;
-    int num_req, num_allowed = 0;
-    int perm_flag, i;
-    int32_t peer_status = 0;
-    orte_ns_cmp_bitmask_t mask;
-
-    OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                         "filem:rsh: permission_callback(? ?): Peer %s ...",
-                         ORTE_NAME_PRINT(sender)));
-
-    /*
-     * Receive the flag indicating if this is:
-     * - Asking for permission (ORTE_FILEM_RSH_ASK)
-     * - Allowing us to send (ORTE_FILEM_RSH_ALLOW)
-     */
-    n = 1;
-    if (ORTE_SUCCESS != (ret = opal_dss.unpack(buffer, &perm_flag, &n, OPAL_INT))) {
-        goto cleanup;
-    }
-
-    /* Asking for permission to send */
-    if( ORTE_FILEM_RSH_ASK == perm_flag ) {
-        OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                             "filem:rsh: permission_callback(ASK): Peer %s Asking permission to send [Used %d of %d]",
-                             ORTE_NAME_PRINT(sender),
-                             cur_num_incomming,
-                             orte_filem_rsh_max_incomming));
-
-        /*
-         * Receive the requested amount
-         */
-        n = 1;
-        if (ORTE_SUCCESS != (ret = opal_dss.unpack(buffer, &num_req, &n, OPAL_INT))) {
-            goto cleanup;
-        }
-
-        /*
-         * Determine how many we can allow
-         * if none then put a request on the waiting list
-         * ow tell the peer to start sending now.
-         * Send back number allowed to be started
-         */
-        if( orte_filem_rsh_max_incomming > 0 && orte_filem_rsh_max_incomming < cur_num_incomming + 1) {
-            /* Add to the waiting list */
-            OPAL_OUTPUT_VERBOSE((5, orte_filem_base_framework.framework_output,
-                                 "filem:rsh: permission_callback(ASK): Add Peer %s request to waiting list",
-                                 ORTE_NAME_PRINT(sender)));
-
-            wp_item = OBJ_NEW(orte_filem_rsh_work_pool_item_t);
-            wp_item->proc_set.source.jobid = sender->jobid;
-            wp_item->proc_set.source.vpid  = sender->vpid;
-
-            opal_list_append(&work_pool_waiting, &(wp_item->super));
-        }
-        /* Start the transfer immediately */
-        else {