Implement timeout for paramiko with an interactive element

Question

This question is a followup to this question: Using paramiko to send commands to an open shell that has an interactive element so please read that before answering here.

I'm successfully able to send data to the array's system shell, however I need help figuring out how to implement a timeout when the array is taking too long to run the script I've sent it. Paramiko's exec_command has a timeout= parameter, but that's not going to help here as the only command I'm sending is "script" which immediately returns and waits for input on the channel, and when I tried implementing it seems to break the rest of the function as nothing returns from the array.

The array is then supposed to processes the script I've sent it and return the output via stdout, however if the array takes a long time I have no way of timing out the connection, and it holds up the rest of the script.

Here's the code I have:

def run_script(self, script_name):
    """ Run a script on the remote array and return the stdout
    """
    try:
        common_functions_file = os.path.join(SCRIPT_DIR, 'zfs_common_functions.js')
        common_functions = open(common_functions_file).read().splitlines()
        # add common_functions to the top of the script
        script_contents = common_functions + open(script_name).read().splitlines()
        stdin, stdout, stderr = self._ssh.exec_command('script')
        for line in script_contents:
            # Skip lines with comments
            if re.match("^//", line):
                continue
            stdin.write(line)
            stdin.write('\n')
        stdin.write('.\n')
        stdin.flush()
        error = stderr.readlines()
        if len(error) == 0:
            try:
                output = ''.join(stdout.readlines())
                if(re.search('aksh', output)):
                    logger.warn("ZFS Shell Error: %s" % output)
                    return None
                return output
            except Exception as e:
                logger.exception(e)

        else:
            logger.error(error)
            return None
    except paramiko.SSHException as e:
        logger.warn(
            "Couldn't execute script on array %s: %s" % (array.name, e))

    except AttributeError as e:
        logger.exception(e)
        raise

    except Exception:
        raise

Answer 1

I ended up solving it by directly interacting with the channel, which you can set a timeout on:

def run_script(self, script_name):
    """ Run a script on the remote array and return the stdout
    """
    try:
        chan = self._ssh.get_transport().open_session()
        # five minute timeout on the channel communication
        chan.settimeout(5*60.0)
        common_functions_file = os.path.join(SCRIPT_DIR, 'zfs_common_functions.js')
        common_functions = open(common_functions_file).read().splitlines()
        # add common_functions to the top of the script
        script_contents = common_functions + open(script_name).read().splitlines()
        chan.exec_command('script')
        if chan.send_ready():
            chan.sendall("\n".join(script_contents))
            chan.send("\n.\n")

        results = StringIO()
        error = StringIO()
        bufsize = 1024
        while not chan.exit_status_ready():
            if chan.recv_ready():
                data = chan.recv(bufsize)
                while data:
                    results.write(data)
                    data = chan.recv(bufsize)

            if chan.recv_stderr_ready():
                error_buf = chan.recv_stderr(bufsize)
                while error_buf:
                    error.write(error_buf)
                    error_buf = chan.recv_stderr(bufsize)

        exit_status = chan.recv_exit_status()
        if exit_status == 0:
            return results.getvalue()
        else:
            raise ZfsScriptError(results.getvalue())

    except socket.timeout:
        logger.warn("%s: Timeout running %s" %(self.hostname, script_name))
        return None

    except paramiko.SSHException as e:
        logger.warn(
            "Couldn't execute script on array %s: %s" % (self.hostname, e))
        raise

    except AttributeError as e:
        logger.exception(e)
        raise

    except Exception:
        raise

    finally:
        results.close()
        error.close()
        chan.close()