How to know if a paramiko SSH channel is disconnected?

Question

I'm desinging test cases in which I use paramiko for SSH connections. Test cases usually contain paramiko.exec_command() calls which I have a wrapper for (called run_command()). Here self.ssh is an intance of paramiko.SSHClient(). I use a decorator to check the ssh connection before each call. (self.get_ssh() negotiates the connection)

def check_connections(function):
    ''' A decorator to check SSH connections. '''
    def deco(self, *args, **kwargs):
        if self.ssh is None:
            self.ssh = self.get_ssh()
        else:
            ret = getattr(self.ssh.get_transport(), 'is_active', None)
            if ret is None or (ret is not None and not ret()):
                self.ssh = self.get_ssh()
        return function(self, *args, **kwargs)
    return deco

@check_connections
def run_command(self, command):
    ''' Executes command via SSH. '''
    stdin, stdout, stderr = self.ssh.exec_command(command)
    stdin.flush()
    stdin.channel.shutdown_write()
    ret = stdout.read()
    err = stderr.read()
    if ret:
        return ret
    elif err:
        return err
    else:
        return None

It works perfectly until my remote node reboots, which can happen sometimes. When it occurs the next run_command() call generates a socket.error exception. The problem is, that the paramiko.Transport object seems to be remain in active state until an exception is thrown:

Python 2.7.3 (default, Mar  7 2013, 14:03:36)
[GCC 4.3.4 [gcc-4_3-branch revision 152973]] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> import paramiko
>>> ssh = paramiko.SSHClient()
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
None
>>> ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
>>> ssh.load_host_keys(os.path.expanduser('~') + '/.ssh/known_hosts')
>>> ssh.connect(hostname = '172.31.77.57', username = 'root', password = 'rootroot', timeout = 5.0)
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>
>>> print ssh.get_transport().is_active()
True
>>> ssh.exec_command('ls')
(<paramiko.ChannelFile from <paramiko.Channel 1 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 1 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 1 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>)
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>
>>> print ssh.get_transport().is_active()
True
>>> ssh.exec_command('reboot')
(<paramiko.ChannelFile from <paramiko.Channel 2 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 2 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 2 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>)
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>
>>> print ssh.get_transport().is_active()
True
>>> ssh.exec_command('ls')
No handlers could be found for logger "paramiko.transport"
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/pytest/lib/python2.7/site-packages/paramiko/client.py", line 370, in exec_command
    chan = self._transport.open_session()
  File "/home/pytest/lib/python2.7/site-packages/paramiko/transport.py", line 662, in open_session
    return self.open_channel('session')
  File "/home/pytest/lib/python2.7/site-packages/paramiko/transport.py", line 764, in open_channel
    raise e
socket.error: [Errno 104] Connection reset by peer
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (unconnected)>
>>> print ssh.get_transport().is_active()
False
>>>

Question: how can I be sure that the connection is really active or not?

Answer 1

This works:

import paramiko
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())        # Setting the missing host policy to auto add it
client.connect('192.168.1.16', port=22, username='admin', password='admin', timeout=3, banner_timeout=2)

channel = client.invoke_shell()                 # Request an interactive shell session on this channel. If the server allows it, the channel will then be directly connected to the stdin, stdout, and stderr of the shell.
print channel.closed          # False
command = 'reboot'
channel.send(command + '\n')
# wait a while
print channel.closed          # True

Answer 2

My solution basically the same as yours, just organized differently:

def connection(self):
    if not self.is_connected():
        self._ssh = paramiko.SSHClient()
        self._ssh.connect(self.server, self.port,
                          username = self.username, password = self.password)

    return self._ssh

def is_connected(self):
    transport = self._ssh.get_transport() if self._ssh else None
    return transport and transport.is_active()

def do_something(self):
    self.connection().exec_command('ls')