---------- CRASH DETAILS ----------
2011-12-02 14:42:36,176 PID: 6804 config.py:551 - DEBUG - Loading config
2011-12-02 14:42:36,176 PID: 6804 config.py:119 - DEBUG - Loading file: /Users/qtrinh/.starcluster/config
2011-12-02 14:42:36,180 PID: 6804 awsutils.py:55 - DEBUG - creating self._conn w/ connection_authenticator kwargs = {'path': '/', 'region': None, 'port': None, 'is_secure': True}
2011-12-02 14:42:36,417 PID: 6804 start.py:178 - INFO - Using default cluster template: smallcluster
2011-12-02 14:42:36,418 PID: 6804 cluster.py:1511 - INFO - Validating cluster template settings...
2011-12-02 14:42:36,748 PID: 6804 cluster.py:1526 - INFO - Cluster template settings are valid
2011-12-02 14:42:36,748 PID: 6804 cluster.py:1403 - INFO - Starting cluster...
2011-12-02 14:42:36,749 PID: 6804 cluster.py:929 - INFO - Launching a 2-node cluster...
2011-12-02 14:42:36,821 PID: 6804 awsutils.py:162 - INFO - Creating security group @sc-smallcluster...
2011-12-02 14:42:38,241 PID: 6804 cluster.py:1213 - INFO - Waiting for cluster to come up... (updating every 30s)
2011-12-02 14:42:39,510 PID: 6804 cluster.py:660 - DEBUG - existing nodes: {}
2011-12-02 14:42:39,510 PID: 6804 cluster.py:676 - DEBUG - returning self._nodes = []
2011-12-02 14:42:39,510 PID: 6804 cluster.py:1118 - INFO - Waiting for instances to activate...
2011-12-02 14:43:09,602 PID: 6804 cluster.py:660 - DEBUG - existing nodes: {}
2011-12-02 14:43:09,603 PID: 6804 cluster.py:676 - DEBUG - returning self._nodes = []
2011-12-02 14:43:39,679 PID: 6804 cluster.py:660 - DEBUG - existing nodes: {}
2011-12-02 14:43:39,680 PID: 6804 cluster.py:676 - DEBUG - returning self._nodes = []
2011-12-02 14:44:09,744 PID: 6804 cluster.py:660 - DEBUG - existing nodes: {}
2011-12-02 14:44:09,744 PID: 6804 cluster.py:676 - DEBUG - returning self._nodes = []
2011-12-02 14:44:39,813 PID: 6804 cluster.py:660 - DEBUG - existing nodes: {}
2011-12-02 14:44:39,813 PID: 6804 cluster.py:676 - DEBUG - returning self._nodes = []
2011-12-02 14:45:09,978 PID: 6804 cluster.py:660 - DEBUG - existing nodes: {}
....
2011-12-02 14:45:41,980 PID: 6804 clustersetup.py:96 - INFO - Configuring hostnames...
2011-12-02 14:45:41,986 PID: 6804 ssh.py:173 - DEBUG - creating sftp connection
2011-12-02 14:45:41,986 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:41,986 PID: 6804 ssh.py:173 - DEBUG - creating sftp connection
2011-12-02 14:45:42,988 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:43,990 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:44,991 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:45,993 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:46,994 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:47,995 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:48,996 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:49,998 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:50,998 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:52,000 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:53,001 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:54,002 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:55,004 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:56,005 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:57,005 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:58,007 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:45:59,008 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:00,010 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:01,011 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:02,012 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:03,014 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:04,015 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:05,016 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:06,018 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:07,019 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:08,020 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:09,022 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:10,023 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:11,025 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:12,025 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:13,027 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:14,028 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:15,029 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:16,030 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:17,032 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:18,033 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:19,034 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:20,036 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:21,037 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:22,038 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:23,039 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:24,040 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:25,041 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:26,043 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:27,044 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:28,046 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:29,047 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:30,048 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:31,049 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:32,050 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:33,051 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:34,052 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:35,053 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:36,055 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:37,055 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:38,056 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:39,057 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:40,059 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:41,060 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:42,061 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:43,062 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:44,063 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:45,064 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:46,066 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:47,067 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:48,069 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:49,070 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:50,071 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:51,072 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:52,074 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:53,075 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:54,076 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:55,077 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:56,079 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:57,080 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:58,081 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:46:59,083 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:00,084 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:01,086 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:02,087 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:03,088 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:04,090 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:05,091 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:06,092 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:07,093 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:08,094 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:09,095 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:10,097 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:11,098 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:12,099 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:13,100 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:14,102 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:15,102 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:16,104 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:17,105 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:18,107 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:19,108 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:20,109 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:21,111 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:22,111 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:23,112 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:24,114 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:25,115 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:26,116 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:27,118 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:28,119 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:29,120 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:30,122 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:31,123 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:32,124 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:33,126 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:34,127 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:35,128 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:36,129 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:37,130 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:38,131 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:39,133 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:40,134 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:41,136 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 2
2011-12-02 14:47:42,137 PID: 6804 threadpool.py:124 - INFO - Shutting down threads...
2011-12-02 14:47:42,140 PID: 6804 threadpool.py:136 - DEBUG - unfinished_tasks = 4
2011-12-02 14:47:43,143 PID: 6804 cli.py:263 - DEBUG - error occured in job (id=master): 
Traceback (most recent call last):
  File "/Library/Python/2.7/site-packages/StarCluster-0.92.1-py2.7.egg/starcluster/threadpool.py", line 32, in run
    job.run()
  File "/Library/Python/2.7/site-packages/StarCluster-0.92.1-py2.7.egg/starcluster/threadpool.py", line 59, in run
    r = self.method(*self.args, **self.kwargs)
  File "/Library/Python/2.7/site-packages/StarCluster-0.92.1-py2.7.egg/starcluster/node.py", line 661, in set_hostname
    hostname_file = self.ssh.remote_file("/etc/hostname", "w")
  File "/Library/Python/2.7/site-packages/StarCluster-0.92.1-py2.7.egg/starcluster/ssh.py", line 284, in remote_file
    rfile = self.sftp.open(file, mode)
  File "/Library/Python/2.7/site-packages/StarCluster-0.92.1-py2.7.egg/starcluster/ssh.py", line 174, in sftp
    self._sftp = paramiko.SFTPClient.from_transport(self.transport)
  File "build/bdist.macosx-10.7-intel/egg/paramiko/sftp_client.py", line 102, in from_transport
    chan = t.open_session()
  File "build/bdist.macosx-10.7-intel/egg/paramiko/transport.py", line 658, in open_session
    return self.open_channel('session')
  File "build/bdist.macosx-10.7-intel/egg/paramiko/transport.py", line 746, in open_channel
    raise e
EOFError

error occured in job (id=node001): 
Traceback (most recent call last):
  File "/Library/Python/2.7/site-packages/StarCluster-0.92.1-py2.7.egg/starcluster/threadpool.py", line 32, in run
    job.run()
  File "/Library/Python/2.7/site-packages/StarCluster-0.92.1-py2.7.egg/starcluster/threadpool.py", line 59, in run
    r = self.method(*self.args, **self.kwargs)
  File "/Library/Python/2.7/site-packages/StarCluster-0.92.1-py2.7.egg/starcluster/node.py", line 661, in set_hostname
    hostname_file = self.ssh.remote_file("/etc/hostname", "w")
  File "/Library/Python/2.7/site-packages/StarCluster-0.92.1-py2.7.egg/starcluster/ssh.py", line 284, in remote_file
    rfile = self.sftp.open(file, mode)
  File "/Library/Python/2.7/site-packages/StarCluster-0.92.1-py2.7.egg/starcluster/ssh.py", line 174, in sftp
    self._sftp = paramiko.SFTPClient.from_transport(self.transport)
  File "build/bdist.macosx-10.7-intel/egg/paramiko/sftp_client.py", line 102, in from_transport
    chan = t.open_session()
  File "build/bdist.macosx-10.7-intel/egg/paramiko/transport.py", line 658, in open_session
    return self.open_channel('session')
  File "build/bdist.macosx-10.7-intel/egg/paramiko/transport.py", line 746, in open_channel
    raise e
EOFError

---------- SYSTEM INFO ----------
StarCluster: 0.92.1
Python: 2.7.1 (r271:86832, Jun 16 2011, 16:59:05)  [GCC 4.2.1 (Based on Apple Inc. build 5658) (LLVM build 2335.15.00)]
Platform: Darwin-11.2.0-x86_64-i386-64bit
boto: 2.0
paramiko: 1.7.7.1 (George)
Crypto: 2.4.1
jinja2: 2.5.5
decorator: 3.3.1