cannot login to reserved nodes

I am able to ssh into my nodes in the sliver using the command line, which possibly is all I need going forward. However, just wanted to report that I still cannot execute the commands that require logging into the nodes in my slice from my notebook.

Here is the error:

—————————————————————————
AuthenticationException Traceback (most recent call last)
/tmp/ipykernel_437/3712757443.py in <module>
—-> 1 node1.execute(‘ls -la’)

/opt/conda/lib/python3.9/site-packages/fabrictestbed_extensions/fablib/node.py in execute(self, command, retry, retry_interval)
721
722 if attempt+1 == retry:
–> 723 raise e
724
725 #Fail, try again

/opt/conda/lib/python3.9/site-packages/fabrictestbed_extensions/fablib/node.py in execute(self, command, retry, retry_interval)
677 bastion=paramiko.SSHClient()
678 bastion.set_missing_host_key_policy(paramiko.AutoAddPolicy())
–> 679 bastion.connect(fablib.get_bastion_public_addr(), username=fablib.get_bastion_username(), key_filename=fablib.get_bastion_key_filename())
680
681 bastion_transport = bastion.get_transport()

/opt/conda/lib/python3.9/site-packages/paramiko/client.py in connect(self, hostname, port, username, password, pkey, key_filename, timeout, allow_agent, look_for_keys, compress, sock, gss_auth, gss_kex, gss_deleg_creds, gss_host, banner_timeout, auth_timeout, gss_trust_dns, passphrase, disabled_algorithms)
433 key_filenames = key_filename
434
–> 435 self._auth(
436 username,
437 password,

/opt/conda/lib/python3.9/site-packages/paramiko/client.py in _auth(self, username, password, pkey, key_filenames, allow_agent, look_for_keys, gss_auth, gss_kex, gss_deleg_creds, gss_host, passphrase)
764 # if we got an auth-failed exception earlier, re-raise it
765 if saved_exception is not None:
–> 766 raise saved_exception
767 raise SSHException(“No authentication methods available”)
768

/opt/conda/lib/python3.9/site-packages/paramiko/client.py in _auth(self, username, password, pkey, key_filenames, allow_agent, look_for_keys, gss_auth, gss_kex, gss_deleg_creds, gss_host, passphrase)
740 # in [‘password’]
741 allowed_types = set(
–> 742 self._transport.auth_publickey(username, key)
743 )
744 two_factor = allowed_types & two_factor_types

/opt/conda/lib/python3.9/site-packages/paramiko/transport.py in auth_publickey(self, username, key, event)
1633 # caller wants to wait for event themselves
1634 return []
-> 1635 return self.auth_handler.wait_for_response(my_event)
1636
1637 def auth_interactive(self, username, handler, submethods=””):

/opt/conda/lib/python3.9/site-packages/paramiko/auth_handler.py in wait_for_response(self, event)
257 if issubclass(e.__class__, PartialAuthentication):
258 return e.allowed_types
–> 259 raise e
260 return []
261

AuthenticationException: Authentication failed.

I tried to trace where the failure occurs, attached, if at all helpful.

I was not trying to login to the bastion host directly.

I am encountering the error I pasted above when I execute a cell in my notebook with the command:

node1.execute("ls -la")

(Attachment was my effort to help trace it on your end)

So, interestingly, I am seeing some inconsistent behaviour…

First: I changed my keys to have no passphrases. I then uploaded them to portal and then to the JHub environment. Even though I had killed my JHub server and logged out, relogged in, restarted JHub server, my environment still loads the keys with passphrases. On a good note, my node1.execute command is working now…

I am stuck this time on the node1.upload_file command. Here is the error:

SCP upload fail. Slice: modelgen_nodes, Node: client, trying again
Fail: Failure

---------------------------------------------------------------------------
OSError Traceback (most recent call last)
/tmp/ipykernel_128/909245904.py in
----> 1 node1.upload_file("/home/fabric/work/test_file", "/home/rocky/")

/opt/conda/lib/python3.9/site-packages/fabrictestbed_extensions/fablib/node.py in upload_file(self, local_file_path, remote_file_path, retry, retry_interval)
809
810 if attempt+1 == retry:
--> 811 raise e
812
813 #Fail, try again

/opt/conda/lib/python3.9/site-packages/fabrictestbed_extensions/fablib/node.py in upload_file(self, local_file_path, remote_file_path, retry, retry_interval)
784
785 ftp_client=client.open_sftp()
--> 786 file_attributes = ftp_client.put(local_file_path, remote_file_path)
787 ftp_client.close()
788

/opt/conda/lib/python3.9/site-packages/paramiko/sftp_client.py in put(self, localpath, remotepath, callback, confirm)
757 file_size = os.stat(localpath).st_size
758 with open(localpath, "rb") as fl:
--> 759 return self.putfo(fl, remotepath, file_size, callback, confirm)
760
761 def getfo(self, remotepath, fl, callback=None, prefetch=True):

/opt/conda/lib/python3.9/site-packages/paramiko/sftp_client.py in putfo(self, fl, remotepath, file_size, callback, confirm)
712 .. versionadded:: 1.10
713 """
--> 714 with self.file(remotepath, "wb") as fr:
715 fr.set_pipelined(True)
716 size = self._transfer_with_callback(

/opt/conda/lib/python3.9/site-packages/paramiko/sftp_client.py in open(self, filename, mode, bufsize)
370 imode |= SFTP_FLAG_CREATE | SFTP_FLAG_EXCL
371 attrblock = SFTPAttributes()
--> 372 t, msg = self._request(CMD_OPEN, filename, imode, attrblock)
373 if t != CMD_HANDLE:
374 raise SFTPError("Expected handle")

/opt/conda/lib/python3.9/site-packages/paramiko/sftp_client.py in _request(self, t, *arg)
820 def _request(self, t, *arg):
821 num = self._async_request(type(None), t, *arg)
--> 822 return self._read_response(num)
823
824 def _async_request(self, fileobj, t, *arg):

/opt/conda/lib/python3.9/site-packages/paramiko/sftp_client.py in _read_response(self, waitfor)
872 # synchronous
873 if t == CMD_STATUS:
--> 874 self._convert_status(msg)
875 return t, msg
876

/opt/conda/lib/python3.9/site-packages/paramiko/sftp_client.py in _convert_status(self, msg)
905 raise IOError(errno.EACCES, text)
906 else:
--> 907 raise IOError(text)
908
909 def _adjust_cwd(self, path):

OSError: Failure

• This reply was modified 2 years, 4 months ago by Deniz Gurkan.