“Expired refresh token” when starting a new JupyterHub server after timeout

Hi, my students and I occasionally encounter an “expired refresh token” error that requires force restarting the JupyterHub container (File > Hub Control Panel) to resolve. (Example, another example.) Recently I have been running some long-running experiments in FABRIC JupyterHub and I noticed  that the following sequence reliably reproduces this error –

1. Leave something running in JH for a long time (e.g. overnight).
2. Get back and see “JH server no longer running” message.
3. See message that says “Server unavailable or unreachable” with option to restart the server. Choose “Restart”.
4. Go through the sequence of selecting server, logging in, etc. Get a new, running, JH server – I expect this to now be working normally, right?
5. Run fablib = fablib_manager() and see this error –

---------------------------------------------------------------------------
SliceManagerException                     Traceback (most recent call last)
Cell In[1], line 2
      1 from fabrictestbed_extensions.fablib.fablib import FablibManager as fablib_manager
----> 2 fablib = fablib_manager() 
      3 conf = fablib.show_config()

File /opt/conda/lib/python3.10/site-packages/fabrictestbed_extensions/fablib/fablib.py:796, in FablibManager.__init__(self, fabric_rc, credmgr_host, orchestrator_host, fabric_token, project_id, bastion_username, bastion_key_filename, log_level, log_file, data_dir, output, execute_thread_pool_size, offline)
    793 self.facility_ports = None
    795 if not offline:
--> 796     self.build_slice_manager()

File /opt/conda/lib/python3.10/site-packages/fabrictestbed_extensions/fablib/fablib.py:985, in FablibManager.build_slice_manager(self)
    982 except Exception as e:
    983     # logging.error(f"{e}")
    984     logging.error(e, exc_info=True)
--> 985     raise e
    987 return self.slice_manager

File /opt/conda/lib/python3.10/site-packages/fabrictestbed_extensions/fablib/fablib.py:971, in FablibManager.build_slice_manager(self)
    961 try:
    962     logging.info(
    963         f"oc_host={self.orchestrator_host},"
    964         f"cm_host={self.credmgr_host},"
   (...)
    968         f"scope='all'"
    969     )
--> 971     self.slice_manager = SliceManager(
    972         oc_host=self.orchestrator_host,
    973         cm_host=self.credmgr_host,
    974         project_id=self.project_id,
    975         token_location=self.fabric_token,
    976         initialize=True,
    977         scope="all",
    978     )
    980     # Initialize the slice manager
    981     self.slice_manager.initialize()

File /opt/conda/lib/python3.10/site-packages/fabrictestbed/slice_manager/slice_manager.py:71, in SliceManager.__init__(self, cm_host, oc_host, token_location, project_id, scope, initialize)
     67     raise SliceManagerException(f"Invalid initialization parameters: cm_proxy={self.cm_proxy}, "
     68                                 f"oc_proxy={self.oc_proxy}, token_location={self.token_location}, "
     69                                 f"project_id={self.project_id}")
     70 if initialize:
---> 71     self.initialize()

File /opt/conda/lib/python3.10/site-packages/fabrictestbed/slice_manager/slice_manager.py:80, in SliceManager.initialize(self)
     74 """
     75 Initialize the Slice Manager object
     76 - Load the tokens
     77 - Refresh if needed
     78 """
     79 if not self.initialized:
---> 80     self.__load_tokens()
     81     self.initialized = True

File /opt/conda/lib/python3.10/site-packages/fabrictestbed/slice_manager/slice_manager.py:127, in SliceManager.__load_tokens(self)
    125     refresh_token = os.environ.get(Constants.CILOGON_REFRESH_TOKEN)
    126 # Renew the tokens to ensure any project_id changes are taken into account
--> 127 self.refresh_tokens(refresh_token=refresh_token)

File /opt/conda/lib/python3.10/site-packages/fabrictestbed/slice_manager/slice_manager.py:164, in SliceManager.refresh_tokens(self, refresh_token)
    162     self.tokens = tokens
    163     return tokens.get(CredmgrProxy.ID_TOKEN, None), tokens.get(CredmgrProxy.REFRESH_TOKEN, None)
--> 164 raise SliceManagerException(tokens.get(CredmgrProxy.ERROR))

SliceManagerException: b'{\n    "errors": [\n        {\n            "details": "(invalid_grant) expired refresh token",\n            "message": "Internal Server Error"\n        }\n    ],\n    "size": 1,\n    "status": 500,\n    "type": "error"\n}'

and to resolve it, I need to stop and start the server again using File > Hub Control Panel.

Sharing this sequence in case it helps the team debug this type of error…