20KB/s bandwidth between Colab and drive

[alister-crawled]

When lucky to not being victim of #5951 issue (which happens 99% of time), if miraculously colab doesn't crash, you get an outstanding 20KB/s connection bandwidth to the drive....

WHAT A SCAM

[alister-crawled]

Shit load of errors now :
Error: Error opening '/content/drive/MyDrive/......': System error.

[alister-crawled]

and crashes again
RuntimeError: Parent directory crowd does not exist.
--- Logging error ---
Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 134, in _forward_responses
_logger.info("Reached EOF.")
Message: 'Reached EOF.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 137, in _forward_responses
self._mailbox.close()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/mailbox/mailbox.py", line 155, in close
_logger.info(
Message: 'Closing mailbox, abandoning 1 handles.'
Arguments: ()
2026-04-14 20:01:28,693 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,694 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,694 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,694 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,694 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,694 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,694 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,694 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,694 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,694 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,695 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,696 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,696 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,696 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,696 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,696 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,696 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,696 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,697 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,697 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,697 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,698 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,698 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,698 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,698 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,699 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,700 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,701 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,701 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,701 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,701 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,701 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,702 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,702 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,702 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,702 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,703 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,703 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,703 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,704 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,704 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,704 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,704 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,704 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,705 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,705 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,705 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,705 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,705 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,706 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,706 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,707 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,707 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,707 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,707 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,707 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,707 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,707 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,707 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,708 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,709 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,709 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,709 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,709 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,709 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,709 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,709 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,709 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,709 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,710 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,710 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,710 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,710 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,710 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,710 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,710 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,710 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,710 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,711 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,712 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,713 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,714 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,714 - asyncio - WARNING - socket.send() raised exception.
2026-04-14 20:01:28,714 - asyncio - WARNING - socket.send() raised exception.
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()
--- Logging error ---
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions
await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 45, in publish
await self._send_server_request(request)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 87, in _send_server_request
await self._drain_writer()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/service/service_client.py", line 96, in _drain_writer
await self._writer.drain()
File "/usr/lib/python3.12/asyncio/streams.py", line 392, in drain
await self._protocol._drain_helper()
File "/usr/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper
raise ConnectionResetError('Connection lost')
ConnectionResetError: Connection lost

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/usr/lib/python3.12/logging/init.py", line 1164, in emit
self.flush()
File "/usr/lib/python3.12/logging/init.py", line 1144, in flush
self.stream.flush()
OSError: [Errno 107] Transport endpoint is not connected
Call stack:
File "/usr/lib/python3.12/threading.py", line 1032, in _bootstrap
self._bootstrap_inner()
File "/usr/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1012, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 234, in _main
self._runner.run(self._main_async)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_compat.py", line 76, in run
return asyncio.run(self._run_or_cancel(fn))
File "/usr/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
File "/usr/lib/python3.12/asyncio/base_events.py", line 678, in run_until_complete
self.run_forever()
File "/usr/lib/python3.12/asyncio/base_events.py", line 645, in run_forever
self._run_once()
File "/usr/lib/python3.12/asyncio/base_events.py", line 1999, in _run_once
handle._run()
File "/usr/lib/python3.12/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 220, in _wrap
return await fn()
File "/usr/local/lib/python3.12/dist-packages/wandb/sdk/lib/asyncio_manager.py", line 184, in fn_wrap_exceptions
_logger.exception("Uncaught exception in run_soon callback.")
Message: 'Uncaught exception in run_soon callback.'
Arguments: ()