Untitled

avatar
unknown
plain_text
a year ago
11 kB
11
Indexable
INFO 09-26 13:30:28 manager.py:157] Engine 0 status=LAUNCHED (1)
Traceback (most recent call last):
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/engine.py", line 260, in launch
    self._main_loop()
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/engine.py", line 591, in _main_loop
    out = getattr(self, inst)(request)
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/engine.py", line 462, in execute_model
    worker = self.workers[model_id]
IndexError: list index out of range
Traceback (most recent call last):
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 36, in wrapper
    return await func(*args, **kwargs)
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 204, in execute_model
    out = await self.stub.ExecuteEmbeddingModel(request)
  File "/cserve_user/.local/lib/python3.10/site-packages/grpc/aio/_call.py", line 318, in __await__
    raise _create_rpc_error(
grpc.aio._call.AioRpcError: <AioRpcError of RPC that terminated with:
        status = StatusCode.UNAVAILABLE
        details = "Socket closed"
        debug_error_string = "UNKNOWN:Error received from peer  {grpc_message:"Socket closed", grpc_status:14, created_time:"2024-09-26T13:30:28.148105605+00:00"}"
>
gRPC error happened during calling function execute_model, exiting...
ERROR:    Exception in ASGI application
Traceback (most recent call last):
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 36, in wrapper
    return await func(*args, **kwargs)
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 204, in execute_model
    out = await self.stub.ExecuteEmbeddingModel(request)
  File "/cserve_user/.local/lib/python3.10/site-packages/grpc/aio/_call.py", line 318, in __await__
    raise _create_rpc_error(
grpc.aio._call.AioRpcError: <AioRpcError of RPC that terminated with:
        status = StatusCode.UNAVAILABLE
        details = "Socket closed"
        debug_error_string = "UNKNOWN:Error received from peer  {grpc_message:"Socket closed", grpc_status:14, created_time:"2024-09-26T13:30:28.148105605+00:00"}"
>

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "uvloop/loop.pyx", line 474, in uvloop.loop.Loop._on_idle
  File "uvloop/cbhandles.pyx", line 83, in uvloop.loop.Handle._run
  File "uvloop/cbhandles.pyx", line 63, in uvloop.loop.Handle._run
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 40, in wrapper
    sys.exit(1)
SystemExit: 1

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/cserve_user/.local/lib/python3.10/site-packages/uvicorn/protocols/http/httptools_impl.py", line 401, in run_asgi
    result = await app(  # type: ignore[func-returns-value]
  File "/cserve_user/.local/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 70, in __call__
    return await self.app(scope, receive, send)
  File "/cserve_user/.local/lib/python3.10/site-packages/fastapi/applications.py", line 1054, in __call__
    await super().__call__(scope, receive, send)
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/applications.py", line 113, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/middleware/errors.py", line 165, in __call__
    await self.app(scope, receive, _send)
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/middleware/cors.py", line 85, in __call__
    await self.app(scope, receive, send)
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/_exception_handler.py", line 51, in wrapped_app
    await app(scope, receive, sender)
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/routing.py", line 715, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/routing.py", line 735, in app
    await route.handle(scope, receive, send)
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/routing.py", line 288, in handle
    await self.app(scope, receive, send)
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/routing.py", line 76, in app
    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/_exception_handler.py", line 51, in wrapped_app
    await app(scope, receive, sender)
INFO:     127.0.0.1:44096 - "POST /openai/v1/embeddings HTTP/1.1" 500 Internal Server Error
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/routing.py", line 73, in app
    response = await f(request)
  File "/cserve_user/.local/lib/python3.10/site-packages/fastapi/routing.py", line 301, in app
    raw_response = await run_endpoint_function(
  File "/cserve_user/.local/lib/python3.10/site-packages/fastapi/routing.py", line 212, in run_endpoint_function
    return await dependant.call(**values)
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/controller/api/openai.py", line 697, in create_embeddings
    embedding = await get_embedding(req)
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/controller/api/openai.py", line 711, in get_embedding
    async for output in embedding_generator:
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/controller/controller.py", line 288, in embedding
    raise e
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/controller/controller.py", line 283, in embedding
    async for request_output in stream:
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/async_llm_engine.py", line 41, in __anext__
    result = await self._queue.get()
  File "/usr/lib/python3.10/asyncio/queues.py", line 159, in get
    await getter
asyncio.exceptions.CancelledError
ERROR:    Traceback (most recent call last):
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 36, in wrapper
    return await func(*args, **kwargs)
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 204, in execute_model
    out = await self.stub.ExecuteEmbeddingModel(request)
  File "/cserve_user/.local/lib/python3.10/site-packages/grpc/aio/_call.py", line 318, in __await__
    raise _create_rpc_error(
grpc.aio._call.AioRpcError: <AioRpcError of RPC that terminated with:
        status = StatusCode.UNAVAILABLE
        details = "Socket closed"
        debug_error_string = "UNKNOWN:Error received from peer  {grpc_message:"Socket closed", grpc_status:14, created_time:"2024-09-26T13:30:28.148105605+00:00"}"
>

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "uvloop/loop.pyx", line 474, in uvloop.loop.Loop._on_idle
  File "uvloop/cbhandles.pyx", line 83, in uvloop.loop.Handle._run
  File "uvloop/cbhandles.pyx", line 63, in uvloop.loop.Handle._run
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 40, in wrapper
    sys.exit(1)
SystemExit: 1

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/cserve_user/.local/lib/python3.10/site-packages/starlette/routing.py", line 700, in lifespan
    await receive()
  File "/cserve_user/.local/lib/python3.10/site-packages/uvicorn/lifespan/on.py", line 137, in receive
    return await self.receive_queue.get()
  File "/usr/lib/python3.10/asyncio/queues.py", line 159, in get
    await getter
asyncio.exceptions.CancelledError

INFO 09-26 13:30:28 controller.py:235] Task run_loop is cancelled
Exception in callback functools.partial(<function _raise_exception_on_finish at 0x728c78daa8c0>, request_tracker=<cserve.engine.async_llm_engine.RequestTracker object at 0x728c58034f40>)
handle: <Handle functools.partial(<function _raise_exception_on_finish at 0x728c78daa8c0>, request_tracker=<cserve.engine.async_llm_engine.RequestTracker object at 0x728c58034f40>)>
Traceback (most recent call last):
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 36, in wrapper
    return await func(*args, **kwargs)
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 204, in execute_model
    out = await self.stub.ExecuteEmbeddingModel(request)
  File "/cserve_user/.local/lib/python3.10/site-packages/grpc/aio/_call.py", line 318, in __await__
    raise _create_rpc_error(
grpc.aio._call.AioRpcError: <AioRpcError of RPC that terminated with:
        status = StatusCode.UNAVAILABLE
        details = "Socket closed"
        debug_error_string = "UNKNOWN:Error received from peer  {grpc_message:"Socket closed", grpc_status:14, created_time:"2024-09-26T13:30:28.148105605+00:00"}"
>

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "uvloop/loop.pyx", line 474, in uvloop.loop.Loop._on_idle
  File "uvloop/cbhandles.pyx", line 83, in uvloop.loop.Handle._run
  File "uvloop/cbhandles.pyx", line 63, in uvloop.loop.Handle._run
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/engine/client.py", line 40, in wrapper
    sys.exit(1)
SystemExit: 1

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "uvloop/cbhandles.pyx", line 63, in uvloop.loop.Handle._run
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/controller/controller.py", line 82, in _raise_exception_on_finish
    raise exc
  File "/cserve_user/.local/lib/python3.10/site-packages/cserve/controller/controller.py", line 79, in _raise_exception_on_finish
    raise ControllerDeadError(msg)
cserve.controller.controller.ControllerDeadError: Task finished unexpectedly. This should not happen. 
WARNING 09-26 13:30:33 manager.py:165] Engine 0 gRPC is not available!
INFO 09-26 13:30:34 engine.py:230] Engine created at 0.0.0.0:5005 for GPU 0
INFO 09-26 13:30:34 engine.py:257] Engine started at 0.0.0.0:5005 for GPU 0
WARNING 09-26 13:30:35 manager.py:165] Engine 0 gRPC is not available!
INFO 09-26 13:30:35 manager.py:125] Waiting for engines to become available
INFO:     10.244.0.1:36178 - "GET /cserve/v1/health_check HTTP/1.1" 200 OK
INFO 09-26 13:30:40 manager.py:157] Engine 0 status=LAUNCHED (1)
INFO 09-26 13:30:40 manager.py:128] Engine manager started for engines [0] on ports [5005]
WARNING 09-26 13:30:40 manager.py:183] Engine manager restarted engines for [0] on ports [5005]
INFO 09-26 13:30:45 manager.py:157] Engine 0 status=LAUNCHED (1)
Editor is loading...
Leave a Comment