瀏覽代碼

Handle very slow model loads

During testing, we're seeing some models take over 3 minutes.
Daniel Hiltgen 1 年之前
父節點
當前提交
c5ff443b9f
共有 1 個文件被更改,包括 2 次插入1 次删除
  1. 2 1
      llm/server.go

+ 2 - 1
llm/server.go

@@ -383,7 +383,8 @@ func (s *LlamaServer) Ping(ctx context.Context) error {
 
 
 func (s *LlamaServer) waitUntilRunning() error {
 func (s *LlamaServer) waitUntilRunning() error {
 	start := time.Now()
 	start := time.Now()
-	expiresAt := time.Now().Add(3 * time.Minute) // be generous with timeout, large models can take a while to load
+	// TODO we need to wire up a better way to detect hangs during model load and startup of the server
+	expiresAt := time.Now().Add(10 * time.Minute) // be generous with timeout, large models can take a while to load
 	ticker := time.NewTicker(50 * time.Millisecond)
 	ticker := time.NewTicker(50 * time.Millisecond)
 	defer ticker.Stop()
 	defer ticker.Stop()