Przeglądaj źródła

fix index out of range on zero layer metal load (#7696)

If the model doesn't fit any layers on metal, and we load zero layers
we would panic trying to look up the GPU size during scheduling ops
Daniel Hiltgen 5 miesięcy temu
rodzic
commit
81d55d3e4d
1 zmienionych plików z 3 dodań i 1 usunięć
  1. 3 1
      llm/server.go

+ 3 - 1
llm/server.go

@@ -1092,7 +1092,9 @@ func (s *llmServer) EstimatedTotal() uint64 {
 func (s *llmServer) EstimatedVRAMByGPU(gpuID string) uint64 {
 	for i, gpu := range s.gpus {
 		if gpu.ID == gpuID {
-			return s.estimate.GPUSizes[i]
+			if i < len(s.estimate.GPUSizes) {
+				return s.estimate.GPUSizes[i]
+			}
 		}
 	}
 	return 0