Browse Source

Load all layers on `arm64` macOS if model is small enough (#2149)

Jeffrey Morgan 1 year ago
parent
commit
4458efb73a
1 changed files with 2 additions and 1 deletions
  1. 2 1
      llm/llm.go

+ 2 - 1
llm/llm.go

@@ -70,7 +70,8 @@ func New(workDir, model string, adapters, projectors []string, opts api.Options)
 			break
 		}
 
-		opts.NumGPU = 1
+		// TODO: implement layer splitting on macOS
+		opts.NumGPU = 999
 	default:
 		if info.Library == "cpu" {
 			slog.Info("GPU not available, falling back to CPU")