comments

llm/filetype.go

@@ -2,137 +2,139 @@ package llm
 
 import "fmt"
 
-type filetype uint32
+type fileType uint32
 
 const (
-	filetypeF32 filetype = iota
-	filetypeF16
-	filetypeQ4_0
-	filetypeQ4_1
-	filetypeQ4_1_F16
-	filetypeQ8_0 filetype = iota + 2
-	filetypeQ5_0
-	filetypeQ5_1
-	filetypeQ2_K
-	filetypeQ3_K_S
-	filetypeQ3_K_M
-	filetypeQ3_K_L
-	filetypeQ4_K_S
-	filetypeQ4_K_M
-	filetypeQ5_K_S
-	filetypeQ5_K_M
-	filetypeQ6_K
-	filetypeIQ2_XXS
-	filetypeIQ2_XS
-	filetypeQ2_K_S
-	filetypeQ3_K_XS
-	filetypeIQ3_XXS
+	fileTypeF32 fileType = iota
+	fileTypeF16
+	fileTypeQ4_0
+	fileTypeQ4_1
+	fileTypeQ4_1_F16
+	fileTypeQ4_2 // unused
+	fileTypeQ4_3 // unused
+	fileTypeQ8_0
+	fileTypeQ5_0
+	fileTypeQ5_1
+	fileTypeQ2_K
+	fileTypeQ3_K_S
+	fileTypeQ3_K_M
+	fileTypeQ3_K_L
+	fileTypeQ4_K_S
+	fileTypeQ4_K_M
+	fileTypeQ5_K_S
+	fileTypeQ5_K_M
+	fileTypeQ6_K
+	fileTypeIQ2_XXS
+	fileTypeIQ2_XS
+	fileTypeQ2_K_S
+	fileTypeQ3_K_XS
+	fileTypeIQ3_XXS
 
-	filetypeUnknown
+	fileTypeUnknown
 )
 
-func ParseFileType(s string) (filetype, error) {
+func ParseFileType(s string) (fileType, error) {
 	switch s {
 	case "F32":
-		return filetypeF32, nil
+		return fileTypeF32, nil
 	case "F16":
-		return filetypeF16, nil
+		return fileTypeF16, nil
 	case "Q4_0":
-		return filetypeQ4_0, nil
+		return fileTypeQ4_0, nil
 	case "Q4_1":
-		return filetypeQ4_1, nil
+		return fileTypeQ4_1, nil
 	case "Q4_1_F16":
-		return filetypeQ4_1_F16, nil
+		return fileTypeQ4_1_F16, nil
 	case "Q8_0":
-		return filetypeQ8_0, nil
+		return fileTypeQ8_0, nil
 	case "Q5_0":
-		return filetypeQ5_0, nil
+		return fileTypeQ5_0, nil
 	case "Q5_1":
-		return filetypeQ5_1, nil
+		return fileTypeQ5_1, nil
 	case "Q2_K":
-		return filetypeQ2_K, nil
+		return fileTypeQ2_K, nil
 	case "Q3_K_S":
-		return filetypeQ3_K_S, nil
+		return fileTypeQ3_K_S, nil
 	case "Q3_K_M":
-		return filetypeQ3_K_M, nil
+		return fileTypeQ3_K_M, nil
 	case "Q3_K_L":
-		return filetypeQ3_K_L, nil
+		return fileTypeQ3_K_L, nil
 	case "Q4_K_S":
-		return filetypeQ4_K_S, nil
+		return fileTypeQ4_K_S, nil
 	case "Q4_K_M":
-		return filetypeQ4_K_M, nil
+		return fileTypeQ4_K_M, nil
 	case "Q5_K_S":
-		return filetypeQ5_K_S, nil
+		return fileTypeQ5_K_S, nil
 	case "Q5_K_M":
-		return filetypeQ5_K_M, nil
+		return fileTypeQ5_K_M, nil
 	case "Q6_K":
-		return filetypeQ6_K, nil
+		return fileTypeQ6_K, nil
 	case "IQ2_XXS":
-		return filetypeIQ2_XXS, nil
+		return fileTypeIQ2_XXS, nil
 	case "IQ2_XS":
-		return filetypeIQ2_XS, nil
+		return fileTypeIQ2_XS, nil
 	case "Q2_K_S":
-		return filetypeQ2_K_S, nil
+		return fileTypeQ2_K_S, nil
 	case "Q3_K_XS":
-		return filetypeQ3_K_XS, nil
+		return fileTypeQ3_K_XS, nil
 	case "IQ3_XXS":
-		return filetypeIQ3_XXS, nil
+		return fileTypeIQ3_XXS, nil
 	default:
-		return filetypeUnknown, fmt.Errorf("unknown filetype: %s", s)
+		return fileTypeUnknown, fmt.Errorf("unknown fileType: %s", s)
 	}
 }
 
-func (t filetype) String() string {
+func (t fileType) String() string {
 	switch t {
-	case filetypeF32:
+	case fileTypeF32:
 		return "F32"
-	case filetypeF16:
+	case fileTypeF16:
 		return "F16"
-	case filetypeQ4_0:
+	case fileTypeQ4_0:
 		return "Q4_0"
-	case filetypeQ4_1:
+	case fileTypeQ4_1:
 		return "Q4_1"
-	case filetypeQ4_1_F16:
+	case fileTypeQ4_1_F16:
 		return "Q4_1_F16"
-	case filetypeQ8_0:
+	case fileTypeQ8_0:
 		return "Q8_0"
-	case filetypeQ5_0:
+	case fileTypeQ5_0:
 		return "Q5_0"
-	case filetypeQ5_1:
+	case fileTypeQ5_1:
 		return "Q5_1"
-	case filetypeQ2_K:
+	case fileTypeQ2_K:
 		return "Q2_K"
-	case filetypeQ3_K_S:
+	case fileTypeQ3_K_S:
 		return "Q3_K_S"
-	case filetypeQ3_K_M:
+	case fileTypeQ3_K_M:
 		return "Q3_K_M"
-	case filetypeQ3_K_L:
+	case fileTypeQ3_K_L:
 		return "Q3_K_L"
-	case filetypeQ4_K_S:
+	case fileTypeQ4_K_S:
 		return "Q4_K_S"
-	case filetypeQ4_K_M:
+	case fileTypeQ4_K_M:
 		return "Q4_K_M"
-	case filetypeQ5_K_S:
+	case fileTypeQ5_K_S:
 		return "Q5_K_S"
-	case filetypeQ5_K_M:
+	case fileTypeQ5_K_M:
 		return "Q5_K_M"
-	case filetypeQ6_K:
+	case fileTypeQ6_K:
 		return "Q6_K"
-	case filetypeIQ2_XXS:
+	case fileTypeIQ2_XXS:
 		return "IQ2_XXS"
-	case filetypeIQ2_XS:
+	case fileTypeIQ2_XS:
 		return "IQ2_XS"
-	case filetypeQ2_K_S:
+	case fileTypeQ2_K_S:
 		return "Q2_K_S"
-	case filetypeQ3_K_XS:
+	case fileTypeQ3_K_XS:
 		return "Q3_K_XS"
-	case filetypeIQ3_XXS:
+	case fileTypeIQ3_XXS:
 		return "IQ3_XXS"
 	default:
 		return "unknown"
 	}
 }
 
-func (t filetype) Value() uint32 {
+func (t fileType) Value() uint32 {
 	return uint32(t)
 }

llm/ggml.go

@@ -47,7 +47,7 @@ func (kv KV) ParameterCount() uint64 {
 
 func (kv KV) FileType() string {
 	if u64 := kv.u64("general.file_type"); u64 > 0 {
-		return filetype(uint32(u64)).String()
+		return fileType(uint32(u64)).String()
 	}
 
 	return "unknown"

llm/llm.go

@@ -20,7 +20,7 @@ func SystemInfo() string {
 	return C.GoString(C.llama_print_system_info())
 }
 
-func Quantize(infile, outfile string, ftype filetype) error {
+func Quantize(infile, outfile string, ftype fileType) error {
 	cinfile := C.CString(infile)
 	defer C.free(unsafe.Pointer(cinfile))
 

server/images.go

@@ -316,9 +316,9 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, c
 
 		switch c.Name {
 		case "model", "adapter":
-			var layers2 *ordered.Map[*Layer, *llm.GGML]
+			var baseLayers *ordered.Map[*Layer, *llm.GGML]
 			if name := model.ParseName(c.Args, ""); name.IsValid() {
-				layers2, err = parseFromModel(ctx, name, fn)
+				baseLayers, err = parseFromModel(ctx, name, fn)
 				if err != nil {
 					return err
 				}
@@ -334,14 +334,14 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, c
 				}
 				defer blob.Close()
 
-				layers2, err = parseFromFile(ctx, blob, fn)
+				baseLayers, err = parseFromFile(ctx, blob, fn)
 				if err != nil {
 					return err
 				}
 			} else if file, err := os.Open(realpath(modelFileDir, c.Args)); err == nil {
 				defer file.Close()
 
-				layers2, err = parseFromFile(ctx, file, fn)
+				baseLayers, err = parseFromFile(ctx, file, fn)
 				if err != nil {
 					return err
 				}
@@ -353,7 +353,7 @@ func CreateModel(ctx context.Context, name, modelFileDir, quantization string, c
 			var tempfiles []*os.File
 
 			// TODO(mxyng): replace with rangefunc
-			layers2.Items()(func(layer *Layer, ggml *llm.GGML) bool {
+			baseLayers.Items()(func(layer *Layer, ggml *llm.GGML) bool {
 				if quantization != "" && ggml != nil && ggml.Name() == "gguf" {
 					ftype, err := llm.ParseFileType(quantization)
 					if err != nil {

server/model.go

@@ -27,7 +27,11 @@ func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressRe
 			return nil, err
 		}
 
-		return parseFromModel(ctx, name, fn)
+		modelpath = ParseModelPath(name.DisplayLongest())
+		manifest, _, err = GetManifest(modelpath)
+		if err != nil {
+			return nil, err
+		}
 	case err != nil:
 		return nil, err
 	}