|
|
@@ -5,7 +5,7 @@ package llm
|
|
|
/*
|
|
|
|
|
|
#include <stdlib.h>
|
|
|
-#include "rocm_shim.h"
|
|
|
+#include "dynamic_shim.h"
|
|
|
|
|
|
*/
|
|
|
import "C"
|
|
|
@@ -18,20 +18,20 @@ import (
|
|
|
"log"
|
|
|
"os"
|
|
|
"path/filepath"
|
|
|
- "runtime"
|
|
|
+ "strings"
|
|
|
"sync"
|
|
|
"unsafe"
|
|
|
|
|
|
"github.com/jmorganca/ollama/api"
|
|
|
)
|
|
|
|
|
|
-//go:embed llama.cpp/gguf/build/*/lib/*
|
|
|
+//go:embed llama.cpp/gguf/build/lib/*
|
|
|
var libEmbed embed.FS
|
|
|
|
|
|
var RocmShimMissing = fmt.Errorf("ROCm shim library not included in this build of ollama. Radeon GPUs are not supported")
|
|
|
|
|
|
type shimExtServer struct {
|
|
|
- s C.struct_rocm_llama_server
|
|
|
+ s C.struct_dynamic_llama_server
|
|
|
options api.Options
|
|
|
}
|
|
|
|
|
|
@@ -40,50 +40,58 @@ var shimMutex sync.Mutex
|
|
|
var llm *shimExtServer
|
|
|
|
|
|
func (llm *shimExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) {
|
|
|
- C.rocm_shim_llama_server_init(llm.s, sparams, err)
|
|
|
+ C.dynamic_shim_llama_server_init(llm.s, sparams, err)
|
|
|
}
|
|
|
func (llm *shimExtServer) llama_server_start() {
|
|
|
- C.rocm_shim_llama_server_start(llm.s)
|
|
|
+ C.dynamic_shim_llama_server_start(llm.s)
|
|
|
}
|
|
|
func (llm *shimExtServer) llama_server_stop() {
|
|
|
- C.rocm_shim_llama_server_stop(llm.s)
|
|
|
+ C.dynamic_shim_llama_server_stop(llm.s)
|
|
|
}
|
|
|
|
|
|
func (llm *shimExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) {
|
|
|
- C.rocm_shim_llama_server_completion(llm.s, json_req, resp)
|
|
|
+ C.dynamic_shim_llama_server_completion(llm.s, json_req, resp)
|
|
|
}
|
|
|
func (llm *shimExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) {
|
|
|
- C.rocm_shim_llama_server_completion_next_result(llm.s, task_id, resp)
|
|
|
+ C.dynamic_shim_llama_server_completion_next_result(llm.s, task_id, resp)
|
|
|
}
|
|
|
func (llm *shimExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) {
|
|
|
- C.rocm_shim_llama_server_completion_cancel(llm.s, task_id, err)
|
|
|
+ C.dynamic_shim_llama_server_completion_cancel(llm.s, task_id, err)
|
|
|
}
|
|
|
func (llm *shimExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) {
|
|
|
- C.rocm_shim_llama_server_release_task_result(llm.s, result)
|
|
|
+ C.dynamic_shim_llama_server_release_task_result(llm.s, result)
|
|
|
}
|
|
|
|
|
|
func (llm *shimExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
|
|
|
- C.rocm_shim_llama_server_tokenize(llm.s, json_req, json_resp, err)
|
|
|
+ C.dynamic_shim_llama_server_tokenize(llm.s, json_req, json_resp, err)
|
|
|
}
|
|
|
func (llm *shimExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
|
|
|
- C.rocm_shim_llama_server_detokenize(llm.s, json_req, json_resp, err)
|
|
|
+ C.dynamic_shim_llama_server_detokenize(llm.s, json_req, json_resp, err)
|
|
|
}
|
|
|
func (llm *shimExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
|
|
|
- C.rocm_shim_llama_server_embedding(llm.s, json_req, json_resp, err)
|
|
|
+ C.dynamic_shim_llama_server_embedding(llm.s, json_req, json_resp, err)
|
|
|
}
|
|
|
func (llm *shimExtServer) llama_server_release_json_resp(json_resp **C.char) {
|
|
|
- C.rocm_shim_llama_server_release_json_resp(llm.s, json_resp)
|
|
|
+ C.dynamic_shim_llama_server_release_json_resp(llm.s, json_resp)
|
|
|
}
|
|
|
|
|
|
-func newRocmShimExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
|
|
|
- if !ShimPresent {
|
|
|
- return nil, RocmShimMissing
|
|
|
+func newDynamicShimExtServer(library, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
|
|
|
+ shimMutex.Lock()
|
|
|
+ defer shimMutex.Unlock()
|
|
|
+ libPath := C.CString(library)
|
|
|
+ defer C.free(unsafe.Pointer(libPath))
|
|
|
+ resp := newExtServerResp(128)
|
|
|
+ defer freeExtServerResp(resp)
|
|
|
+ var srv C.struct_dynamic_llama_server
|
|
|
+ C.dynamic_shim_init(libPath, &srv, &resp)
|
|
|
+ if resp.id < 0 {
|
|
|
+ return nil, fmt.Errorf("Unable to load dynamic library: %s", C.GoString(resp.msg))
|
|
|
}
|
|
|
- log.Printf("Loading ROCM llm server")
|
|
|
- if llm == nil {
|
|
|
- return nil, fmt.Errorf("nativeInit wasnt called or libary load failed")
|
|
|
+ llm = &shimExtServer{
|
|
|
+ s: srv,
|
|
|
+ options: opts,
|
|
|
}
|
|
|
- llm.options = opts
|
|
|
+ log.Printf("Loading Dynamic Shim llm server: %s", library)
|
|
|
return newExtServer(llm, model, adapters, projectors, numLayers, opts)
|
|
|
}
|
|
|
|
|
|
@@ -108,64 +116,37 @@ func (llm *shimExtServer) Close() {
|
|
|
}
|
|
|
|
|
|
func nativeInit(workdir string) error {
|
|
|
- err := extractLib(workdir, "llama.cpp/gguf/build/*/lib/*rocm_server*")
|
|
|
+ libs, err := extractDynamicLibs(workdir, "llama.cpp/gguf/build/lib/*server*")
|
|
|
if err != nil {
|
|
|
if err == payloadMissing {
|
|
|
- log.Printf("%s", RocmShimMissing)
|
|
|
+ log.Printf("%s", payloadMissing)
|
|
|
return nil
|
|
|
}
|
|
|
return err
|
|
|
- } else {
|
|
|
- ShimPresent = true
|
|
|
+ }
|
|
|
+ for _, lib := range libs {
|
|
|
+ libName := strings.Split(strings.TrimPrefix(filepath.Base(lib), "lib"), ".")[0]
|
|
|
+ AvailableShims[libName] = lib
|
|
|
}
|
|
|
|
|
|
- // Verify we have permissions - either running as root, or we have group access to the driver
|
|
|
- fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0666)
|
|
|
- if err != nil {
|
|
|
- if errors.Is(err, fs.ErrPermission) {
|
|
|
- log.Fatalf("Radeon card detected, but permissions not set up properly. Either run ollama as root, or add you user account to the render group.")
|
|
|
- return err
|
|
|
- } else if errors.Is(err, fs.ErrNotExist) {
|
|
|
- // expected behavior without a radeon card
|
|
|
- return nil
|
|
|
+ // Only check ROCm access if we have the dynamic lib loaded
|
|
|
+ if _, rocmPresent := AvailableShims["rocm_server"]; rocmPresent {
|
|
|
+ // Verify we have permissions - either running as root, or we have group access to the driver
|
|
|
+ fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0666)
|
|
|
+ if err != nil {
|
|
|
+ if errors.Is(err, fs.ErrPermission) {
|
|
|
+ log.Fatalf("Radeon card detected, but permissions not set up properly. Either run ollama as root, or add you user account to the render group.")
|
|
|
+ return err
|
|
|
+ } else if errors.Is(err, fs.ErrNotExist) {
|
|
|
+ // expected behavior without a radeon card
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+
|
|
|
+ return fmt.Errorf("failed to check permission on /dev/kfd: %w", err)
|
|
|
}
|
|
|
+ fd.Close()
|
|
|
|
|
|
- return fmt.Errorf("failed to check permission on /dev/kfd: %w", err)
|
|
|
}
|
|
|
- fd.Close()
|
|
|
|
|
|
- shimMutex.Lock()
|
|
|
- defer shimMutex.Unlock()
|
|
|
- if llm != nil {
|
|
|
- return nil
|
|
|
- }
|
|
|
- var libName string
|
|
|
- switch runtime.GOOS {
|
|
|
- case "darwin":
|
|
|
- // shouldn't happen
|
|
|
- return nil
|
|
|
- case "linux":
|
|
|
- libName = "librocm_server.so"
|
|
|
- case "windows":
|
|
|
- libName = "rocm_server.dll"
|
|
|
- default:
|
|
|
- // shouldn't happen
|
|
|
- return nil
|
|
|
- }
|
|
|
- libPath := C.CString(filepath.Join(workdir, libName))
|
|
|
- defer C.free(unsafe.Pointer(libPath))
|
|
|
- resp := newExtServerResp(128)
|
|
|
- defer freeExtServerResp(resp)
|
|
|
- var srv C.struct_rocm_llama_server
|
|
|
- C.rocm_shim_init(libPath, &srv, &resp)
|
|
|
- if resp.id < 0 {
|
|
|
- // TODO - consider softening this failure mode to allow fall-back to the CUDA based built-in llm
|
|
|
- // and run against CPU
|
|
|
- return fmt.Errorf("Unable to load AMD GPU library: %s", C.GoString(resp.msg))
|
|
|
- }
|
|
|
- llm = &shimExtServer{
|
|
|
- s: srv,
|
|
|
- options: api.DefaultOptions(),
|
|
|
- }
|
|
|
return nil
|
|
|
}
|
Daniel Hiltgen