Refactoring of legacy and new

api/types.go

@@ -246,7 +246,7 @@ type EmbeddingRequest struct {
 
 // EmbeddingResponse is the response from [Client.Embeddings].
 type EmbeddingResponse struct {
-	Embedding []float64 `json:"embedding"`
+	Embedding []float32 `json:"embedding"`
 }
 
 // CreateRequest is the request passed to [Client.Create].

llm/ext_server/server.cpp

@@ -3191,7 +3191,6 @@ int main(int argc, char **argv) {
                         if (result.result_json.count("results")) {
                             // result for multi-task
                             responses = result.result_json.at("results");
-                            LOG_INFO("results", {result.result_json});
                         } else {
                             // result for single task
                             responses = std::vector<json>(1, result.result_json);

llm/server.go

@@ -33,7 +33,7 @@ type LlamaServer interface {
 	Ping(ctx context.Context) error
 	WaitUntilRunning(ctx context.Context) error
 	Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error
-	Embedding(ctx context.Context, prompt string) ([]float64, error)
+	// Embedding(ctx context.Context, prompt string) ([]float64, error)
 	Embed(ctx context.Context, input []string) ([][]float32, error)
 	Tokenize(ctx context.Context, content string) ([]int, error)
 	Detokenize(ctx context.Context, tokens []int) (string, error)
@@ -905,64 +905,6 @@ func (s *llmServer) Embed(ctx context.Context, input []string) ([][]float32, err
 	return embedding.Embedding, nil
 }
 
-type EmbeddingRequest struct {
-	Content string `json:"content"`
-}
-
-type EmbeddingResponse struct {
-	Embedding [][]float64 `json:"embedding"`
-}
-
-func (s *llmServer) Embedding(ctx context.Context, prompt string) ([]float64, error) {
-	if err := s.sem.Acquire(ctx, 1); err != nil {
-		slog.Error("Failed to acquire semaphore", "error", err)
-		return nil, err
-	}
-	defer s.sem.Release(1)
-
-	// Make sure the server is ready
-	status, err := s.getServerStatusRetry(ctx)
-	if err != nil {
-		return nil, err
-	} else if status != ServerStatusReady {
-		return nil, fmt.Errorf("unexpected server status: %s", status.ToString())
-	}
-
-	data, err := json.Marshal(EmbeddingRequest{Content: prompt})
-	if err != nil {
-		return nil, fmt.Errorf("error marshaling embed data: %w", err)
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/embedding", s.port), bytes.NewBuffer(data))
-	if err != nil {
-		return nil, fmt.Errorf("error creating embed request: %w", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("do embedding request: %w", err)
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("error reading embed response: %w", err)
-	}
-
-	if resp.StatusCode >= 400 {
-		log.Printf("llm encode error: %s", body)
-		return nil, fmt.Errorf("%s", body)
-	}
-
-	var embedding EmbeddingResponse
-	if err := json.Unmarshal(body, &embedding); err != nil {
-		return nil, fmt.Errorf("unmarshal tokenize response: %w", err)
-	}
-
-	return embedding.Embedding[0], nil
-}
-
 type TokenizeRequest struct {
 	Content string `json:"content"`
 }

server/routes.go

@@ -554,11 +554,11 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) {
 
 	// an empty request loads the model
 	if req.Prompt == "" {
-		c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float64{}})
+		c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float32{}})
 		return
 	}
 
-	embedding, err := runner.llama.Embedding(c.Request.Context(), req.Prompt)
+	embedding, err := runner.llama.Embed(c.Request.Context(), []string{req.Prompt})
 	if err != nil {
 		slog.Info(fmt.Sprintf("embedding generation failed: %v", err))
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
@@ -566,7 +566,7 @@ func (s *Server) EmbeddingsHandler(c *gin.Context) {
 	}
 
 	resp := api.EmbeddingResponse{
-		Embedding: embedding,
+		Embedding: embedding[0],
 	}
 	c.JSON(http.StatusOK, resp)
 }

server/sched_test.go

@@ -630,8 +630,6 @@ type mockLlm struct {
 	pingResp           error
 	waitResp           error
 	completionResp     error
-	embeddingResp      []float64
-	embeddingRespErr   error
 	embedResp          [][]float32
 	embedRespErr       error
 	tokenizeResp       []int
@@ -650,9 +648,10 @@ func (s *mockLlm) WaitUntilRunning(ctx context.Context) error { return s.waitRes
 func (s *mockLlm) Completion(ctx context.Context, req llm.CompletionRequest, fn func(llm.CompletionResponse)) error {
 	return s.completionResp
 }
-func (s *mockLlm) Embedding(ctx context.Context, prompt string) ([]float64, error) {
-	return s.embeddingResp, s.embeddingRespErr
-}
+
+//	func (s *mockLlm) Embedding(ctx context.Context, prompt string) ([]float64, error) {
+//		return s.embeddingResp, s.embeddingRespErr
+//	}
 func (s *mockLlm) Embed(ctx context.Context, input []string) ([][]float32, error) {
 	return s.embedResp, s.embedRespErr
 }