input

llm/ext_server/server.cpp

@@ -1855,6 +1855,8 @@ struct llama_server_context
 
         if (batch.n_tokens == 0)
         {
+            // HANGING HERE
+            LOG_INFO("no tokens to process", {});
             all_slots_are_idle = true;
             return true;
         }
@@ -3200,7 +3202,6 @@ int main(int argc, char **argv) {
                 else {
                     image_data = "";
                 }
-                // TODO: prompt needs to represent the image data
 
                 // create and queue the task
                 json responses;

server/routes.go

@@ -265,38 +265,42 @@ func (s *Server) EmbedHandler(c *gin.Context) {
 		truncate = false
 	}
 
-	inputCheck := true
+	var input []string
 
-	if req.Images != nil {
-		inputCheck = false
+	images := make([]llm.ImageData, len(req.Images))
+	for i := range req.Images {
+		images[i] = llm.ImageData{ID: 0, Data: req.Images[i]}
 	}
 
-	var input []string
-
-	if inputCheck {
+	if req.Images != nil {
+		// for _, _ := range images {
+		// input = append(input, fmt.Sprintf("[img-%d]", i.ID))
+		input = append(input, "[img-0]")
+		// }
+		req.Input = ""
+	}
 
-		switch i := req.Input.(type) {
-		case string:
-			if len(i) > 0 {
-				input = append(input, i)
-			}
-		case []any:
-			for _, v := range i {
-				if _, ok := v.(string); !ok {
-					c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
-					return
-				}
-				input = append(input, v.(string))
+	switch i := req.Input.(type) {
+	case string:
+		if len(i) > 0 {
+			input = append(input, i)
+		}
+	case []any:
+		for _, v := range i {
+			if _, ok := v.(string); !ok {
+				c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
+				return
 			}
-		default:
-			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
-			return
+			input = append(input, v.(string))
 		}
+	default:
+		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
+		return
+	}
 
-		if len(input) == 0 {
-			c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
-			return
-		}
+	if len(input) == 0 {
+		c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
+		return
 	}
 
 	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
@@ -336,11 +340,6 @@ func (s *Server) EmbedHandler(c *gin.Context) {
 		input[i] = s
 	}
 
-	images := make([]llm.ImageData, len(req.Images))
-	for i := range req.Images {
-		images[i] = llm.ImageData{ID: i, Data: req.Images[i]}
-	}
-
 	embeddings, err := r.Embed(c.Request.Context(), input, images)
 
 	if err != nil {