|
@@ -1,64 +0,0 @@
|
|
|
-#! /usr/bin/env bash
|
|
|
-# Compare multiple models by running them with the same questions
|
|
|
-
|
|
|
-NUMBEROFCHOICES=4
|
|
|
-SELECTIONS=()
|
|
|
-declare -a SUMS=()
|
|
|
-
|
|
|
-# Get the list of models
|
|
|
-CHOICES=$(ollama list | awk '{print $1}')
|
|
|
-
|
|
|
-# Select which models to run as a comparison
|
|
|
-echo "Select $NUMBEROFCHOICES models to compare:"
|
|
|
-select ITEM in $CHOICES; do
|
|
|
- if [[ -n $ITEM ]]; then
|
|
|
- echo "You have selected $ITEM"
|
|
|
- SELECTIONS+=("$ITEM")
|
|
|
- ((COUNT++))
|
|
|
- if [[ $COUNT -eq $NUMBEROFCHOICES ]]; then
|
|
|
- break
|
|
|
- fi
|
|
|
- else
|
|
|
- echo "Invalid selection"
|
|
|
- fi
|
|
|
-done
|
|
|
-
|
|
|
-# Loop through each of the selected models
|
|
|
-for ITEM in "${SELECTIONS[@]}"; do
|
|
|
- echo "--------------------------------------------------------------"
|
|
|
- echo "Loading the model $ITEM into memory"
|
|
|
- ollama run "$ITEM" ""
|
|
|
- echo "--------------------------------------------------------------"
|
|
|
- echo "Running the questions through the model $ITEM"
|
|
|
- COMMAND_OUTPUT=$(ollama run "$ITEM" --verbose < sourcequestions.txt 2>&1| tee /dev/stderr)
|
|
|
-
|
|
|
- # eval duration is sometimes listed in seconds and sometimes in milliseconds.
|
|
|
- # Add up the values for each model
|
|
|
- SUM=$(echo "$COMMAND_OUTPUT" | awk '
|
|
|
- /eval duration:/ {
|
|
|
- value = $3
|
|
|
- if (index(value, "ms") > 0) {
|
|
|
- gsub("ms", "", value)
|
|
|
- value /= 1000
|
|
|
- } else {
|
|
|
- gsub("s", "", value)
|
|
|
- }
|
|
|
- sum += value
|
|
|
- }
|
|
|
- END { print sum }')
|
|
|
-
|
|
|
-
|
|
|
- SUMS+=("All questions for $ITEM completed in $SUM seconds")
|
|
|
-done
|
|
|
-
|
|
|
-echo ""
|
|
|
-echo "--------------------------------------------------------------"
|
|
|
-echo -e "Sums of eval durations for each run:"
|
|
|
-for val in "${SUMS[@]}"; do
|
|
|
- echo "$val"
|
|
|
-done
|
|
|
-
|
|
|
-echo "--------------------------------------------------------------"
|
|
|
-echo "Comparison complete. Now you can decide"
|
|
|
-echo "which model is best."
|
|
|
-echo "--------------------------------------------------------------"
|