Merge pull request #955 from jmorganca/mattw/example-bash-compare

docs: add examples using bash to compare models

examples/bash-comparemodels/README.md

@@ -0,0 +1,10 @@
+# Bash Shell examples
+
+When calling `ollama`, you can pass it a file to run all the prompts in the file, one after the other:
+
+`ollama run llama2 < sourcequestions.txt`
+
+This concept is used in the following example.
+
+## Compare Models
+`comparemodels.sh` is a script that runs all the questions in `sourcequestions.txt` using any 4 models you choose that you have already pulled from the Ollama library or have created locally.

examples/bash-comparemodels/comparemodels.sh

@@ -0,0 +1,64 @@
+#! /usr/bin/env bash
+# Compare multiple models by running them with the same questions
+
+NUMBEROFCHOICES=4
+SELECTIONS=()
+declare -a SUMS=()
+
+# Get the list of models
+CHOICES=$(ollama list | awk '{print $1}')
+
+# Select which models to run as a comparison
+echo "Select $NUMBEROFCHOICES models to compare:"
+select ITEM in $CHOICES; do
+    if [[ -n $ITEM ]]; then
+        echo "You have selected $ITEM"
+        SELECTIONS+=("$ITEM")
+        ((COUNT++))
+        if [[ $COUNT -eq $NUMBEROFCHOICES ]]; then
+            break
+        fi
+    else
+        echo "Invalid selection"
+    fi
+done
+
+# Loop through each of the selected models
+for ITEM in "${SELECTIONS[@]}"; do
+    echo "--------------------------------------------------------------"
+    echo "Loading the model $ITEM into memory"
+    ollama run "$ITEM" ""
+    echo "--------------------------------------------------------------"
+    echo "Running the questions through the model $ITEM"
+    COMMAND_OUTPUT=$(ollama run "$ITEM" --verbose < sourcequestions.txt 2>&1| tee /dev/stderr)
+
+    # eval duration is sometimes listed in seconds and sometimes in milliseconds. 
+    # Add up the values for each model
+    SUM=$(echo "$COMMAND_OUTPUT" | awk '
+    /eval duration:/ {
+        value = $3
+        if (index(value, "ms") > 0) {
+            gsub("ms", "", value)
+            value /= 1000
+        } else {
+            gsub("s", "", value)
+        }
+        sum += value
+    }
+    END { print sum }')
+
+
+    SUMS+=("All questions for $ITEM completed in $SUM seconds")
+done
+
+echo ""
+echo "--------------------------------------------------------------"
+echo -e "Sums of eval durations for each run:"
+for val in "${SUMS[@]}"; do
+    echo "$val"
+done
+
+echo "--------------------------------------------------------------"
+echo "Comparison complete. Now you can decide"
+echo "which model is best."
+echo "--------------------------------------------------------------"

examples/bash-comparemodels/sourcequestions.txt

@@ -0,0 +1,7 @@
+Why is the sky blue
+What is a black hole
+Explain the big bang theory like I am 5?
+What is the quickest way to win a game of Monopoly with 3 others?
+Why does a vacuum bottle keep my coffee hot and my milkshake cold?
+What is the difference between a meteor, a meteorite, and a meteoroid?
+Create an array with 5 items and print to the console. Do this in Python, C#, Typescript, and Rust.