Spaces:
Running
Running
Add helper script to benchmark all models
Browse filesSimply run:
$ ./extra/bench-all.sh
- extra/bench-all.sh +53 -0
extra/bench-all.sh
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
|
| 3 |
+
# Helper script to run the bench tool on all models and print the results in share-able format
|
| 4 |
+
|
| 5 |
+
printf "Usage: ./bench.sh [n_threads]\n"
|
| 6 |
+
|
| 7 |
+
if [ -z "$1" ]; then
|
| 8 |
+
n_threads=4
|
| 9 |
+
else
|
| 10 |
+
n_threads=$1
|
| 11 |
+
fi
|
| 12 |
+
|
| 13 |
+
models=( "tiny" "base" "small" "medium" "large" )
|
| 14 |
+
|
| 15 |
+
printf "\n"
|
| 16 |
+
printf "Running benchmark for all models\n"
|
| 17 |
+
printf "This can take a while!\n"
|
| 18 |
+
printf "\n"
|
| 19 |
+
|
| 20 |
+
printf "| CPU | OS | Config | Model | Threads | Load [ms] | Encode [ms] |\n"
|
| 21 |
+
printf "| --- | -- | ------ | ----- | ------- | --------- | ----------- |\n"
|
| 22 |
+
|
| 23 |
+
for model in "${models[@]}"; do
|
| 24 |
+
# run once to heat-up the cache
|
| 25 |
+
./bench -m ./models/ggml-$model.bin -t $n_threads 2>/dev/null 1>/dev/null
|
| 26 |
+
|
| 27 |
+
# actual run
|
| 28 |
+
# store stderr output in a variable in order to parse it later
|
| 29 |
+
output=$(./bench -m ./models/ggml-$model.bin -t $n_threads 2>&1)
|
| 30 |
+
|
| 31 |
+
# parse the output:
|
| 32 |
+
load_time=$(echo "$output" | grep "load time" | awk '{print $5}')
|
| 33 |
+
encode_time=$(echo "$output" | grep "encode time" | awk '{print $5}')
|
| 34 |
+
system_info=$(echo "$output" | grep "system_info")
|
| 35 |
+
n_threads=$(echo "$output" | grep "system_info" | awk '{print $4}')
|
| 36 |
+
|
| 37 |
+
config=""
|
| 38 |
+
|
| 39 |
+
if [[ $system_info == *"AVX2 = 1"* ]]; then
|
| 40 |
+
config="$config AVX2"
|
| 41 |
+
fi
|
| 42 |
+
|
| 43 |
+
if [[ $system_info == *"NEON = 1"* ]]; then
|
| 44 |
+
config="$config NEON"
|
| 45 |
+
fi
|
| 46 |
+
|
| 47 |
+
if [[ $system_info == *"BLAS = 1"* ]]; then
|
| 48 |
+
config="$config BLAS"
|
| 49 |
+
fi
|
| 50 |
+
|
| 51 |
+
printf "| <todo> | <todo> | $config | $model | $n_threads | $load_time | $encode_time |\n"
|
| 52 |
+
done
|
| 53 |
+
|