ggerganov commited on
Commit
b1dfc60
·
unverified ·
1 Parent(s): 72754e0

Add helper script to benchmark all models

Browse files

Simply run:

$ ./extra/bench-all.sh

Files changed (1) hide show
  1. extra/bench-all.sh +53 -0
extra/bench-all.sh ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # Helper script to run the bench tool on all models and print the results in share-able format
4
+
5
+ printf "Usage: ./bench.sh [n_threads]\n"
6
+
7
+ if [ -z "$1" ]; then
8
+ n_threads=4
9
+ else
10
+ n_threads=$1
11
+ fi
12
+
13
+ models=( "tiny" "base" "small" "medium" "large" )
14
+
15
+ printf "\n"
16
+ printf "Running benchmark for all models\n"
17
+ printf "This can take a while!\n"
18
+ printf "\n"
19
+
20
+ printf "| CPU | OS | Config | Model | Threads | Load [ms] | Encode [ms] |\n"
21
+ printf "| --- | -- | ------ | ----- | ------- | --------- | ----------- |\n"
22
+
23
+ for model in "${models[@]}"; do
24
+ # run once to heat-up the cache
25
+ ./bench -m ./models/ggml-$model.bin -t $n_threads 2>/dev/null 1>/dev/null
26
+
27
+ # actual run
28
+ # store stderr output in a variable in order to parse it later
29
+ output=$(./bench -m ./models/ggml-$model.bin -t $n_threads 2>&1)
30
+
31
+ # parse the output:
32
+ load_time=$(echo "$output" | grep "load time" | awk '{print $5}')
33
+ encode_time=$(echo "$output" | grep "encode time" | awk '{print $5}')
34
+ system_info=$(echo "$output" | grep "system_info")
35
+ n_threads=$(echo "$output" | grep "system_info" | awk '{print $4}')
36
+
37
+ config=""
38
+
39
+ if [[ $system_info == *"AVX2 = 1"* ]]; then
40
+ config="$config AVX2"
41
+ fi
42
+
43
+ if [[ $system_info == *"NEON = 1"* ]]; then
44
+ config="$config NEON"
45
+ fi
46
+
47
+ if [[ $system_info == *"BLAS = 1"* ]]; then
48
+ config="$config BLAS"
49
+ fi
50
+
51
+ printf "| <todo> | <todo> | $config | $model | $n_threads | $load_time | $encode_time |\n"
52
+ done
53
+