1dcc1e11d5
- Update Makefile with native build targets (preparing for C++) - Add profiler and performance regression detector commands - Update CI/testing scripts - Add additional unit tests for API, jupyter, queue, manifest
721 lines
29 KiB
Bash
Executable file
721 lines
29 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Simple performance tracking script
|
|
|
|
set -euo pipefail
|
|
|
|
RESULTS_DIR="test_results/performance"
|
|
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
|
RESULTS_FILE="$RESULTS_DIR/load_test_$TIMESTAMP.json"
|
|
RESULTS_TMP_FILE="$RESULTS_FILE.tmp"
|
|
RAW_LOG_FILE="$RESULTS_DIR/raw_$TIMESTAMP.log"
|
|
RESOURCES_DIR="$RESULTS_DIR/resources_$TIMESTAMP"
|
|
|
|
PROM_URL="${PROM_URL:-}"
|
|
PROM_INSTANCE="${PROM_INSTANCE:-}"
|
|
PROM_NODE_JOB="${PROM_NODE_JOB:-node}" # e.g. node-exporter job label
|
|
PROM_REDIS_JOB="${PROM_REDIS_JOB:-redis}" # e.g. redis-exporter job label
|
|
PROM_NET_IFACE="${PROM_NET_IFACE:-eth0}" # override for server interface
|
|
PROM_DISK_DEVICE="${PROM_DISK_DEVICE:-}" # e.g. nvme0n1, sda; empty => aggregate all
|
|
PROM_STEP_SECONDS="${PROM_STEP_SECONDS:-5}"
|
|
|
|
mkdir -p "$RESULTS_DIR"
|
|
mkdir -p "$RESOURCES_DIR"
|
|
|
|
echo "Running load test performance tracking..."
|
|
echo "Timestamp: $TIMESTAMP"
|
|
|
|
json_string_or_null() {
|
|
if [ -z "${1:-}" ]; then
|
|
echo "null"
|
|
return
|
|
fi
|
|
printf '%s' "$1" | jq -Rs '.'
|
|
}
|
|
|
|
json_number_or_null() {
|
|
if [ -z "${1:-}" ]; then
|
|
echo "null"
|
|
return
|
|
fi
|
|
if printf '%s' "$1" | grep -Eq '^[0-9]+(\.[0-9]+)?$'; then
|
|
echo "$1"
|
|
return
|
|
fi
|
|
echo "null"
|
|
}
|
|
|
|
prom_urlencode() {
|
|
python3 - <<'PY' "$1" 2>/dev/null || true
|
|
import sys
|
|
from urllib.parse import quote
|
|
|
|
print(quote(sys.argv[1], safe=''))
|
|
PY
|
|
}
|
|
|
|
prom_query_range() {
|
|
local query="$1"
|
|
local start="$2"
|
|
local end="$3"
|
|
local step="$4"
|
|
|
|
if [ -z "${PROM_URL:-}" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
local q
|
|
q=$(prom_urlencode "$query")
|
|
if [ -z "${q:-}" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
curl -fsS "${PROM_URL%/}/api/v1/query_range?query=${q}&start=${start}&end=${end}&step=${step}" 2>/dev/null || true
|
|
}
|
|
|
|
prom_series_avg_max() {
|
|
# Emits: "<avg> <max>" or empty
|
|
jq -r '
|
|
if .status != "success" then empty
|
|
else
|
|
(.data.result[0].values // [])
|
|
| map(.[1] | tonumber)
|
|
| if length == 0 then empty
|
|
else
|
|
{avg: (add/length), max: max}
|
|
| "\(.avg) \(.max)"
|
|
end
|
|
end
|
|
' 2>/dev/null || true
|
|
}
|
|
|
|
prom_scalar_last() {
|
|
jq -r '
|
|
if .status != "success" then empty
|
|
else
|
|
(.data.result[0].values // [])
|
|
| if length == 0 then empty else .[-1][1] end
|
|
end
|
|
' 2>/dev/null || true
|
|
}
|
|
|
|
bytes_or_empty() {
|
|
if [ -z "${1:-}" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
if printf '%s' "$1" | grep -Eq '^[0-9]+$'; then
|
|
echo "$1"
|
|
return 0
|
|
fi
|
|
echo ""
|
|
}
|
|
|
|
read_net_bytes() {
|
|
local ifname="$1"
|
|
netstat -ibn 2>/dev/null \
|
|
| awk -v ifname="$ifname" '$1==ifname && $2 ~ /^[0-9]+$/ {print $7 " " $10; exit}' \
|
|
|| true
|
|
}
|
|
|
|
sample_process() {
|
|
local pid="$1"
|
|
local out_file="$2"
|
|
: > "$out_file"
|
|
while kill -0 "$pid" 2>/dev/null; do
|
|
local cpu rss
|
|
cpu=$(ps -p "$pid" -o %cpu= 2>/dev/null | tr -d ' ' || true)
|
|
rss=$(ps -p "$pid" -o rss= 2>/dev/null | tr -d ' ' || true)
|
|
printf '%s,%s,%s\n' "$(date +%s)" "${cpu:-}" "${rss:-}" >> "$out_file" || true
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
summarize_process_samples() {
|
|
local file="$1"
|
|
if [ ! -f "$file" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
awk -F',' '
|
|
$2 ~ /^[0-9]+(\.[0-9]+)?$/ {cpu_sum += $2; cpu_n += 1; if ($2 > cpu_max) cpu_max = $2}
|
|
$3 ~ /^[0-9]+$/ {if ($3 > rss_max) rss_max = $3}
|
|
END {
|
|
if (cpu_n > 0) printf "cpu_avg=%.4f cpu_max=%.4f rss_max_kb=%d", cpu_sum/cpu_n, cpu_max, rss_max
|
|
}
|
|
' "$file" 2>/dev/null || true
|
|
}
|
|
|
|
summarize_iostat() {
|
|
local file="$1"
|
|
if [ ! -f "$file" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
awk '
|
|
$1 ~ /^[0-9]/ {
|
|
if (NF % 3 == 0 && NF >= 3) {
|
|
mb = 0
|
|
for (i = 3; i <= NF; i += 3) mb += $i
|
|
sum += mb
|
|
n += 1
|
|
if (mb > max) max = mb
|
|
}
|
|
}
|
|
END {
|
|
if (n > 0) printf "mbps_avg=%.4f mbps_max=%.4f", sum/n, max
|
|
}
|
|
' "$file" 2>/dev/null || true
|
|
}
|
|
|
|
extract_kv() {
|
|
local file="$1"
|
|
local key="$2"
|
|
if [ ! -f "$file" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
awk -F: -v k="$key" '$1==k {gsub(/^ +/, "", $2); print $2; exit}' "$file" 2>/dev/null || true
|
|
}
|
|
|
|
extract_field() {
|
|
local label="$1"
|
|
local grep_pat="$2"
|
|
local sed_expr="$3"
|
|
|
|
local line
|
|
line=$(awk -v label="$label" -v pat="$grep_pat" '
|
|
$0 ~ "Load test results for " label ":" {inside=1; next}
|
|
inside && $0 ~ /^=== RUN/ {inside=0}
|
|
inside && $0 ~ pat {print; exit}
|
|
' "$RESULTS_DIR/raw_$TIMESTAMP.log" 2>/dev/null || true)
|
|
if [ -z "$line" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
printf '%s\n' "$line" \
|
|
| sed -E "$sed_expr" \
|
|
| tr -d '\r' \
|
|
|| true
|
|
return 0
|
|
}
|
|
|
|
extract_criteria_field_from_log() {
|
|
local log_file="$1"
|
|
local label="$2"
|
|
local grep_pat="$3"
|
|
local sed_expr="$4"
|
|
|
|
local line
|
|
line=$(awk -v label="$label" -v pat="$grep_pat" '
|
|
$0 ~ "Load test criteria for " label ":" {inside=1; next}
|
|
inside && ($0 ~ /^=== RUN/ || $0 ~ "Load test config for " label ":" || $0 ~ "Load test results for " label ":") {inside=0}
|
|
inside && $0 ~ pat {print; exit}
|
|
' "$log_file" 2>/dev/null || true)
|
|
if [ -z "$line" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
printf '%s\n' "$line" \
|
|
| sed -E "$sed_expr" \
|
|
| tr -d '\r' \
|
|
|| true
|
|
return 0
|
|
}
|
|
|
|
extract_config_field_from_log() {
|
|
local log_file="$1"
|
|
local label="$2"
|
|
local grep_pat="$3"
|
|
local sed_expr="$4"
|
|
|
|
local line
|
|
line=$(awk -v label="$label" -v pat="$grep_pat" '
|
|
$0 ~ "Load test config for " label ":" {inside=1; next}
|
|
inside && ($0 ~ /^=== RUN/ || $0 ~ "Load test results for " label ":") {inside=0}
|
|
inside && $0 ~ pat {print; exit}
|
|
' "$log_file" 2>/dev/null || true)
|
|
if [ -z "$line" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
printf '%s\n' "$line" \
|
|
| sed -E "$sed_expr" \
|
|
| tr -d '\r' \
|
|
|| true
|
|
return 0
|
|
}
|
|
|
|
extract_field_from_log() {
|
|
local log_file="$1"
|
|
local label="$2"
|
|
local grep_pat="$3"
|
|
local sed_expr="$4"
|
|
|
|
local line
|
|
line=$(awk -v label="$label" -v pat="$grep_pat" '
|
|
$0 ~ "Load test results for " label ":" {inside=1; next}
|
|
inside && $0 ~ /^=== RUN/ {inside=0}
|
|
inside && $0 ~ pat {print; exit}
|
|
' "$log_file" 2>/dev/null || true)
|
|
if [ -z "$line" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
printf '%s\n' "$line" \
|
|
| sed -E "$sed_expr" \
|
|
| tr -d '\r' \
|
|
|| true
|
|
return 0
|
|
}
|
|
|
|
get_light_rps() {
|
|
local results_file="$1"
|
|
local v
|
|
v=$(jq -r '.tests[] | select(.name=="LightLoad") | .throughput_rps // empty' "$results_file" 2>/dev/null || true)
|
|
if printf '%s' "${v:-}" | grep -Eq '^[0-9]+(\.[0-9]+)?$'; then
|
|
echo "$v"
|
|
return 0
|
|
fi
|
|
|
|
local ts
|
|
ts=$(jq -r '.timestamp // empty' "$results_file" 2>/dev/null || true)
|
|
if [ -z "${ts:-}" ]; then
|
|
ts=$(basename "$results_file" | sed -E 's/^load_test_([0-9]{8}_[0-9]{6})\.json$/\1/')
|
|
fi
|
|
if [ -z "${ts:-}" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
local raw_file="$RESULTS_DIR/raw_${ts}.log"
|
|
if [ ! -f "$raw_file" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
extract_field_from_log "$raw_file" "LightLoad" "Throughput:" 's/.*Throughput: ([0-9]+(\.[0-9]+)?) RPS.*/\1/'
|
|
}
|
|
|
|
get_test_criteria_summary() {
|
|
local results_file="$1"
|
|
local name="$2"
|
|
|
|
local min_t max_e max_p
|
|
min_t=$(jq -r --arg n "$name" '.tests[] | select(.name==$n) | .criteria.min_throughput_rps // empty' "$results_file" 2>/dev/null || true)
|
|
max_e=$(jq -r --arg n "$name" '.tests[] | select(.name==$n) | .criteria.max_error_rate_percent // empty' "$results_file" 2>/dev/null || true)
|
|
max_p=$(jq -r --arg n "$name" '.tests[] | select(.name==$n) | .criteria.max_p99_latency_ms // empty' "$results_file" 2>/dev/null || true)
|
|
|
|
if [ -z "${min_t:-}" ] && [ -z "${max_e:-}" ] && [ -z "${max_p:-}" ]; then
|
|
echo ""
|
|
return 0
|
|
fi
|
|
|
|
echo ">= ${min_t:-?} RPS, <= ${max_e:-?}% err, <= ${max_p:-?}ms p99"
|
|
}
|
|
|
|
# Run tests and capture results
|
|
GO_TEST_EXIT_CODE=0
|
|
|
|
test_start_epoch="$(date +%s)"
|
|
|
|
lo0_before=$(read_net_bytes lo0)
|
|
en0_before=$(read_net_bytes en0)
|
|
|
|
if command -v redis-cli >/dev/null 2>&1; then
|
|
redis-cli -n 7 INFO memory > "$RESOURCES_DIR/redis_memory_before.txt" 2>/dev/null || true
|
|
redis-cli -n 7 INFO clients > "$RESOURCES_DIR/redis_clients_before.txt" 2>/dev/null || true
|
|
fi
|
|
|
|
go test ./tests/load -run=TestLoadTestSuite -v -load-suite=medium -timeout=10m > "$RAW_LOG_FILE" 2>&1 &
|
|
GO_TEST_PID=$!
|
|
|
|
sample_process "$GO_TEST_PID" "$RESOURCES_DIR/process.csv" &
|
|
SAMPLE_PID=$!
|
|
|
|
iostat -d 1 > "$RESOURCES_DIR/iostat.txt" 2>/dev/null &
|
|
IOSTAT_PID=$!
|
|
|
|
wait "$GO_TEST_PID" || GO_TEST_EXIT_CODE=$?
|
|
|
|
test_end_epoch="$(date +%s)"
|
|
|
|
kill "$IOSTAT_PID" 2>/dev/null || true
|
|
wait "$IOSTAT_PID" 2>/dev/null || true
|
|
|
|
wait "$SAMPLE_PID" 2>/dev/null || true
|
|
|
|
lo0_after=$(read_net_bytes lo0)
|
|
en0_after=$(read_net_bytes en0)
|
|
|
|
if command -v redis-cli >/dev/null 2>&1; then
|
|
redis-cli -n 7 INFO memory > "$RESOURCES_DIR/redis_memory_after.txt" 2>/dev/null || true
|
|
redis-cli -n 7 INFO clients > "$RESOURCES_DIR/redis_clients_after.txt" 2>/dev/null || true
|
|
fi
|
|
|
|
if command -v podman >/dev/null 2>&1; then
|
|
podman stats --no-stream --format '{{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.NetIO}} {{.BlockIO}}' > "$RESOURCES_DIR/podman_stats.txt" 2>/dev/null || true
|
|
fi
|
|
|
|
if [ "$GO_TEST_EXIT_CODE" -ne 0 ]; then
|
|
echo "Load test failed (exit code: $GO_TEST_EXIT_CODE). See raw log: $RAW_LOG_FILE" >&2
|
|
fi
|
|
|
|
proc_summary=$(summarize_process_samples "$RESOURCES_DIR/process.csv")
|
|
cpu_avg=$(printf '%s' "$proc_summary" | sed -nE 's/.*cpu_avg=([0-9]+(\.[0-9]+)?).*/\1/p')
|
|
cpu_max=$(printf '%s' "$proc_summary" | sed -nE 's/.*cpu_max=([0-9]+(\.[0-9]+)?).*/\1/p')
|
|
rss_max_kb=$(printf '%s' "$proc_summary" | sed -nE 's/.*rss_max_kb=([0-9]+).*/\1/p')
|
|
|
|
io_summary=$(summarize_iostat "$RESOURCES_DIR/iostat.txt")
|
|
disk_mbps_avg=$(printf '%s' "$io_summary" | sed -nE 's/.*mbps_avg=([0-9]+(\.[0-9]+)?).*/\1/p')
|
|
disk_mbps_max=$(printf '%s' "$io_summary" | sed -nE 's/.*mbps_max=([0-9]+(\.[0-9]+)?).*/\1/p')
|
|
|
|
lo0_in_before=$(bytes_or_empty "$(printf '%s' "$lo0_before" | awk '{print $1}')")
|
|
lo0_out_before=$(bytes_or_empty "$(printf '%s' "$lo0_before" | awk '{print $2}')")
|
|
lo0_in_after=$(bytes_or_empty "$(printf '%s' "$lo0_after" | awk '{print $1}')")
|
|
lo0_out_after=$(bytes_or_empty "$(printf '%s' "$lo0_after" | awk '{print $2}')")
|
|
|
|
en0_in_before=$(bytes_or_empty "$(printf '%s' "$en0_before" | awk '{print $1}')")
|
|
en0_out_before=$(bytes_or_empty "$(printf '%s' "$en0_before" | awk '{print $2}')")
|
|
en0_in_after=$(bytes_or_empty "$(printf '%s' "$en0_after" | awk '{print $1}')")
|
|
en0_out_after=$(bytes_or_empty "$(printf '%s' "$en0_after" | awk '{print $2}')")
|
|
|
|
lo0_in_delta=""
|
|
lo0_out_delta=""
|
|
if [ -n "${lo0_in_before:-}" ] && [ -n "${lo0_in_after:-}" ]; then
|
|
lo0_in_delta=$((lo0_in_after - lo0_in_before))
|
|
fi
|
|
if [ -n "${lo0_out_before:-}" ] && [ -n "${lo0_out_after:-}" ]; then
|
|
lo0_out_delta=$((lo0_out_after - lo0_out_before))
|
|
fi
|
|
|
|
en0_in_delta=""
|
|
en0_out_delta=""
|
|
if [ -n "${en0_in_before:-}" ] && [ -n "${en0_in_after:-}" ]; then
|
|
en0_in_delta=$((en0_in_after - en0_in_before))
|
|
fi
|
|
if [ -n "${en0_out_before:-}" ] && [ -n "${en0_out_after:-}" ]; then
|
|
en0_out_delta=$((en0_out_after - en0_out_before))
|
|
fi
|
|
|
|
redis_used_mem_before=$(extract_kv "$RESOURCES_DIR/redis_memory_before.txt" used_memory)
|
|
redis_used_mem_after=$(extract_kv "$RESOURCES_DIR/redis_memory_after.txt" used_memory)
|
|
redis_clients_before=$(extract_kv "$RESOURCES_DIR/redis_clients_before.txt" connected_clients)
|
|
redis_clients_after=$(extract_kv "$RESOURCES_DIR/redis_clients_after.txt" connected_clients)
|
|
|
|
run_mode="local"
|
|
metrics_scope="client_only"
|
|
if [ -n "${PROM_URL:-}" ]; then
|
|
run_mode="remote"
|
|
metrics_scope="both"
|
|
fi
|
|
|
|
server_cpu_avg=""; server_cpu_max=""
|
|
server_mem_avg=""; server_mem_max=""
|
|
server_disk_read_avg=""; server_disk_read_max=""
|
|
server_disk_write_avg=""; server_disk_write_max=""
|
|
server_net_rx_avg=""; server_net_rx_max=""
|
|
server_net_tx_avg=""; server_net_tx_max=""
|
|
server_redis_mem_last=""; server_redis_clients_last=""
|
|
|
|
if [ -n "${PROM_URL:-}" ] && [ -n "${PROM_INSTANCE:-}" ]; then
|
|
instance_sel="instance=\"${PROM_INSTANCE}\",job=\"${PROM_NODE_JOB}\""
|
|
|
|
cpu_q="(1 - avg(rate(node_cpu_seconds_total{${instance_sel},mode=\"idle\"}[1m])))*100"
|
|
mem_q="node_memory_MemAvailable_bytes{${instance_sel}}"
|
|
mem_total_q="node_memory_MemTotal_bytes{${instance_sel}}"
|
|
|
|
cpu_json=$(prom_query_range "$cpu_q" "$test_start_epoch" "$test_end_epoch" "$PROM_STEP_SECONDS")
|
|
cpu_pair=$(printf '%s' "$cpu_json" | prom_series_avg_max)
|
|
server_cpu_avg=$(printf '%s' "$cpu_pair" | awk '{print $1}')
|
|
server_cpu_max=$(printf '%s' "$cpu_pair" | awk '{print $2}')
|
|
|
|
avail_json=$(prom_query_range "$mem_q" "$test_start_epoch" "$test_end_epoch" "$PROM_STEP_SECONDS")
|
|
total_json=$(prom_query_range "$mem_total_q" "$test_start_epoch" "$test_end_epoch" "$PROM_STEP_SECONDS")
|
|
avail_pair=$(printf '%s' "$avail_json" | prom_series_avg_max)
|
|
total_pair=$(printf '%s' "$total_json" | prom_series_avg_max)
|
|
avail_avg=$(printf '%s' "$avail_pair" | awk '{print $1}')
|
|
avail_min=$(printf '%s' "$avail_pair" | awk '{print $2}')
|
|
total_avg=$(printf '%s' "$total_pair" | awk '{print $1}')
|
|
|
|
if printf '%s' "${avail_avg:-}" | grep -Eq '^[0-9]+(\.[0-9]+)?$' && printf '%s' "${total_avg:-}" | grep -Eq '^[0-9]+(\.[0-9]+)?$'; then
|
|
server_mem_avg=$(awk -v a="$avail_avg" -v t="$total_avg" 'BEGIN{printf "%.0f", (t-a)}')
|
|
fi
|
|
if printf '%s' "${avail_min:-}" | grep -Eq '^[0-9]+(\.[0-9]+)?$' && printf '%s' "${total_avg:-}" | grep -Eq '^[0-9]+(\.[0-9]+)?$'; then
|
|
server_mem_max=$(awk -v a="$avail_min" -v t="$total_avg" 'BEGIN{printf "%.0f", (t-a)}')
|
|
fi
|
|
|
|
disk_dev_sel=""
|
|
if [ -n "${PROM_DISK_DEVICE:-}" ]; then
|
|
disk_dev_sel=",device=\"${PROM_DISK_DEVICE}\""
|
|
fi
|
|
read_q="sum(rate(node_disk_read_bytes_total{${instance_sel}${disk_dev_sel}}[1m]))"
|
|
write_q="sum(rate(node_disk_written_bytes_total{${instance_sel}${disk_dev_sel}}[1m]))"
|
|
|
|
read_json=$(prom_query_range "$read_q" "$test_start_epoch" "$test_end_epoch" "$PROM_STEP_SECONDS")
|
|
write_json=$(prom_query_range "$write_q" "$test_start_epoch" "$test_end_epoch" "$PROM_STEP_SECONDS")
|
|
read_pair=$(printf '%s' "$read_json" | prom_series_avg_max)
|
|
write_pair=$(printf '%s' "$write_json" | prom_series_avg_max)
|
|
server_disk_read_avg=$(printf '%s' "$read_pair" | awk '{print $1}')
|
|
server_disk_read_max=$(printf '%s' "$read_pair" | awk '{print $2}')
|
|
server_disk_write_avg=$(printf '%s' "$write_pair" | awk '{print $1}')
|
|
server_disk_write_max=$(printf '%s' "$write_pair" | awk '{print $2}')
|
|
|
|
rx_q="sum(rate(node_network_receive_bytes_total{${instance_sel},device=\"${PROM_NET_IFACE}\"}[1m]))"
|
|
tx_q="sum(rate(node_network_transmit_bytes_total{${instance_sel},device=\"${PROM_NET_IFACE}\"}[1m]))"
|
|
rx_json=$(prom_query_range "$rx_q" "$test_start_epoch" "$test_end_epoch" "$PROM_STEP_SECONDS")
|
|
tx_json=$(prom_query_range "$tx_q" "$test_start_epoch" "$test_end_epoch" "$PROM_STEP_SECONDS")
|
|
rx_pair=$(printf '%s' "$rx_json" | prom_series_avg_max)
|
|
tx_pair=$(printf '%s' "$tx_json" | prom_series_avg_max)
|
|
server_net_rx_avg=$(printf '%s' "$rx_pair" | awk '{print $1}')
|
|
server_net_rx_max=$(printf '%s' "$rx_pair" | awk '{print $2}')
|
|
server_net_tx_avg=$(printf '%s' "$tx_pair" | awk '{print $1}')
|
|
server_net_tx_max=$(printf '%s' "$tx_pair" | awk '{print $2}')
|
|
|
|
if [ -n "${PROM_REDIS_JOB:-}" ]; then
|
|
redis_sel="instance=\"${PROM_INSTANCE}\",job=\"${PROM_REDIS_JOB}\""
|
|
redis_mem_q="redis_memory_used_bytes{${redis_sel}}"
|
|
redis_clients_q="redis_connected_clients{${redis_sel}}"
|
|
redis_mem_json=$(prom_query_range "$redis_mem_q" "$test_start_epoch" "$test_end_epoch" "$PROM_STEP_SECONDS")
|
|
redis_clients_json=$(prom_query_range "$redis_clients_q" "$test_start_epoch" "$test_end_epoch" "$PROM_STEP_SECONDS")
|
|
server_redis_mem_last=$(printf '%s' "$redis_mem_json" | prom_scalar_last)
|
|
server_redis_clients_last=$(printf '%s' "$redis_clients_json" | prom_scalar_last)
|
|
fi
|
|
fi
|
|
|
|
# Extract key metrics
|
|
{
|
|
echo "{"
|
|
echo " \"timestamp\": \"$TIMESTAMP\","
|
|
echo " \"go_test_exit_code\": $GO_TEST_EXIT_CODE,"
|
|
echo " \"run_context\": {"
|
|
echo " \"mode\": \"$run_mode\","
|
|
echo " \"metrics_scope\": \"$metrics_scope\","
|
|
echo " \"prom_url\": $(json_string_or_null "$PROM_URL"),"
|
|
echo " \"prom_instance\": $(json_string_or_null "$PROM_INSTANCE"),"
|
|
echo " \"prom_node_job\": $(json_string_or_null "$PROM_NODE_JOB"),"
|
|
echo " \"prom_redis_job\": $(json_string_or_null "$PROM_REDIS_JOB"),"
|
|
echo " \"prom_net_iface\": $(json_string_or_null "$PROM_NET_IFACE"),"
|
|
echo " \"prom_disk_device\": $(json_string_or_null "$PROM_DISK_DEVICE"),"
|
|
echo " \"test_start_epoch\": $(json_number_or_null "$test_start_epoch"),"
|
|
echo " \"test_end_epoch\": $(json_number_or_null "$test_end_epoch")"
|
|
echo " },"
|
|
echo " \"resources\": {"
|
|
echo " \"process\": {"
|
|
echo " \"cpu_percent_avg\": $(json_number_or_null "$cpu_avg"),"
|
|
echo " \"cpu_percent_max\": $(json_number_or_null "$cpu_max"),"
|
|
echo " \"rss_max_kb\": $(json_number_or_null "$rss_max_kb")"
|
|
echo " },"
|
|
echo " \"disk\": {"
|
|
echo " \"mbps_avg\": $(json_number_or_null "$disk_mbps_avg"),"
|
|
echo " \"mbps_max\": $(json_number_or_null "$disk_mbps_max")"
|
|
echo " },"
|
|
echo " \"network\": {"
|
|
echo " \"lo0_in_bytes\": $(json_number_or_null "$lo0_in_delta"),"
|
|
echo " \"lo0_out_bytes\": $(json_number_or_null "$lo0_out_delta"),"
|
|
echo " \"en0_in_bytes\": $(json_number_or_null "$en0_in_delta"),"
|
|
echo " \"en0_out_bytes\": $(json_number_or_null "$en0_out_delta")"
|
|
echo " },"
|
|
echo " \"redis\": {"
|
|
echo " \"used_memory_bytes_before\": $(json_number_or_null "$redis_used_mem_before"),"
|
|
echo " \"used_memory_bytes_after\": $(json_number_or_null "$redis_used_mem_after"),"
|
|
echo " \"connected_clients_before\": $(json_number_or_null "$redis_clients_before"),"
|
|
echo " \"connected_clients_after\": $(json_number_or_null "$redis_clients_after")"
|
|
echo " },"
|
|
echo " \"podman_stats\": $(json_string_or_null "$(cat \"$RESOURCES_DIR/podman_stats.txt\" 2>/dev/null || true)")"
|
|
echo " },"
|
|
echo " \"resources_server\": {"
|
|
echo " \"cpu_percent_avg\": $(json_number_or_null "$server_cpu_avg"),"
|
|
echo " \"cpu_percent_max\": $(json_number_or_null "$server_cpu_max"),"
|
|
echo " \"mem_used_bytes_avg\": $(json_number_or_null "$server_mem_avg"),"
|
|
echo " \"mem_used_bytes_max\": $(json_number_or_null "$server_mem_max"),"
|
|
echo " \"disk_read_bytes_per_sec_avg\": $(json_number_or_null "$server_disk_read_avg"),"
|
|
echo " \"disk_read_bytes_per_sec_max\": $(json_number_or_null "$server_disk_read_max"),"
|
|
echo " \"disk_write_bytes_per_sec_avg\": $(json_number_or_null "$server_disk_write_avg"),"
|
|
echo " \"disk_write_bytes_per_sec_max\": $(json_number_or_null "$server_disk_write_max"),"
|
|
echo " \"net_rx_bytes_per_sec_avg\": $(json_number_or_null "$server_net_rx_avg"),"
|
|
echo " \"net_rx_bytes_per_sec_max\": $(json_number_or_null "$server_net_rx_max"),"
|
|
echo " \"net_tx_bytes_per_sec_avg\": $(json_number_or_null "$server_net_tx_avg"),"
|
|
echo " \"net_tx_bytes_per_sec_max\": $(json_number_or_null "$server_net_tx_max"),"
|
|
echo " \"redis_used_memory_bytes_last\": $(json_number_or_null "$server_redis_mem_last"),"
|
|
echo " \"redis_connected_clients_last\": $(json_number_or_null "$server_redis_clients_last")"
|
|
echo " },"
|
|
echo " \"tests\": ["
|
|
|
|
# Parse light load
|
|
LIGHT_RPS=$(extract_field "LightLoad" "Throughput:" 's/.*Throughput: ([0-9]+(\.[0-9]+)?) RPS.*/\1/')
|
|
LIGHT_ERROR=$(extract_field "LightLoad" "Error rate:" 's/.*Error rate: ([0-9]+(\.[0-9]+)?)%.*/\1/')
|
|
LIGHT_P99=$(extract_field "LightLoad" "P99 latency:" 's/.*P99 latency: ([^ ]+).*/\1/')
|
|
|
|
LIGHT_CRIT_MIN_RPS=$(extract_criteria_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "LightLoad" "Min throughput:" 's/.*Min throughput: ([0-9]+(\.[0-9]+)?) RPS.*/\1/')
|
|
LIGHT_CRIT_MAX_ERR=$(extract_criteria_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "LightLoad" "Max error rate:" 's/.*Max error rate: ([0-9]+(\.[0-9]+)?)%.*/\1/')
|
|
LIGHT_CRIT_MAX_P99=$(extract_criteria_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "LightLoad" "Max P99 latency:" 's/.*Max P99 latency: ([0-9]+(\.[0-9]+)?)ms.*/\1/')
|
|
|
|
LIGHT_CFG_CONCURRENCY=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "LightLoad" "Concurrency:" 's/.*Concurrency: ([0-9]+).*/\1/')
|
|
LIGHT_CFG_DURATION=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "LightLoad" "Duration:" 's/.*Duration: (.*)$/\1/')
|
|
LIGHT_CFG_RAMP_UP=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "LightLoad" "Ramp up:" 's/.*Ramp up: (.*)$/\1/')
|
|
LIGHT_CFG_TARGET_RPS=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "LightLoad" "Target RPS:" 's/.*Target RPS: ([0-9]+).*/\1/')
|
|
LIGHT_CFG_PAYLOAD_SIZE=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "LightLoad" "Payload size:" 's/.*Payload size: ([0-9]+).*/\1/')
|
|
LIGHT_CFG_METHOD=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "LightLoad" "Method:" 's/.*Method: (.*)$/\1/')
|
|
LIGHT_CFG_ENDPOINT=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "LightLoad" "Endpoint:" 's/.*Endpoint: (.*)$/\1/')
|
|
|
|
echo " {"
|
|
echo " \"name\": \"LightLoad\","
|
|
echo " \"config\": {"
|
|
echo " \"concurrency\": $(json_number_or_null "$LIGHT_CFG_CONCURRENCY"),"
|
|
echo " \"duration\": $(json_string_or_null "$LIGHT_CFG_DURATION"),"
|
|
echo " \"ramp_up\": $(json_string_or_null "$LIGHT_CFG_RAMP_UP"),"
|
|
echo " \"target_rps\": $(json_number_or_null "$LIGHT_CFG_TARGET_RPS"),"
|
|
echo " \"payload_size_bytes\": $(json_number_or_null "$LIGHT_CFG_PAYLOAD_SIZE"),"
|
|
echo " \"method\": $(json_string_or_null "$LIGHT_CFG_METHOD"),"
|
|
echo " \"endpoint\": $(json_string_or_null "$LIGHT_CFG_ENDPOINT")"
|
|
echo " },"
|
|
echo " \"criteria\": {"
|
|
echo " \"min_throughput_rps\": $(json_number_or_null "$LIGHT_CRIT_MIN_RPS"),"
|
|
echo " \"max_error_rate_percent\": $(json_number_or_null "$LIGHT_CRIT_MAX_ERR"),"
|
|
echo " \"max_p99_latency_ms\": $(json_number_or_null "$LIGHT_CRIT_MAX_P99")"
|
|
echo " },"
|
|
echo " \"throughput_rps\": $(json_number_or_null "$LIGHT_RPS"),"
|
|
echo " \"error_rate_percent\": $(json_number_or_null "$LIGHT_ERROR"),"
|
|
echo " \"p99_latency_ms\": $(json_string_or_null "$LIGHT_P99")"
|
|
echo " },"
|
|
|
|
# Parse medium load
|
|
MEDIUM_RPS=$(extract_field "MediumLoad" "Throughput:" 's/.*Throughput: ([0-9]+(\.[0-9]+)?) RPS.*/\1/')
|
|
MEDIUM_ERROR=$(extract_field "MediumLoad" "Error rate:" 's/.*Error rate: ([0-9]+(\.[0-9]+)?)%.*/\1/')
|
|
MEDIUM_P99=$(extract_field "MediumLoad" "P99 latency:" 's/.*P99 latency: ([^ ]+).*/\1/')
|
|
|
|
MEDIUM_CRIT_MIN_RPS=$(extract_criteria_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "MediumLoad" "Min throughput:" 's/.*Min throughput: ([0-9]+(\.[0-9]+)?) RPS.*/\1/')
|
|
MEDIUM_CRIT_MAX_ERR=$(extract_criteria_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "MediumLoad" "Max error rate:" 's/.*Max error rate: ([0-9]+(\.[0-9]+)?)%.*/\1/')
|
|
MEDIUM_CRIT_MAX_P99=$(extract_criteria_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "MediumLoad" "Max P99 latency:" 's/.*Max P99 latency: ([0-9]+(\.[0-9]+)?)ms.*/\1/')
|
|
|
|
MEDIUM_CFG_CONCURRENCY=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "MediumLoad" "Concurrency:" 's/.*Concurrency: ([0-9]+).*/\1/')
|
|
MEDIUM_CFG_DURATION=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "MediumLoad" "Duration:" 's/.*Duration: (.*)$/\1/')
|
|
MEDIUM_CFG_RAMP_UP=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "MediumLoad" "Ramp up:" 's/.*Ramp up: (.*)$/\1/')
|
|
MEDIUM_CFG_TARGET_RPS=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "MediumLoad" "Target RPS:" 's/.*Target RPS: ([0-9]+).*/\1/')
|
|
MEDIUM_CFG_PAYLOAD_SIZE=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "MediumLoad" "Payload size:" 's/.*Payload size: ([0-9]+).*/\1/')
|
|
MEDIUM_CFG_METHOD=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "MediumLoad" "Method:" 's/.*Method: (.*)$/\1/')
|
|
MEDIUM_CFG_ENDPOINT=$(extract_config_field_from_log "$RESULTS_DIR/raw_$TIMESTAMP.log" "MediumLoad" "Endpoint:" 's/.*Endpoint: (.*)$/\1/')
|
|
|
|
echo " {"
|
|
echo " \"name\": \"MediumLoad\","
|
|
echo " \"config\": {"
|
|
echo " \"concurrency\": $(json_number_or_null "$MEDIUM_CFG_CONCURRENCY"),"
|
|
echo " \"duration\": $(json_string_or_null "$MEDIUM_CFG_DURATION"),"
|
|
echo " \"ramp_up\": $(json_string_or_null "$MEDIUM_CFG_RAMP_UP"),"
|
|
echo " \"target_rps\": $(json_number_or_null "$MEDIUM_CFG_TARGET_RPS"),"
|
|
echo " \"payload_size_bytes\": $(json_number_or_null "$MEDIUM_CFG_PAYLOAD_SIZE"),"
|
|
echo " \"method\": $(json_string_or_null "$MEDIUM_CFG_METHOD"),"
|
|
echo " \"endpoint\": $(json_string_or_null "$MEDIUM_CFG_ENDPOINT")"
|
|
echo " },"
|
|
echo " \"criteria\": {"
|
|
echo " \"min_throughput_rps\": $(json_number_or_null "$MEDIUM_CRIT_MIN_RPS"),"
|
|
echo " \"max_error_rate_percent\": $(json_number_or_null "$MEDIUM_CRIT_MAX_ERR"),"
|
|
echo " \"max_p99_latency_ms\": $(json_number_or_null "$MEDIUM_CRIT_MAX_P99")"
|
|
echo " },"
|
|
echo " \"throughput_rps\": $(json_number_or_null "$MEDIUM_RPS"),"
|
|
echo " \"error_rate_percent\": $(json_number_or_null "$MEDIUM_ERROR"),"
|
|
echo " \"p99_latency_ms\": $(json_string_or_null "$MEDIUM_P99")"
|
|
echo " }"
|
|
echo " ]"
|
|
echo "}"
|
|
} > "$RESULTS_TMP_FILE"
|
|
|
|
# Write atomically so a partial file never becomes a "previous run".
|
|
mv "$RESULTS_TMP_FILE" "$RESULTS_FILE"
|
|
|
|
echo "Results saved to: $RESULTS_FILE"
|
|
echo "Raw logs: $RAW_LOG_FILE"
|
|
|
|
# Show comparison with previous run if exists
|
|
PREV_FILE=$(ls -t "$RESULTS_DIR"/load_test_*.json | sed -n '2p')
|
|
if [ -n "$PREV_FILE" ]; then
|
|
echo ""
|
|
echo "=== Comparison with previous run ==="
|
|
echo "Previous: $(basename $PREV_FILE)"
|
|
echo "Current: $(basename $RESULTS_FILE)"
|
|
echo ""
|
|
echo "Light Load Throughput:"
|
|
|
|
crit=$(get_test_criteria_summary "$RESULTS_FILE" "LightLoad")
|
|
if [ -n "${crit:-}" ]; then
|
|
echo " Criteria: $crit"
|
|
fi
|
|
if ! jq -e . "$PREV_FILE" >/dev/null 2>&1; then
|
|
echo " Previous: (invalid JSON; skipping comparison)"
|
|
exit 0
|
|
fi
|
|
if ! jq -e . "$RESULTS_FILE" >/dev/null 2>&1; then
|
|
echo " Current: (invalid JSON; skipping comparison)"
|
|
exit 0
|
|
fi
|
|
|
|
prev=$(get_light_rps "$PREV_FILE")
|
|
curr=$(get_light_rps "$RESULTS_FILE")
|
|
echo " Previous: ${prev:-N/A} RPS"
|
|
echo " Current: ${curr:-N/A} RPS"
|
|
if printf '%s' "${prev:-}" | grep -Eq '^[0-9]+(\.[0-9]+)?$' && printf '%s' "${curr:-}" | grep -Eq '^[0-9]+(\.[0-9]+)?$'; then
|
|
delta=$(awk -v a="$curr" -v b="$prev" 'BEGIN{printf "%.4f", (a-b)}')
|
|
echo " Change: $delta RPS"
|
|
else
|
|
echo " Change: N/A"
|
|
fi
|
|
|
|
cpu=$(jq -r '.resources.process.cpu_percent_avg // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
rss=$(jq -r '.resources.process.rss_max_kb // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
disk=$(jq -r '.resources.disk.mbps_avg // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
net_in=$(jq -r '.resources.network.lo0_in_bytes // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
redis_mem=$(jq -r '.resources.redis.used_memory_bytes_after // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
redis_clients=$(jq -r '.resources.redis.connected_clients_after // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
|
|
if [ -n "${cpu:-}" ] || [ -n "${rss:-}" ] || [ -n "${disk:-}" ] || [ -n "${net_in:-}" ] || [ -n "${redis_mem:-}" ] || [ -n "${redis_clients:-}" ]; then
|
|
echo " Resources (current run):"
|
|
echo " CPU avg%: ${cpu:-N/A}"
|
|
echo " RSS max KB: ${rss:-N/A}"
|
|
echo " Disk MB/s avg: ${disk:-N/A}"
|
|
echo " Network lo0 in bytes: ${net_in:-N/A}"
|
|
echo " Redis used_memory (after): ${redis_mem:-N/A}"
|
|
echo " Redis connected_clients (after): ${redis_clients:-N/A}"
|
|
fi
|
|
|
|
server_cpu=$(jq -r '.resources_server.cpu_percent_avg // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
server_disk_r=$(jq -r '.resources_server.disk_read_bytes_per_sec_avg // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
server_disk_w=$(jq -r '.resources_server.disk_write_bytes_per_sec_avg // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
server_net_rx=$(jq -r '.resources_server.net_rx_bytes_per_sec_avg // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
server_mem=$(jq -r '.resources_server.mem_used_bytes_avg // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
if [ -n "${server_cpu:-}" ] || [ -n "${server_disk_r:-}" ] || [ -n "${server_disk_w:-}" ] || [ -n "${server_net_rx:-}" ] || [ -n "${server_mem:-}" ]; then
|
|
echo " Server metrics (Prometheus):"
|
|
echo " CPU avg%: ${server_cpu:-N/A}"
|
|
echo " Mem used avg bytes: ${server_mem:-N/A}"
|
|
echo " Disk read B/s avg: ${server_disk_r:-N/A}"
|
|
echo " Disk write B/s avg: ${server_disk_w:-N/A}"
|
|
echo " Net rx B/s avg: ${server_net_rx:-N/A}"
|
|
fi
|
|
|
|
|
|
echo ""
|
|
echo "Medium Load Throughput:"
|
|
|
|
crit=$(get_test_criteria_summary "$RESULTS_FILE" "MediumLoad")
|
|
if [ -n "${crit:-}" ]; then
|
|
echo " Criteria: $crit"
|
|
fi
|
|
|
|
prev=$(jq -r '.tests[] | select(.name=="MediumLoad") | .throughput_rps // empty' "$PREV_FILE" 2>/dev/null || true)
|
|
curr=$(jq -r '.tests[] | select(.name=="MediumLoad") | .throughput_rps // empty' "$RESULTS_FILE" 2>/dev/null || true)
|
|
echo " Previous: ${prev:-N/A} RPS"
|
|
echo " Current: ${curr:-N/A} RPS"
|
|
if printf '%s' "${prev:-}" | grep -Eq '^[0-9]+(\.[0-9]+)?$' && printf '%s' "${curr:-}" | grep -Eq '^[0-9]+(\.[0-9]+)?$'; then
|
|
delta=$(awk -v a="$curr" -v b="$prev" 'BEGIN{printf "%.4f", (a-b)}')
|
|
echo " Change: $delta RPS"
|
|
else
|
|
echo " Change: N/A"
|
|
fi
|
|
fi
|