Ankur Mhatre
e1f8aa7402
Kafka, RabbitMQ and Pulsar measured side by side on ordering, replay and consumer scaling by driving the three client libraries directly, plus an operational-footprint measurement of each broker's own distribution. Nine tests, three brokers started without Docker, and every number in the documentation regenerated by scripts/run-all.sh.
74 lines
3.5 KiB
Bash
Executable File
74 lines
3.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Measures the operational footprint of one broker: how long it takes to accept a connection,
|
|
# how much memory it holds at idle, how many processes and listening ports it opens, and how
|
|
# large its shipped default configuration is.
|
|
#
|
|
# These are proxies, not a benchmark. "Ops burden" is not directly measurable, but the number of
|
|
# moving parts, the memory floor and the size of the configuration surface are, and they are what
|
|
# people are actually asking about when they ask which broker is heavier to run.
|
|
#
|
|
# scripts/footprint.sh kafka|rabbit|pulsar
|
|
set -euo pipefail
|
|
BROKER="${1:?kafka|rabbit|pulsar}"
|
|
|
|
extra=""
|
|
port_open() { (echo > "/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
|
|
rss_mb() { ps -eo rss,cmd | grep -F "$1" | grep -v grep | awk '{s+=$1} END {printf "%.0f", s/1024}'; }
|
|
procs() { ps -eo cmd | grep -F "$1" | grep -cv grep; }
|
|
settings() { grep -cE '^[a-zA-Z]' "$1" 2>/dev/null || echo 0; }
|
|
|
|
start=$(date +%s%3N)
|
|
case "$BROKER" in
|
|
kafka)
|
|
: "${KAFKA_HOME:?set KAFKA_HOME}"
|
|
# Wipe whatever log.dirs points at, so the format below is not refused with
|
|
# "Invalid cluster.id ... Expected X, but read Y" from a previous run.
|
|
KAFKA_LOG_DIRS=$(grep -E '^log.dirs=' "$KAFKA_HOME/config/server.properties" | cut -d= -f2)
|
|
rm -rf ${KAFKA_LOG_DIRS//,/ }
|
|
"$KAFKA_HOME/bin/kafka-storage.sh" format --standalone -t "$("$KAFKA_HOME/bin/kafka-storage.sh" random-uuid)" \
|
|
-c "$KAFKA_HOME/config/server.properties" --ignore-formatted >/dev/null
|
|
start=$(date +%s%3N) # after formatting: kafka-storage.sh is two more JVM starts and is a
|
|
# one-off, not part of what a restart costs
|
|
setsid nohup "$KAFKA_HOME/bin/kafka-server-start.sh" "$KAFKA_HOME/config/server.properties" \
|
|
> /tmp/kafka-start.log 2>&1 < /dev/null &
|
|
for _ in $(seq 1 90); do port_open 9092 && break; sleep 1; done
|
|
ready=$(( $(date +%s%3N) - start ))
|
|
marker="kafka.Kafka"; ports="9092 (broker), 9093 (controller)"; conf="$KAFKA_HOME/config/server.properties"
|
|
dist="$KAFKA_HOME"
|
|
;;
|
|
rabbit)
|
|
: "${RABBITMQ_HOME:?set RABBITMQ_HOME}"; : "${ERL_ROOT:?set ERL_ROOT}"
|
|
scripts/rabbit-broker.sh >/dev/null
|
|
ready=$(( $(date +%s%3N) - start ))
|
|
# The broker is an Erlang VM, so the process to measure is beam.smp, not anything named
|
|
# "rabbitmq". epmd is a second, tiny process and is counted separately below.
|
|
marker="beam.smp"; ports="5672 (AMQP), 4369 (epmd), 25672 (inter-node)"
|
|
conf="$RABBITMQ_HOME/etc/rabbitmq/rabbitmq.conf.example"; dist="$RABBITMQ_HOME"
|
|
extra="epmd processes : $(procs epmd)
|
|
NOTE: RabbitMQ ships no configuration file at all. rabbitmq.conf.example is
|
|
entirely commented out and the boot log records \"Config file(s): (none)\".
|
|
Everything in that file is documentation, not a default."
|
|
;;
|
|
pulsar)
|
|
: "${PULSAR_HOME:?set PULSAR_HOME}"
|
|
scripts/pulsar-broker.sh >/dev/null
|
|
ready=$(( $(date +%s%3N) - start ))
|
|
marker="PulsarStandaloneStarter"; ports="6650 (binary), 8080 (admin/REST), 2181 (ZooKeeper)"
|
|
conf="$PULSAR_HOME/conf/standalone.conf"; dist="$PULSAR_HOME"
|
|
;;
|
|
*) echo "unknown broker $BROKER" >&2; exit 2 ;;
|
|
esac
|
|
|
|
cat <<EOF
|
|
broker : $BROKER
|
|
time from launch to first
|
|
accepted connection : ${ready} ms
|
|
resident memory at idle : $(rss_mb "$marker") MB
|
|
server processes : $(procs "$marker")
|
|
listening ports : $ports
|
|
unpacked distribution size : $(du -sm "$dist" | cut -f1) MB
|
|
settings in shipped default
|
|
configuration file : $(settings "$conf") ($(basename "$conf"))
|
|
${extra:-}
|
|
EOF
|