System design URL shortener: Spring Boot 4.1.1, base62 ids, Redis cache-aside, Redis Lua token bucket, 24 tests and live transcripts
This commit is contained in:
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
# Exercises a RUNNING instance (java -jar ... on :8080, PostgreSQL and Redis up) and writes every transcript in output/.
|
||||
# Each section resets the per-client rate-limit bucket first, so sections do not affect each other.
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
H=http://127.0.0.1:8080
|
||||
PSQL=${PSQL:-"psql -h /tmp -p 5433 -U pgrun -d urlshort -X -q"}
|
||||
mkdir -p output
|
||||
|
||||
reset_bucket() { redis-cli --raw DEL rl:127.0.0.1 > /dev/null; }
|
||||
|
||||
# show METHOD PATH [json] -> status line + the headers that matter + body
|
||||
show() {
|
||||
local method=$1 path=$2 body=${3:-}
|
||||
echo "\$ curl -si -X $method $path${body:+ -d '$body'}"
|
||||
if [ -n "$body" ]; then
|
||||
curl -si -X "$method" -H 'Content-Type: application/json' -d "$body" "$H$path"
|
||||
else
|
||||
curl -si -X "$method" "$H$path"
|
||||
fi | tr -d '\r' | grep -E '^(HTTP/|Location:|Retry-After:|X-RateLimit-Remaining:|\{|$)' | grep -v '^$'
|
||||
echo
|
||||
}
|
||||
code_of() { sed -n 's/.*"code":"\([^"]*\)".*/\1/p'; }
|
||||
|
||||
{
|
||||
reset_bucket
|
||||
BODY='{"url":"https://ankurm.com/java-interview-preparation-guide/"}'
|
||||
echo "\$ curl -si -X POST /api/v1/urls -d '$BODY'"
|
||||
RESP=$(curl -si -X POST -H 'Content-Type: application/json' -d "$BODY" $H/api/v1/urls | tr -d '\r')
|
||||
echo "$RESP" | grep -E '^(HTTP/|Location:|X-RateLimit-Remaining:|\{)'; echo
|
||||
C=$(echo "$RESP" | code_of)
|
||||
show GET "/$C"
|
||||
} > output/01-create-and-redirect.txt 2>&1
|
||||
|
||||
{
|
||||
reset_bucket
|
||||
C=$(curl -s -X POST -H 'Content-Type: application/json' -d '{"url":"https://example.com/cache-demo"}' $H/api/v1/urls | code_of)
|
||||
stat() { curl -s $H/api/v1/debug/cache-stats | sed 's/.*"hits":\([0-9]*\).*"misses":\([0-9]*\).*/\1 \2/; s/.*"misses":\([0-9]*\).*"hits":\([0-9]*\).*/\2 \1/'; }
|
||||
read h0 m0 < <(stat)
|
||||
delta() { read h m < <(stat); echo "cache hits +$((h-h0)), misses +$((m-m0))"; }
|
||||
echo "code = $C"
|
||||
echo "\$ redis-cli EXISTS url:$C # before the first redirect"
|
||||
redis-cli EXISTS "url:$C"
|
||||
curl -s -o /dev/null "$H/$C"
|
||||
echo "--- after 1st redirect (cache miss, database read, cache filled)"
|
||||
echo "\$ redis-cli GET url:$C"; redis-cli GET "url:$C"
|
||||
echo "\$ redis-cli TTL url:$C"; redis-cli TTL "url:$C"
|
||||
delta
|
||||
for i in 1 2 3 4 5 6 7 8 9; do curl -s -o /dev/null "$H/$C"; done
|
||||
echo "--- after 9 more redirects"
|
||||
delta
|
||||
echo "\$ redis-cli GET clicks:$C"; redis-cli GET "clicks:$C"
|
||||
echo "\$ curl -s $H/api/v1/urls/$C/stats"; curl -s "$H/api/v1/urls/$C/stats" | sed 's/"createdAt".*/.../'; echo
|
||||
} > output/02-cache-aside-and-clicks.txt 2>&1
|
||||
|
||||
{
|
||||
before=$(curl -s $H/api/v1/debug/cache-stats | sed 's/.*"misses":\([0-9]*\).*/\1/')
|
||||
show GET /Zq9Zq9
|
||||
mid=$(curl -s $H/api/v1/debug/cache-stats | sed 's/.*"misses":\([0-9]*\).*/\1/')
|
||||
show GET /Zq9Zq9
|
||||
after=$(curl -s $H/api/v1/debug/cache-stats | sed 's/.*"misses":\([0-9]*\).*/\1/')
|
||||
echo "cache misses: before=$before after-first-404=$mid after-second-404=$after"
|
||||
echo "\$ redis-cli GET url:Zq9Zq9 # the negative-cache marker"; redis-cli GET url:Zq9Zq9
|
||||
echo "\$ redis-cli TTL url:Zq9Zq9"; redis-cli TTL url:Zq9Zq9
|
||||
} > output/03-negative-cache.txt 2>&1
|
||||
|
||||
{
|
||||
reset_bucket
|
||||
redis-cli --raw DEL url:demo-link > /dev/null; $PSQL -c "DELETE FROM urls WHERE code='demo-link'" > /dev/null
|
||||
show POST /api/v1/urls '{"url":"https://example.com/one","alias":"demo-link"}'
|
||||
show POST /api/v1/urls '{"url":"https://example.com/two","alias":"demo-link"}'
|
||||
show POST /api/v1/urls '{"url":"ftp://example.com/file"}'
|
||||
show POST /api/v1/urls '{"url":"https://example.com","alias":"api"}'
|
||||
} > output/04-alias-and-validation.txt 2>&1
|
||||
|
||||
{
|
||||
reset_bucket
|
||||
echo "capacity 5, refill 0.5 tokens/second (one every 2 s)"
|
||||
for i in 1 2 3 4 5 6 7; do
|
||||
echo -n "request $i -> "
|
||||
curl -si -X POST -H 'Content-Type: application/json' -d '{"url":"https://example.com/rl"}' $H/api/v1/urls \
|
||||
| tr -d '\r' | grep -E '^(HTTP/|Retry-After:|X-RateLimit-Remaining:)' | paste -sd' ' -
|
||||
done
|
||||
echo "\$ redis-cli HGETALL rl:127.0.0.1"; redis-cli HGETALL rl:127.0.0.1
|
||||
echo "--- sleeping 2.2 s"; sleep 2.2
|
||||
echo -n "request 8 -> "
|
||||
curl -si -X POST -H 'Content-Type: application/json' -d '{"url":"https://example.com/rl"}' $H/api/v1/urls \
|
||||
| tr -d '\r' | grep -E '^(HTTP/|Retry-After:|X-RateLimit-Remaining:)' | paste -sd' ' -
|
||||
echo "--- redirects are not limited: 30 GETs in a row"
|
||||
reset_bucket
|
||||
C=$(curl -s -X POST -H 'Content-Type: application/json' -d '{"url":"https://example.com/free"}' $H/api/v1/urls | code_of)
|
||||
for i in $(seq 1 30); do curl -s -o /dev/null -w '%{http_code} ' "$H/${C:-x}"; done; echo
|
||||
} > output/05-rate-limit.txt 2>&1
|
||||
|
||||
{
|
||||
reset_bucket
|
||||
C=$(curl -s -X POST -H 'Content-Type: application/json' -d '{"url":"https://example.com/temp","ttlSeconds":2}' $H/api/v1/urls | code_of)
|
||||
echo "created $C with ttlSeconds=2"
|
||||
echo -n "t=0s -> "; curl -s -o /dev/null -w '%{http_code}\n' "$H/$C"
|
||||
echo "\$ redis-cli TTL url:$C # capped at the link's remaining life, not the 1 h default"; redis-cli TTL "url:$C"
|
||||
sleep 3
|
||||
echo -n "t=3s -> "; curl -s -o /dev/null -w '%{http_code}\n' "$H/$C"
|
||||
} > output/06-expiry.txt 2>&1
|
||||
|
||||
|
||||
echo "transcripts 01-06 written to output/ (run scripts/id-generation-demo.sh for 07 and 08)"
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
# Starts its own instances (ports 8082 and 8083) against the same PostgreSQL + Redis, so it needs the packaged jar:
|
||||
# mvn -DskipTests package
|
||||
# Shows (07) block allocation across a restart and (08) the snowflake profile's longer codes.
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
JAR=target/system-design-url-shortener-1.0.0.jar
|
||||
PSQL=${PSQL:-"psql -h /tmp -p 5433 -U pgrun -d urlshort -X -q"}
|
||||
mkdir -p output
|
||||
|
||||
start() { # start PORT [extra args] ; waits until it answers
|
||||
local port=$1; shift
|
||||
java -Duser.timezone=UTC -jar $JAR --server.port=$port "$@" > /tmp/urlshort-$port.log 2>&1 &
|
||||
echo $! > /tmp/urlshort-$port.pid
|
||||
for _ in $(seq 1 60); do curl -s -o /dev/null http://127.0.0.1:$port/api/v1/debug/cache-stats && return 0; sleep 0.5; done
|
||||
echo "instance on $port did not start" >&2; return 1
|
||||
}
|
||||
stop() { kill "$(cat /tmp/urlshort-$1.pid)" 2>/dev/null; wait "$(cat /tmp/urlshort-$1.pid)" 2>/dev/null; }
|
||||
create() { # create PORT URL -> prints the code
|
||||
redis-cli --raw DEL rl:127.0.0.1 > /dev/null
|
||||
curl -s -X POST -H 'Content-Type: application/json' -d "{\"url\":\"$2\"}" http://127.0.0.1:$1/api/v1/urls | sed -n 's/.*"code":"\([^"]*\)".*/\1/p'
|
||||
}
|
||||
|
||||
{
|
||||
$PSQL -c "TRUNCATE urls" -c "ALTER SEQUENCE url_id_seq RESTART WITH 1"
|
||||
start 8082
|
||||
echo "--- instance A: three links"
|
||||
for n in 1 2 3; do create 8082 "https://example.com/a$n" > /dev/null; done
|
||||
$PSQL -c "SELECT id, code, length(code) AS len FROM urls ORDER BY id"
|
||||
stop 8082
|
||||
echo "--- instance A stopped and restarted (ids 4..100 of its first block are never used)"
|
||||
start 8082
|
||||
create 8082 "https://example.com/after-restart" > /dev/null
|
||||
$PSQL -c "SELECT id, code, length(code) AS len FROM urls ORDER BY id"
|
||||
echo "--- two instances running at once never overlap: B starts while A is alive"
|
||||
start 8083
|
||||
create 8083 "https://example.com/from-b" > /dev/null
|
||||
create 8082 "https://example.com/from-a" > /dev/null
|
||||
$PSQL -c "SELECT id, code FROM urls WHERE long_url IN ('https://example.com/from-b','https://example.com/from-a') ORDER BY id"
|
||||
stop 8083; stop 8082
|
||||
} > output/07-id-generation.txt 2>&1
|
||||
|
||||
{
|
||||
start 8083 --spring.profiles.active=snowflake --app.node-id=7
|
||||
for n in 1 2 3; do c=$(create 8083 "https://example.com/snow$n"); echo "code=$c length=${#c}"; done
|
||||
$PSQL -c "SELECT id, code FROM urls WHERE long_url LIKE 'https://example.com/snow%' ORDER BY id"
|
||||
stop 8083
|
||||
} > output/08-snowflake.txt 2>&1
|
||||
echo "transcripts 07 and 08 written to output/"
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# Starts the app on :8080 against PostgreSQL (localhost:5433, db urlshort) and Redis (localhost:6379).
|
||||
# -Duser.timezone=UTC matters: the PostgreSQL JDBC driver sends the JVM's zone to the server, and a legacy
|
||||
# zone name such as Asia/Calcutta can be rejected ("invalid value for parameter TimeZone").
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
mvn -q -DskipTests package
|
||||
exec java -Duser.timezone=UTC -jar target/system-design-url-shortener-1.0.0.jar "$@"
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the tests and writes one line per test (from the Surefire XML) to output/09-test-results.txt.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
mvn -B -q test > /dev/null
|
||||
python3 - <<'PY' > output/09-test-results.txt
|
||||
import glob, xml.etree.ElementTree as ET
|
||||
total = fail = 0
|
||||
for f in sorted(glob.glob("target/surefire-reports/TEST-*.xml")):
|
||||
root = ET.parse(f).getroot()
|
||||
cls = root.get("name").rsplit(".", 1)[-1]
|
||||
for tc in root.iter("testcase"):
|
||||
bad = tc.find("failure") is not None or tc.find("error") is not None
|
||||
total += 1; fail += bad
|
||||
print(("FAIL " if bad else "PASS ") + cls + "." + tc.get("name"))
|
||||
print(f"{total} tests, {fail} failures")
|
||||
PY
|
||||
Reference in New Issue
Block a user