Skip to content

Commit 1271253

Browse files
jyeshejyeshe
committed
Adds a script for local dev clusters
1 parent 83be24a commit 1271253

4 files changed

Lines changed: 254 additions & 49 deletions

File tree

‎bin/local_cluster‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
#!/usr/bin/env bash
2+
3+
COOKIE="rate-limiter-secret"
4+
5+
# Function to show usage
6+
show_usage() {
7+
echo "Usage:"
8+
echo " $0 [--proxy] --count <number> Start local cluster (default: 2 instances)"
9+
echo " $0 connect <node_number> Connect to a specific node (1-4)"
10+
echo ""
11+
echo "Options:"
12+
echo " --proxy Start a Caddy reverse proxy on port 4000 (nodes will start from 4001)"
13+
echo " --count <num> Number of nodes to start (1-4, default: 2)"
14+
exit 1
15+
}
16+
17+
# Handle connect subcommand
18+
if [ "$1" = "connect" ]; then
19+
if [ -z "$2" ] || ! [[ "$2" =~ ^[1-4]$ ]]; then
20+
echo "Error: Please specify a valid node number (1-4)"
21+
show_usage
22+
fi
23+
24+
NODE_NUM=$2
25+
echo "Connecting to node${NODE_NUM}@127.0.0.1..."
26+
exec iex --name "remote_shell${NODE_NUM}@127.0.0.1" --cookie "${COOKIE}" --remsh "node${NODE_NUM}@127.0.0.1"
27+
# The exec command replaces the current process, so we don't need an explicit exit
28+
# If we reach this point, it means the exec failed, so we'll exit with its status code
29+
exit $?
30+
fi
31+
32+
# Parse arguments
33+
USE_PROXY=false
34+
INSTANCES=2
35+
36+
while [[ $# -gt 0 ]]; do
37+
case $1 in
38+
--proxy)
39+
USE_PROXY=true
40+
shift
41+
;;
42+
--count)
43+
if [ -z "$2" ] || ! [[ "$2" =~ ^[0-9]+$ ]]; then
44+
echo "Error: --count requires a numeric argument"
45+
show_usage
46+
fi
47+
INSTANCES=$2
48+
shift 2
49+
;;
50+
*)
51+
echo "Unknown argument: $1"
52+
show_usage
53+
;;
54+
esac
55+
done
56+
57+
# Validate number of instances
58+
if ! [[ "$INSTANCES" =~ ^[0-9]+$ ]]; then
59+
echo "Error: Number of instances must be a positive integer"
60+
show_usage
61+
fi
62+
63+
if [ "$INSTANCES" -lt 1 ] || [ "$INSTANCES" -gt 4 ]; then
64+
echo "Error: Number of instances must be between 1 and 4"
65+
show_usage
66+
fi
67+
68+
# Check for Caddy if proxy is requested
69+
if [ "$USE_PROXY" = true ]; then
70+
if ! command -v caddy &>/dev/null; then
71+
echo "Error: Caddy is required for proxy mode but it's not installed"
72+
echo "Please install Caddy first:"
73+
echo " Mac: brew install caddy"
74+
echo " Linux: sudo apt install caddy"
75+
echo " Or visit: https://caddyserver.com/docs/install"
76+
exit 1
77+
fi
78+
fi
79+
80+
# Array to store background PIDs
81+
declare -a PIDS
82+
83+
# Colors for different processes
84+
declare -a COLORS=(
85+
"\033[0;36m" # Cyan
86+
"\033[0;32m" # Green
87+
"\033[0;35m" # Purple
88+
"\033[0;33m" # Yellow
89+
"\033[0;37m" # Gray (for proxy)
90+
)
91+
RESET="\033[0m"
92+
93+
# Cleanup function to kill all child processes
94+
cleanup() {
95+
echo "Shutting down all processes..."
96+
for pid in "${PIDS[@]}"; do
97+
kill "$pid" 2>/dev/null
98+
done
99+
exit 0
100+
}
101+
102+
# Set up trap for cleanup
103+
trap cleanup INT TERM
104+
105+
# Function to run a command with colored output
106+
run_with_color() {
107+
local color=$1
108+
local prefix=$2
109+
shift 2
110+
# Run the command and color its output
111+
"$@" 2>&1 | while read -r line; do
112+
echo -e "${color}${prefix} | ${line}${RESET}"
113+
done
114+
}
115+
116+
# Create Caddy configuration if proxy is enabled
117+
if [ "$USE_PROXY" = true ]; then
118+
BASE_PORT=4001
119+
CADDY_CONFIG=$(mktemp)
120+
echo "Creating Caddy configuration..."
121+
cat >"$CADDY_CONFIG" <<EOF
122+
# Global options
123+
{
124+
admin off
125+
auto_https off
126+
http_port 4000
127+
}
128+
129+
# Reverse proxy configuration
130+
localhost:4000 {
131+
reverse_proxy {
132+
to $(for i in $(seq 1 "$INSTANCES"); do echo "localhost:$((BASE_PORT + i - 1))"; done | paste -sd " " -)
133+
lb_policy round_robin
134+
}
135+
}
136+
EOF
137+
138+
# Only log Caddy config if LOG_LEVEL is debug
139+
if [ "${LOG_LEVEL:-}" = "debug" ]; then
140+
echo "Caddy config:"
141+
cat "$CADDY_CONFIG"
142+
fi
143+
144+
# Start Caddy
145+
run_with_color "${COLORS[4]}" "proxy" caddy run --adapter caddyfile --config "$CADDY_CONFIG" &
146+
PIDS+=($!)
147+
148+
# Cleanup Caddy config on exit
149+
trap 'rm -f "$CADDY_CONFIG"' EXIT
150+
151+
echo "Started reverse proxy on port 4000"
152+
else
153+
BASE_PORT=4000
154+
fi
155+
156+
# Start the requested number of instances
157+
for i in $(seq 1 "$INSTANCES"); do
158+
export RTM_PORT=$((2222 + i - 1)) PORT=$((BASE_PORT + i - 1))
159+
run_with_color "${COLORS[$i - 1]}" "node$i" elixir --cookie "${COOKIE}" --name "node$i@127.0.0.1" -S mix phx.server &
160+
PIDS+=($!)
161+
done
162+
163+
if [ "$USE_PROXY" = true ]; then
164+
echo "Started $INSTANCES node(s) on ports $((BASE_PORT))-$((BASE_PORT + INSTANCES - 1)) with load balancer on port 4000"
165+
echo "RTM ports: 2222-$((2222 + INSTANCES - 1))"
166+
else
167+
echo "Started $INSTANCES node(s) on ports $((BASE_PORT))-$((BASE_PORT + INSTANCES - 1))"
168+
echo "RTM ports: 2222-$((2222 + INSTANCES - 1))"
169+
fi
170+
echo "To connect to a specific node, use: $0 connect <node_number>"
171+
172+
# Wait for all background processes
173+
wait

‎config/dev.exs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ config :lightning, Lightning.Vault,
4848

4949
config :lightning, Lightning.Runtime.RuntimeManager, start: true
5050

51+
config :lightning, Lightning.DistributedRateLimiter, start: true
52+
5153
config :lightning, :workers,
5254
private_key: """
5355
-----BEGIN PRIVATE KEY-----

‎lib/lightning/config/bootstrap.ex‎

Lines changed: 73 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,8 @@ defmodule Lightning.Config.Bootstrap do
114114
"RTM",
115115
&Utils.ensure_boolean/1,
116116
Utils.get_env([:lightning, Lightning.Runtime.RuntimeManager, :start])
117-
)
117+
),
118+
port: env!("RTM_PORT", :integer, 2222)
118119

119120
config :lightning, :workers,
120121
private_key:
@@ -408,6 +409,10 @@ defmodule Lightning.Config.Bootstrap do
408409
config :logger, :level, log_level
409410
end
410411

412+
if log_level == :debug do
413+
config :libcluster, debug: true
414+
end
415+
411416
database_url = env!("DATABASE_URL", :string, nil)
412417

413418
config :lightning, Lightning.Repo,
@@ -418,7 +423,6 @@ defmodule Lightning.Config.Bootstrap do
418423
queue_interval: env!("DATABASE_QUEUE_INTERVAL", :integer, 1000)
419424

420425
host = env!("URL_HOST", :string, "example.com")
421-
port = env!("PORT", :integer, 4000)
422426
url_port = env!("URL_PORT", :integer, 443)
423427

424428
config :lightning,
@@ -464,18 +468,6 @@ defmodule Lightning.Config.Bootstrap do
464468
You can generate one by calling: mix phx.gen.secret
465469
"""
466470

467-
listen_address =
468-
env!(
469-
"LISTEN_ADDRESS",
470-
fn address ->
471-
address
472-
|> String.split(".")
473-
|> Enum.map(&String.to_integer/1)
474-
|> List.to_tuple()
475-
end,
476-
{127, 0, 0, 1}
477-
)
478-
479471
origins =
480472
env!(
481473
"ORIGINS",
@@ -490,40 +482,10 @@ defmodule Lightning.Config.Bootstrap do
490482

491483
url_scheme = env!("URL_SCHEME", :string, "https")
492484

493-
idle_timeout =
494-
env!(
495-
"IDLE_TIMEOUT",
496-
fn str ->
497-
case Integer.parse(str) do
498-
:error -> 60_000
499-
{val, _} -> val * 1_000
500-
end
501-
end,
502-
60_000
503-
)
504-
505485
config :lightning, LightningWeb.Endpoint,
506486
url: [host: host, port: url_port, scheme: url_scheme],
507487
secret_key_base: secret_key_base,
508488
check_origin: origins,
509-
http: [
510-
ip: listen_address,
511-
port: port,
512-
compress: true,
513-
protocol_options: [
514-
# Note that if a request is more than 10x the max dataclip size, we cut
515-
# the connection immediately to prevent memory issues via the
516-
# :max_skip_body_length setting.
517-
max_skip_body_length:
518-
Application.get_env(
519-
:lightning,
520-
:max_dataclip_size_bytes,
521-
10_000_000
522-
) *
523-
10,
524-
idle_timeout: idle_timeout
525-
]
526-
],
527489
server: true
528490
end
529491

@@ -539,6 +501,8 @@ defmodule Lightning.Config.Bootstrap do
539501
assert_receive_timeout: env!("ASSERT_RECEIVE_TIMEOUT", :integer, 1000)
540502
end
541503

504+
config :lightning, LightningWeb.Endpoint, http: http_config(config_env())
505+
542506
config :sentry,
543507
dsn: env!("SENTRY_DSN", :string, nil),
544508
filter: Lightning.SentryEventFilter,
@@ -814,4 +778,69 @@ defmodule Lightning.Config.Bootstrap do
814778
value -> value
815779
end
816780
end
781+
782+
defp http_config(env, opts \\ [])
783+
# Production environment configuration
784+
defp http_config(:prod, opts) do
785+
port = Keyword.get(opts, :port) || env!("PORT", :integer, 4000)
786+
787+
listen_address =
788+
env!(
789+
"LISTEN_ADDRESS",
790+
fn address ->
791+
address
792+
|> String.split(".")
793+
|> Enum.map(&String.to_integer/1)
794+
|> List.to_tuple()
795+
end,
796+
{127, 0, 0, 1}
797+
)
798+
799+
idle_timeout =
800+
env!(
801+
"IDLE_TIMEOUT",
802+
fn str ->
803+
case Integer.parse(str) do
804+
:error -> 60_000
805+
{val, _} -> val * 1_000
806+
end
807+
end,
808+
60_000
809+
)
810+
811+
[
812+
ip: listen_address,
813+
port: port,
814+
compress: true,
815+
protocol_options: [
816+
# Note that if a request is more than 10x the max dataclip size, we cut
817+
# the connection immediately to prevent memory issues via the
818+
# :max_skip_body_length setting.
819+
max_skip_body_length:
820+
Application.get_env(
821+
:lightning,
822+
:max_dataclip_size_bytes,
823+
10_000_000
824+
) * 10,
825+
idle_timeout: idle_timeout
826+
]
827+
]
828+
end
829+
830+
# Default configuration for non-production environments
831+
defp http_config(_env, opts) do
832+
port =
833+
Keyword.get(opts, :port) ||
834+
env!(
835+
"PORT",
836+
:integer,
837+
get_env(:lightning, [LightningWeb.Endpoint, :http, :port])
838+
)
839+
840+
[
841+
ip: {0, 0, 0, 0},
842+
port: port,
843+
compress: true
844+
]
845+
end
817846
end

‎test/lightning/distributed_rate_limiter_test.exs‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,12 @@ defmodule Lightning.DistributedRateLimiterTest do
5858
assert 500 < wait_ms and wait_ms <= 1_000
5959
end
6060

61-
# For testing the replication use manual procedure or use this test case isolated to avoid interfering the tests above:
62-
# 0. Disable Endpoint server
63-
# 1. Run node1 on one terminal: iex --sname node1@localhost --cookie hordecookie -S mix phx.server
64-
# 2. Run node2 on another terminal: iex --sname node2@localhost --cookie hordecookie -S mix phx.server
65-
# 3. Call Lightning.DistributedRateLimiter.inspect_table() on both iex and they show the same ets table process and node.
61+
# Run this distributed integration test case separately to avoid interfering with the tests above.
62+
# For testing the replication use `bin/local_cluster` on the shell:
63+
# 1. In one shell run `./bin/local_cluster`
64+
# 2. On another shell run `./bin/local_cluster connect 2`
65+
# 3. Type `Lightning.DistributedRateLimiter.inspect_table()` to see that the ETS table is distributed
66+
# (on node1 or vice-versa if it was spawned on node2 when you connect to node 1).
6667
@tag :dist_integration
6768
test "works on top of a single worker of a distributed dynamic supervisor" do
6869
{:ok, peer, _node1, node2} = start_nodes(:node1, :node2, ~c"localhost")

0 commit comments

Comments
 (0)