Elixir cluster!
This commit is contained in:
parent
ce5f598331
commit
b6d4f7044f
2
.gitignore
vendored
2
.gitignore
vendored
@ -27,3 +27,5 @@ server-*.tar
|
|||||||
|
|
||||||
.env
|
.env
|
||||||
.env.prod
|
.env.prod
|
||||||
|
|
||||||
|
.hosts.erlang
|
||||||
|
@ -62,6 +62,10 @@ ENV LC_ALL en_US.UTF-8
|
|||||||
WORKDIR "/app"
|
WORKDIR "/app"
|
||||||
RUN chown nobody /app
|
RUN chown nobody /app
|
||||||
|
|
||||||
|
# set OTP_ROOT so elixir can find nodes
|
||||||
|
ENV OTP_ROOT=/app
|
||||||
|
COPY .hosts.erlang /app/.hosts.erlang
|
||||||
|
|
||||||
# set runner ENV
|
# set runner ENV
|
||||||
ENV MIX_ENV="prod"
|
ENV MIX_ENV="prod"
|
||||||
|
|
||||||
|
1
buildscripts/build/.gitignore
vendored
1
buildscripts/build/.gitignore
vendored
@ -1,2 +1,3 @@
|
|||||||
.env
|
.env
|
||||||
keys/
|
keys/
|
||||||
|
.hosts.erlang
|
||||||
|
@ -10,8 +10,9 @@ server_node_ids=(4 5 6)
|
|||||||
build_dir="${HOME}/src/chessh/buildscripts/build"
|
build_dir="${HOME}/src/chessh/buildscripts/build"
|
||||||
|
|
||||||
server_name="chessh.linux.usu.edu"
|
server_name="chessh.linux.usu.edu"
|
||||||
|
erlang_hosts_file="${build_dir}/.hosts.erlang"
|
||||||
load_balancer_nginx_site_file="/etc/nginx/sites-enabled/${server_name}.conf"
|
load_balancer_nginx_site_file="/etc/nginx/sites-enabled/${server_name}.conf"
|
||||||
ha_proxy_cfg="/etc/haproxy/haproxy.cfg"
|
ha_proxy_cfg_file="/etc/haproxy/haproxy.cfg"
|
||||||
ssl_cert_path="/etc/letsencrypt/live/${server_name}"
|
ssl_cert_path="/etc/letsencrypt/live/${server_name}"
|
||||||
certbot_webroot_path="/var/www/html/${server_name}"
|
certbot_webroot_path="/var/www/html/${server_name}"
|
||||||
load_balancer_nginx_site="
|
load_balancer_nginx_site="
|
||||||
@ -63,34 +64,29 @@ server {
|
|||||||
return 404;
|
return 404;
|
||||||
}
|
}
|
||||||
"
|
"
|
||||||
# TODO: Servers should pull server ids from array
|
|
||||||
ha_proxy_cfg="
|
|
||||||
global\n
|
|
||||||
log /dev/log local0\n
|
|
||||||
log /dev/log local1 notice\n
|
|
||||||
maxconn 2500\n
|
|
||||||
user haproxy\n
|
|
||||||
group haproxy\n
|
|
||||||
daemon\n
|
|
||||||
\n
|
|
||||||
defaults\n
|
|
||||||
log global\n
|
|
||||||
mode tcp\n
|
|
||||||
timeout connect 10s\n
|
|
||||||
timeout client 36h\n
|
|
||||||
timeout server 36h\n
|
|
||||||
option dontlognull\n
|
|
||||||
\n
|
|
||||||
listen ssh\n
|
|
||||||
bind *:${ssh_port}\n
|
|
||||||
balance leastconn\n
|
|
||||||
mode tcp\n
|
|
||||||
|
|
||||||
server hostA 192.168.100.4:${ssh_port} check inter 10s fall 2 rise 1\n
|
ha_proxy_cfg="
|
||||||
server hostB 192.168.100.5:${ssh_port} check inter 10s fall 2 rise 1\n
|
global
|
||||||
server hostC 192.168.100.6:${ssh_port} check inter 10s fall 2 rise 1\n
|
log /dev/log local0
|
||||||
|
log /dev/log local1 notice
|
||||||
|
maxconn 2500
|
||||||
|
user haproxy
|
||||||
|
group haproxy
|
||||||
|
daemon
|
||||||
|
defaults
|
||||||
|
log global
|
||||||
|
mode tcp
|
||||||
|
timeout connect 10s
|
||||||
|
timeout client 36h
|
||||||
|
timeout server 36h
|
||||||
|
option dontlognull
|
||||||
|
listen ssh
|
||||||
|
bind 0.0.0.0:${ssh_port}
|
||||||
|
balance leastconn
|
||||||
|
mode tcp
|
||||||
|
|
||||||
|
$(echo "${server_node_ids[@]}" | python3 -c "print(\"\\n\".join([f\" server pi{i} 192.168.100.{i}:${ssh_port} check inter 10s fall rise 1 \" for i in input().split()]))")
|
||||||
"
|
"
|
||||||
echo $ha_proxy_cfg
|
|
||||||
|
|
||||||
ssh_opts="-oStrictHostKeyChecking=no"
|
ssh_opts="-oStrictHostKeyChecking=no"
|
||||||
|
|
||||||
@ -116,9 +112,8 @@ function copy_ssh_keys() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function reload_loadbalancer_conf() {
|
||||||
function reload_nginx_confs() {
|
dead_files=("/etc/nginx/sites-enabled/default" "/etc/nginx/nginx.conf" "$load_balancer_nginx_site_file" "$ha_proxy_cfg_file")
|
||||||
dead_files=("/etc/nginx/sites-enabled/default" "/etc/nginx/nginx.conf" "$load_balancer_nginx_site_file")
|
|
||||||
for file in "${dead_files[@]}"
|
for file in "${dead_files[@]}"
|
||||||
do
|
do
|
||||||
[ -e $file ] && sudo rm $file
|
[ -e $file ] && sudo rm $file
|
||||||
@ -128,6 +123,10 @@ function reload_nginx_confs() {
|
|||||||
echo $load_balancer_nginx_site | sudo tee $load_balancer_nginx_site_file
|
echo $load_balancer_nginx_site | sudo tee $load_balancer_nginx_site_file
|
||||||
|
|
||||||
sudo systemctl restart nginx
|
sudo systemctl restart nginx
|
||||||
|
|
||||||
|
printf '$ha_proxy_cfg' | sudo tee $ha_proxy_cfg_file
|
||||||
|
|
||||||
|
sudo systemctl restart haproxy
|
||||||
}
|
}
|
||||||
|
|
||||||
function build_frontend() {
|
function build_frontend() {
|
||||||
@ -152,7 +151,7 @@ function build_server() {
|
|||||||
temp_file=$(mktemp)
|
temp_file=$(mktemp)
|
||||||
|
|
||||||
cp "${build_dir}/.env" $temp_file
|
cp "${build_dir}/.env" $temp_file
|
||||||
printf "\nNODE_ID=$node_id\n" >> $temp_file
|
printf "\nNODE_ID=$node_conn\nRELEASE_NODE=chessh@192.168.100.${node_id}\n" >> $temp_file
|
||||||
scp $ssh_opts $temp_file $node_conn:~/.env
|
scp $ssh_opts $temp_file $node_conn:~/.env
|
||||||
|
|
||||||
cp "${build_dir}/chessh.service" $temp_file
|
cp "${build_dir}/chessh.service" $temp_file
|
||||||
@ -161,11 +160,14 @@ function build_server() {
|
|||||||
|
|
||||||
scp $ssh_opts "${build_dir}/build_server.sh" $node_conn:~/
|
scp $ssh_opts "${build_dir}/build_server.sh" $node_conn:~/
|
||||||
|
|
||||||
|
scp $ssh_opts $erlang_hosts_file $node_conn:~/
|
||||||
|
|
||||||
ssh $ssh_opts $node_conn "~/build_server.sh"
|
ssh $ssh_opts $node_conn "~/build_server.sh"
|
||||||
}
|
}
|
||||||
|
|
||||||
function build_server_nodes() {
|
function build_server_nodes() {
|
||||||
copy_ssh_keys
|
copy_ssh_keys
|
||||||
|
"$(printf "'192.168.100.%s'\n" ${server_node_ids[@]})" > $erlang_hosts_file
|
||||||
|
|
||||||
for node_id in "${server_node_ids[@]}"
|
for node_id in "${server_node_ids[@]}"
|
||||||
do
|
do
|
||||||
@ -173,6 +175,6 @@ function build_server_nodes() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
reload_nginx_confs
|
reload_loadbalancer_conf
|
||||||
build_server_nodes
|
build_server_nodes
|
||||||
build_frontend_nodes
|
build_frontend_nodes
|
||||||
|
@ -30,6 +30,7 @@ fi
|
|||||||
# Build
|
# Build
|
||||||
cd $chessh_path
|
cd $chessh_path
|
||||||
[ -d "$chessh_path/priv/keys" ] && cp ~/keys/* "$chessh_path/priv/keys/" || cp -r ~/keys "$chessh_path/priv"
|
[ -d "$chessh_path/priv/keys" ] && cp ~/keys/* "$chessh_path/priv/keys/" || cp -r ~/keys "$chessh_path/priv"
|
||||||
|
mv $HOME/.hosts.erlang .
|
||||||
sudo docker build . -t chessh/server
|
sudo docker build . -t chessh/server
|
||||||
|
|
||||||
# Systemd service
|
# Systemd service
|
||||||
|
@ -23,4 +23,12 @@ config :chessh, Web,
|
|||||||
|
|
||||||
config :joken, default_signer: "secret"
|
config :joken, default_signer: "secret"
|
||||||
|
|
||||||
|
config :libcluster,
|
||||||
|
topologies: [
|
||||||
|
erlang_hosts_example: [
|
||||||
|
strategy: Elixir.Cluster.Strategy.ErlangHosts,
|
||||||
|
config: [timeout: 30_000]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
|
||||||
import_config "#{config_env()}.exs"
|
import_config "#{config_env()}.exs"
|
||||||
|
@ -10,3 +10,11 @@ config :chessh, RateLimits,
|
|||||||
max_concurrent_user_sessions: 5,
|
max_concurrent_user_sessions: 5,
|
||||||
player_session_message_burst_ms: 750,
|
player_session_message_burst_ms: 750,
|
||||||
player_session_message_burst_rate: 8
|
player_session_message_burst_rate: 8
|
||||||
|
|
||||||
|
config :libcluster,
|
||||||
|
topologies: [
|
||||||
|
erlang_hosts_example: [
|
||||||
|
strategy: Elixir.Cluster.Strategy.ErlangHosts,
|
||||||
|
config: [timeout: 30_000]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
@ -22,7 +22,9 @@ defmodule Chessh.Application do
|
|||||||
scheme: :http,
|
scheme: :http,
|
||||||
plug: Chessh.Web.Endpoint,
|
plug: Chessh.Web.Endpoint,
|
||||||
options: [port: Application.get_env(:chessh, Web)[:port]]
|
options: [port: Application.get_env(:chessh, Web)[:port]]
|
||||||
)
|
),
|
||||||
|
{Cluster.Supervisor,
|
||||||
|
[Application.get_env(:libcluster, :topologies), [name: Chessh.ClusterSupervisor]]}
|
||||||
]
|
]
|
||||||
|
|
||||||
opts = [strategy: :one_for_one, name: Chessh.Supervisor]
|
opts = [strategy: :one_for_one, name: Chessh.Supervisor]
|
||||||
|
3
mix.exs
3
mix.exs
@ -37,7 +37,8 @@ defmodule Chessh.MixProject do
|
|||||||
{:syn, "~> 3.3"},
|
{:syn, "~> 3.3"},
|
||||||
{:jason, "~> 1.3"},
|
{:jason, "~> 1.3"},
|
||||||
{:plug_cowboy, "~> 2.2"},
|
{:plug_cowboy, "~> 2.2"},
|
||||||
{:joken, "~> 2.5"}
|
{:joken, "~> 2.5"},
|
||||||
|
{:libcluster, "~> 3.3"}
|
||||||
]
|
]
|
||||||
end
|
end
|
||||||
|
|
||||||
|
1
mix.lock
1
mix.lock
@ -19,6 +19,7 @@
|
|||||||
"jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
|
"jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
|
||||||
"joken": {:hex, :joken, "2.5.0", "09be497d804b8115eb6f07615cef2e60c2a1008fb89dc0aef0d4c4b4609b99aa", [:mix], [{:jose, "~> 1.11.2", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "22b25c89617c5ed8ca7b31026340a25ea0f9ca7160f9706b79be9ed81fdf74e7"},
|
"joken": {:hex, :joken, "2.5.0", "09be497d804b8115eb6f07615cef2e60c2a1008fb89dc0aef0d4c4b4609b99aa", [:mix], [{:jose, "~> 1.11.2", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "22b25c89617c5ed8ca7b31026340a25ea0f9ca7160f9706b79be9ed81fdf74e7"},
|
||||||
"jose": {:hex, :jose, "1.11.5", "3bc2d75ffa5e2c941ca93e5696b54978323191988eb8d225c2e663ddfefd515e", [:mix, :rebar3], [], "hexpm", "dcd3b215bafe02ea7c5b23dafd3eb8062a5cd8f2d904fd9caa323d37034ab384"},
|
"jose": {:hex, :jose, "1.11.5", "3bc2d75ffa5e2c941ca93e5696b54978323191988eb8d225c2e663ddfefd515e", [:mix, :rebar3], [], "hexpm", "dcd3b215bafe02ea7c5b23dafd3eb8062a5cd8f2d904fd9caa323d37034ab384"},
|
||||||
|
"libcluster": {:hex, :libcluster, "3.3.2", "84c6ebfdc72a03805955abfb5ff573f71921a3e299279cc3445445d5af619ad1", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8b691ce8185670fc8f3fc0b7ed59eff66c6889df890d13411f8f1a0e6871d8a5"},
|
||||||
"mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"},
|
"mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"},
|
||||||
"plug": {:hex, :plug, "1.14.0", "ba4f558468f69cbd9f6b356d25443d0b796fbdc887e03fa89001384a9cac638f", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bf020432c7d4feb7b3af16a0c2701455cbbbb95e5b6866132cb09eb0c29adc14"},
|
"plug": {:hex, :plug, "1.14.0", "ba4f558468f69cbd9f6b356d25443d0b796fbdc887e03fa89001384a9cac638f", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bf020432c7d4feb7b3af16a0c2701455cbbbb95e5b6866132cb09eb0c29adc14"},
|
||||||
"plug_cowboy": {:hex, :plug_cowboy, "2.6.0", "d1cf12ff96a1ca4f52207c5271a6c351a4733f413803488d75b70ccf44aebec2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "073cf20b753ce6682ed72905cd62a2d4bd9bad1bf9f7feb02a1b8e525bd94fa6"},
|
"plug_cowboy": {:hex, :plug_cowboy, "2.6.0", "d1cf12ff96a1ca4f52207c5271a6c351a4733f413803488d75b70ccf44aebec2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "073cf20b753ce6682ed72905cd62a2d4bd9bad1bf9f7feb02a1b8e525bd94fa6"},
|
||||||
|
Loading…
Reference in New Issue
Block a user