From b6d4f7044fa9bec32489d0ca5c3392e76ad2c1bc Mon Sep 17 00:00:00 2001 From: Logan Hunt Date: Tue, 31 Jan 2023 19:05:01 -0700 Subject: [PATCH] Elixir cluster! --- .gitignore | 2 + Dockerfile | 4 ++ buildscripts/build/.gitignore | 1 + buildscripts/build/build.sh | 66 +++++++++++++++--------------- buildscripts/build/build_server.sh | 1 + config/config.exs | 8 ++++ config/prod.exs | 8 ++++ lib/chessh/application.ex | 4 +- mix.exs | 3 +- mix.lock | 1 + 10 files changed, 64 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 233f5d2..6d0459e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ server-*.tar .env .env.prod + +.hosts.erlang diff --git a/Dockerfile b/Dockerfile index 3d85542..acd4a26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,6 +62,10 @@ ENV LC_ALL en_US.UTF-8 WORKDIR "/app" RUN chown nobody /app +# set OTP_ROOT so elixir can find nodes +ENV OTP_ROOT=/app +COPY .hosts.erlang /app/.hosts.erlang + # set runner ENV ENV MIX_ENV="prod" diff --git a/buildscripts/build/.gitignore b/buildscripts/build/.gitignore index 0b5e2d0..de95680 100644 --- a/buildscripts/build/.gitignore +++ b/buildscripts/build/.gitignore @@ -1,2 +1,3 @@ .env keys/ +.hosts.erlang diff --git a/buildscripts/build/build.sh b/buildscripts/build/build.sh index c57235b..d0b59c4 100755 --- a/buildscripts/build/build.sh +++ b/buildscripts/build/build.sh @@ -10,8 +10,9 @@ server_node_ids=(4 5 6) build_dir="${HOME}/src/chessh/buildscripts/build" server_name="chessh.linux.usu.edu" +erlang_hosts_file="${build_dir}/.hosts.erlang" load_balancer_nginx_site_file="/etc/nginx/sites-enabled/${server_name}.conf" -ha_proxy_cfg="/etc/haproxy/haproxy.cfg" +ha_proxy_cfg_file="/etc/haproxy/haproxy.cfg" ssl_cert_path="/etc/letsencrypt/live/${server_name}" certbot_webroot_path="/var/www/html/${server_name}" load_balancer_nginx_site=" @@ -63,34 +64,29 @@ server { return 404; } " -# TODO: Servers should pull server ids from array -ha_proxy_cfg=" -global\n - log /dev/log local0\n - log /dev/log local1 notice\n - maxconn 2500\n - user haproxy\n - group haproxy\n - daemon\n -\n -defaults\n - log global\n - mode tcp\n - timeout connect 10s\n - timeout client 36h\n - timeout server 36h\n - option dontlognull\n -\n -listen ssh\n - bind *:${ssh_port}\n - balance leastconn\n - mode tcp\n - server hostA 192.168.100.4:${ssh_port} check inter 10s fall 2 rise 1\n - server hostB 192.168.100.5:${ssh_port} check inter 10s fall 2 rise 1\n - server hostC 192.168.100.6:${ssh_port} check inter 10s fall 2 rise 1\n +ha_proxy_cfg=" +global + log /dev/log local0 + log /dev/log local1 notice + maxconn 2500 + user haproxy + group haproxy + daemon +defaults + log global + mode tcp + timeout connect 10s + timeout client 36h + timeout server 36h + option dontlognull +listen ssh + bind 0.0.0.0:${ssh_port} + balance leastconn + mode tcp + +$(echo "${server_node_ids[@]}" | python3 -c "print(\"\\n\".join([f\" server pi{i} 192.168.100.{i}:${ssh_port} check inter 10s fall rise 1 \" for i in input().split()]))") " -echo $ha_proxy_cfg ssh_opts="-oStrictHostKeyChecking=no" @@ -116,9 +112,8 @@ function copy_ssh_keys() { done } - -function reload_nginx_confs() { - dead_files=("/etc/nginx/sites-enabled/default" "/etc/nginx/nginx.conf" "$load_balancer_nginx_site_file") +function reload_loadbalancer_conf() { + dead_files=("/etc/nginx/sites-enabled/default" "/etc/nginx/nginx.conf" "$load_balancer_nginx_site_file" "$ha_proxy_cfg_file") for file in "${dead_files[@]}" do [ -e $file ] && sudo rm $file @@ -128,6 +123,10 @@ function reload_nginx_confs() { echo $load_balancer_nginx_site | sudo tee $load_balancer_nginx_site_file sudo systemctl restart nginx + + printf '$ha_proxy_cfg' | sudo tee $ha_proxy_cfg_file + + sudo systemctl restart haproxy } function build_frontend() { @@ -152,7 +151,7 @@ function build_server() { temp_file=$(mktemp) cp "${build_dir}/.env" $temp_file - printf "\nNODE_ID=$node_id\n" >> $temp_file + printf "\nNODE_ID=$node_conn\nRELEASE_NODE=chessh@192.168.100.${node_id}\n" >> $temp_file scp $ssh_opts $temp_file $node_conn:~/.env cp "${build_dir}/chessh.service" $temp_file @@ -161,11 +160,14 @@ function build_server() { scp $ssh_opts "${build_dir}/build_server.sh" $node_conn:~/ + scp $ssh_opts $erlang_hosts_file $node_conn:~/ + ssh $ssh_opts $node_conn "~/build_server.sh" } function build_server_nodes() { copy_ssh_keys + "$(printf "'192.168.100.%s'\n" ${server_node_ids[@]})" > $erlang_hosts_file for node_id in "${server_node_ids[@]}" do @@ -173,6 +175,6 @@ function build_server_nodes() { done } -reload_nginx_confs +reload_loadbalancer_conf build_server_nodes build_frontend_nodes diff --git a/buildscripts/build/build_server.sh b/buildscripts/build/build_server.sh index 80e6e91..1857e6b 100755 --- a/buildscripts/build/build_server.sh +++ b/buildscripts/build/build_server.sh @@ -30,6 +30,7 @@ fi # Build cd $chessh_path [ -d "$chessh_path/priv/keys" ] && cp ~/keys/* "$chessh_path/priv/keys/" || cp -r ~/keys "$chessh_path/priv" +mv $HOME/.hosts.erlang . sudo docker build . -t chessh/server # Systemd service diff --git a/config/config.exs b/config/config.exs index 44230ae..4d32140 100644 --- a/config/config.exs +++ b/config/config.exs @@ -23,4 +23,12 @@ config :chessh, Web, config :joken, default_signer: "secret" +config :libcluster, + topologies: [ + erlang_hosts_example: [ + strategy: Elixir.Cluster.Strategy.ErlangHosts, + config: [timeout: 30_000] + ] + ] + import_config "#{config_env()}.exs" diff --git a/config/prod.exs b/config/prod.exs index bb12243..15d52a1 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -10,3 +10,11 @@ config :chessh, RateLimits, max_concurrent_user_sessions: 5, player_session_message_burst_ms: 750, player_session_message_burst_rate: 8 + +config :libcluster, + topologies: [ + erlang_hosts_example: [ + strategy: Elixir.Cluster.Strategy.ErlangHosts, + config: [timeout: 30_000] + ] + ] diff --git a/lib/chessh/application.ex b/lib/chessh/application.ex index 5538e39..59926cc 100644 --- a/lib/chessh/application.ex +++ b/lib/chessh/application.ex @@ -22,7 +22,9 @@ defmodule Chessh.Application do scheme: :http, plug: Chessh.Web.Endpoint, options: [port: Application.get_env(:chessh, Web)[:port]] - ) + ), + {Cluster.Supervisor, + [Application.get_env(:libcluster, :topologies), [name: Chessh.ClusterSupervisor]]} ] opts = [strategy: :one_for_one, name: Chessh.Supervisor] diff --git a/mix.exs b/mix.exs index 8c89083..e977912 100644 --- a/mix.exs +++ b/mix.exs @@ -37,7 +37,8 @@ defmodule Chessh.MixProject do {:syn, "~> 3.3"}, {:jason, "~> 1.3"}, {:plug_cowboy, "~> 2.2"}, - {:joken, "~> 2.5"} + {:joken, "~> 2.5"}, + {:libcluster, "~> 3.3"} ] end diff --git a/mix.lock b/mix.lock index 73438d2..9ae7f30 100644 --- a/mix.lock +++ b/mix.lock @@ -19,6 +19,7 @@ "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, "joken": {:hex, :joken, "2.5.0", "09be497d804b8115eb6f07615cef2e60c2a1008fb89dc0aef0d4c4b4609b99aa", [:mix], [{:jose, "~> 1.11.2", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "22b25c89617c5ed8ca7b31026340a25ea0f9ca7160f9706b79be9ed81fdf74e7"}, "jose": {:hex, :jose, "1.11.5", "3bc2d75ffa5e2c941ca93e5696b54978323191988eb8d225c2e663ddfefd515e", [:mix, :rebar3], [], "hexpm", "dcd3b215bafe02ea7c5b23dafd3eb8062a5cd8f2d904fd9caa323d37034ab384"}, + "libcluster": {:hex, :libcluster, "3.3.2", "84c6ebfdc72a03805955abfb5ff573f71921a3e299279cc3445445d5af619ad1", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8b691ce8185670fc8f3fc0b7ed59eff66c6889df890d13411f8f1a0e6871d8a5"}, "mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"}, "plug": {:hex, :plug, "1.14.0", "ba4f558468f69cbd9f6b356d25443d0b796fbdc887e03fa89001384a9cac638f", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bf020432c7d4feb7b3af16a0c2701455cbbbb95e5b6866132cb09eb0c29adc14"}, "plug_cowboy": {:hex, :plug_cowboy, "2.6.0", "d1cf12ff96a1ca4f52207c5271a6c351a4733f413803488d75b70ccf44aebec2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "073cf20b753ce6682ed72905cd62a2d4bd9bad1bf9f7feb02a1b8e525bd94fa6"},