Elixir cluster!

.gitignore

@@ -27,3 +27,5 @@ server-*.tar
 
 .env
 .env.prod
+
+.hosts.erlang

Dockerfile

@@ -62,6 +62,10 @@ ENV LC_ALL en_US.UTF-8
 WORKDIR "/app"
 RUN chown nobody /app
 
+# set OTP_ROOT so elixir can find nodes
+ENV OTP_ROOT=/app
+COPY .hosts.erlang /app/.hosts.erlang
+
 # set runner ENV
 ENV MIX_ENV="prod"
 

buildscripts/build/.gitignore

@@ -1,2 +1,3 @@
 .env
 keys/
+.hosts.erlang

buildscripts/build/build.sh

@@ -10,8 +10,9 @@ server_node_ids=(4 5 6)
 build_dir="${HOME}/src/chessh/buildscripts/build"
 
 server_name="chessh.linux.usu.edu"
+erlang_hosts_file="${build_dir}/.hosts.erlang"
 load_balancer_nginx_site_file="/etc/nginx/sites-enabled/${server_name}.conf"
-ha_proxy_cfg="/etc/haproxy/haproxy.cfg"
+ha_proxy_cfg_file="/etc/haproxy/haproxy.cfg"
 ssl_cert_path="/etc/letsencrypt/live/${server_name}"
 certbot_webroot_path="/var/www/html/${server_name}"
 load_balancer_nginx_site="
@@ -63,34 +64,29 @@ server {
 	return 404;
 }
 "
-# TODO: Servers should pull server ids from array
-ha_proxy_cfg="
-global\n
-	log /dev/log 	local0\n
-	log /dev/log	local1 notice\n
-	maxconn 2500\n
-	user haproxy\n
-	group haproxy\n
-	daemon\n
-\n
-defaults\n
-	log 	global\n
-	mode	tcp\n
-	timeout connect 10s\n
-	timeout client 36h\n
-	timeout server 36h\n
-	option 	dontlognull\n
-\n
-listen ssh\n
-	bind 	*:${ssh_port}\n
-	balance	leastconn\n
-	mode	tcp\n
 
-	server hostA  192.168.100.4:${ssh_port} check inter 10s fall 2 rise 1\n
+ha_proxy_cfg="
-	server hostB  192.168.100.5:${ssh_port} check inter 10s fall 2 rise 1\n
+global
-	server hostC  192.168.100.6:${ssh_port} check inter 10s fall 2 rise 1\n
+	log /dev/log 	local0
+	log /dev/log	local1 notice
+	maxconn 2500
+	user haproxy
+	group haproxy
+	daemon
+defaults
+	log 	global
+	mode	tcp
+	timeout connect 10s
+	timeout client 36h
+	timeout server 36h
+	option 	dontlognull
+listen ssh
+	bind 	0.0.0.0:${ssh_port}
+	balance	leastconn
+	mode	tcp
+
+$(echo "${server_node_ids[@]}" | python3 -c "print(\"\\n\".join([f\"  server pi{i} 192.168.100.{i}:${ssh_port} check inter 10s fall rise 1 \" for i in input().split()]))")
 "
-echo $ha_proxy_cfg
 
 ssh_opts="-oStrictHostKeyChecking=no"
 
@@ -116,9 +112,8 @@ function copy_ssh_keys() {
     done
 }
 
-
+function reload_loadbalancer_conf() {
-function reload_nginx_confs() {
+    dead_files=("/etc/nginx/sites-enabled/default" "/etc/nginx/nginx.conf" "$load_balancer_nginx_site_file" "$ha_proxy_cfg_file")
-    dead_files=("/etc/nginx/sites-enabled/default" "/etc/nginx/nginx.conf" "$load_balancer_nginx_site_file")
     for file in "${dead_files[@]}"
     do
         [ -e $file ] && sudo rm $file
@@ -128,6 +123,10 @@ function reload_nginx_confs() {
     echo $load_balancer_nginx_site | sudo tee $load_balancer_nginx_site_file
     
     sudo systemctl restart nginx
+
+    printf '$ha_proxy_cfg' | sudo tee $ha_proxy_cfg_file
+
+    sudo systemctl restart haproxy
 }
 
 function build_frontend() {
@@ -152,7 +151,7 @@ function build_server() {
     temp_file=$(mktemp)
     
     cp "${build_dir}/.env" $temp_file
-    printf "\nNODE_ID=$node_id\n" >> $temp_file
+    printf "\nNODE_ID=$node_conn\nRELEASE_NODE=chessh@192.168.100.${node_id}\n" >> $temp_file
     scp $ssh_opts $temp_file $node_conn:~/.env
 
     cp "${build_dir}/chessh.service" $temp_file
@@ -161,11 +160,14 @@ function build_server() {
     
     scp $ssh_opts "${build_dir}/build_server.sh" $node_conn:~/
 
+    scp $ssh_opts $erlang_hosts_file $node_conn:~/
+
     ssh $ssh_opts $node_conn "~/build_server.sh"
 }
 
 function build_server_nodes() {
     copy_ssh_keys
+    "$(printf "'192.168.100.%s'\n" ${server_node_ids[@]})" > $erlang_hosts_file
     
     for node_id in "${server_node_ids[@]}"
     do
@@ -173,6 +175,6 @@ function build_server_nodes() {
     done
 }
 
-reload_nginx_confs
+reload_loadbalancer_conf
 build_server_nodes
 build_frontend_nodes

buildscripts/build/build_server.sh

@@ -30,6 +30,7 @@ fi
 # Build
 cd $chessh_path
 [ -d "$chessh_path/priv/keys" ] && cp ~/keys/* "$chessh_path/priv/keys/" || cp -r ~/keys "$chessh_path/priv"
+mv $HOME/.hosts.erlang .
 sudo docker build . -t chessh/server
 
 # Systemd service

config/config.exs

@@ -23,4 +23,12 @@ config :chessh, Web,
 
 config :joken, default_signer: "secret"
 
+config :libcluster,
+  topologies: [
+    erlang_hosts_example: [
+      strategy: Elixir.Cluster.Strategy.ErlangHosts,
+      config: [timeout: 30_000]
+    ]
+  ]
+
 import_config "#{config_env()}.exs"

config/prod.exs

@@ -10,3 +10,11 @@ config :chessh, RateLimits,
   max_concurrent_user_sessions: 5,
   player_session_message_burst_ms: 750,
   player_session_message_burst_rate: 8
+
+config :libcluster,
+  topologies: [
+    erlang_hosts_example: [
+      strategy: Elixir.Cluster.Strategy.ErlangHosts,
+      config: [timeout: 30_000]
+    ]
+  ]

lib/chessh/application.ex

@@ -22,7 +22,9 @@ defmodule Chessh.Application do
         scheme: :http,
         plug: Chessh.Web.Endpoint,
         options: [port: Application.get_env(:chessh, Web)[:port]]
-      )
+      ),
+      {Cluster.Supervisor,
+       [Application.get_env(:libcluster, :topologies), [name: Chessh.ClusterSupervisor]]}
     ]
 
     opts = [strategy: :one_for_one, name: Chessh.Supervisor]

mix.exs

@@ -37,7 +37,8 @@ defmodule Chessh.MixProject do
       {:syn, "~> 3.3"},
       {:jason, "~> 1.3"},
       {:plug_cowboy, "~> 2.2"},
-      {:joken, "~> 2.5"}
+      {:joken, "~> 2.5"},
+      {:libcluster, "~> 3.3"}
     ]
   end
 

mix.lock

@@ -19,6 +19,7 @@
   "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"},
   "joken": {:hex, :joken, "2.5.0", "09be497d804b8115eb6f07615cef2e60c2a1008fb89dc0aef0d4c4b4609b99aa", [:mix], [{:jose, "~> 1.11.2", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "22b25c89617c5ed8ca7b31026340a25ea0f9ca7160f9706b79be9ed81fdf74e7"},
   "jose": {:hex, :jose, "1.11.5", "3bc2d75ffa5e2c941ca93e5696b54978323191988eb8d225c2e663ddfefd515e", [:mix, :rebar3], [], "hexpm", "dcd3b215bafe02ea7c5b23dafd3eb8062a5cd8f2d904fd9caa323d37034ab384"},
+  "libcluster": {:hex, :libcluster, "3.3.2", "84c6ebfdc72a03805955abfb5ff573f71921a3e299279cc3445445d5af619ad1", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8b691ce8185670fc8f3fc0b7ed59eff66c6889df890d13411f8f1a0e6871d8a5"},
   "mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"},
   "plug": {:hex, :plug, "1.14.0", "ba4f558468f69cbd9f6b356d25443d0b796fbdc887e03fa89001384a9cac638f", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bf020432c7d4feb7b3af16a0c2701455cbbbb95e5b6866132cb09eb0c29adc14"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.6.0", "d1cf12ff96a1ca4f52207c5271a6c351a4733f413803488d75b70ccf44aebec2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "073cf20b753ce6682ed72905cd62a2d4bd9bad1bf9f7feb02a1b8e525bd94fa6"},