From 0818af279e20bafdb655198be5b649c3551d524d Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 3 Mar 2023 19:31:26 +0100 Subject: [PATCH 1/3] allow using bootstrap mode for no ACLs on nomad or non bootstrap mode with a self-service approach --- examples/advanced-setup/main.tf | 4 ---- .../nomad/jobs/demo-webapp.nomad | 0 .../advanced-setup/nomad/jobs/traefik.nomad | 0 examples/advanced-setup/nomad/main.tf | 0 hcloud_load_balancer.tf | 3 --- hcloud_server.tf | 23 +++---------------- output.tf | 6 +---- scripts/client_setup.sh | 2 +- scripts/server_setup.sh | 2 +- variables.tf | 6 +++++ 10 files changed, 12 insertions(+), 34 deletions(-) mode change 100644 => 100755 examples/advanced-setup/nomad/jobs/demo-webapp.nomad mode change 100644 => 100755 examples/advanced-setup/nomad/jobs/traefik.nomad mode change 100644 => 100755 examples/advanced-setup/nomad/main.tf mode change 100644 => 100755 hcloud_load_balancer.tf mode change 100644 => 100755 hcloud_server.tf mode change 100644 => 100755 output.tf diff --git a/examples/advanced-setup/main.tf b/examples/advanced-setup/main.tf index fa1b517..40beeb5 100644 --- a/examples/advanced-setup/main.tf +++ b/examples/advanced-setup/main.tf @@ -65,10 +65,6 @@ output "server_info" { value = module.hetzner-nomad-consul.server_info } -output "nomad_token" { - value = module.hetzner-nomad-consul.nomad_token -} - output "nomad_address" { value = module.hetzner-nomad-consul.nomad_address } diff --git a/examples/advanced-setup/nomad/jobs/demo-webapp.nomad b/examples/advanced-setup/nomad/jobs/demo-webapp.nomad old mode 100644 new mode 100755 diff --git a/examples/advanced-setup/nomad/jobs/traefik.nomad b/examples/advanced-setup/nomad/jobs/traefik.nomad old mode 100644 new mode 100755 diff --git a/examples/advanced-setup/nomad/main.tf b/examples/advanced-setup/nomad/main.tf old mode 100644 new mode 100755 diff --git a/hcloud_load_balancer.tf b/hcloud_load_balancer.tf old mode 100644 new mode 100755 index 4783913..a96bf4b --- a/hcloud_load_balancer.tf +++ b/hcloud_load_balancer.tf @@ -1,7 +1,4 @@ resource "hcloud_load_balancer" "load_balancer" { - depends_on = [ - null_resource.fetch_nomad_token - ] name = "nomad-load-balancer" load_balancer_type = "lb11" location = var.hetzner_datacenter diff --git a/hcloud_server.tf b/hcloud_server.tf old mode 100644 new mode 100755 index 0280e43..c29490a --- a/hcloud_server.tf +++ b/hcloud_server.tf @@ -26,7 +26,7 @@ resource "hcloud_server" "main" { inline = [ "echo 'Waiting for cloud-init to complete...'", "cloud-init status --wait > /dev/null", - "echo 'Completed cloud-init!'", + "echo 'Completed cloud-init!'" ] connection { @@ -53,11 +53,13 @@ resource "null_resource" "deployment" { provisioner "file" { content = each.value.type == "server" ? templatefile("${path.module}/scripts/server_setup.sh", { + bootstrap = var.bootstrap SERVER_COUNT = length(local.Server_Count) IP_RANGE = local.IP_range SERVER_IPs = jsonencode([for key, value in local.Extended_Aggregator_IPs : value.private_ipv4[0] if value.type == "server"]) }) : templatefile("${path.module}/scripts/client_setup.sh", { + bootstrap = var.bootstrap SERVER_COUNT = length(local.Server_Count) IP_RANGE = local.IP_range SERVER_IPs = jsonencode([for key, value in local.Extended_Aggregator_IPs : value.private_ipv4[0] if value.type == "server"]) @@ -73,24 +75,6 @@ resource "null_resource" "deployment" { } } -resource "time_sleep" "wait_60_seconds" { - depends_on = [null_resource.deployment] - create_duration = "60s" -} - -resource "null_resource" "fetch_nomad_token" { - depends_on = [time_sleep.wait_60_seconds] - - provisioner "local-exec" { - command = < 0])} - do - ssh -i ${path.root}/certs/machines.pem -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" root@$i curl --request POST http://localhost:4646/v1/acl/bootstrap | jq -r -R 'fromjson? | .SecretID?' >> ${path.root}/certs/nomad_token - done - EOF - } -} - resource "local_file" "private_key" { content = tls_private_key.machines.private_key_openssh filename = "${path.root}/certs/machines.pem" @@ -109,7 +93,6 @@ resource "hcloud_ssh_key" "default" { resource "null_resource" "clean_up" { provisioner "local-exec" { command = < Date: Fri, 3 Mar 2023 19:57:32 +0100 Subject: [PATCH 2/3] simplified setup bash script --- hcloud_server.tf | 5 +- scripts/base_configuration.sh | 129 ++++++---------------------------- variables.tf | 12 ++++ 3 files changed, 39 insertions(+), 107 deletions(-) diff --git a/hcloud_server.tf b/hcloud_server.tf index c29490a..2f9c2e3 100755 --- a/hcloud_server.tf +++ b/hcloud_server.tf @@ -20,7 +20,10 @@ resource "hcloud_server" "main" { ipv6_enabled = false } - user_data = file("${path.module}/scripts/base_configuration.sh") + user_data = templatefile("${path.module}/scripts/base_configuration.sh", { + CONSUL_VERSION = var.apt_consul_version + NOMAD_VERSION = var.apt_nomad_version + }) provisioner "remote-exec" { inline = [ diff --git a/scripts/base_configuration.sh b/scripts/base_configuration.sh index b945675..a5ff93e 100644 --- a/scripts/base_configuration.sh +++ b/scripts/base_configuration.sh @@ -1,110 +1,27 @@ #!/bin/bash # Update the server and install needed packages -apt update -apt upgrade -y -apt install unzip jq -y -cd /root/ - -# To install HashiCorp Consul, we need to download and install the respective binary. First, define the version and host in an environment variable -export CONSUL_VERSION="1.14.3" -export CONSUL_URL="https://releases.hashicorp.com/consul" - -# Download the binary, decompress it and install it on your server -curl --silent --remote-name ${CONSUL_URL}/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip -unzip consul_${CONSUL_VERSION}_linux_amd64.zip -chown root:root consul -mv consul /usr/local/bin/ -rm consul_${CONSUL_VERSION}_linux_amd64.zip - -# We can now add autocomplete functionality for Consul (optional) -consul -autocomplete-install -complete -C /usr/local/bin/consul consul - -# Create a user for Consul -useradd --system --home /etc/consul.d --shell /bin/false consul -mkdir --parents /opt/consul -chown --recursive consul:consul /opt/consul - -# Prepare the Consul configuration -mkdir --parents /etc/consul.d -touch /etc/consul.d/consul.hcl -chown --recursive consul:consul /etc/consul.d -chmod 640 /etc/consul.d/consul.hcl - -# Similar to the Consul binary, we first define the version as a variable -export NOMAD_VERSION="1.4.3" - -# Download and install the binary -curl --silent --remote-name https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip -unzip nomad_${NOMAD_VERSION}_linux_amd64.zip -chown root:root nomad -mv nomad /usr/local/bin/ -rm nomad_${NOMAD_VERSION}_linux_amd64.zip - -# Add autocomplete functionality to nomad (optional) -nomad -autocomplete-install -complete -C /usr/local/bin/nomad nomad - -# Prepare the data directory -mkdir --parents /opt/nomad - -# Create the basic configuration file for nomad -mkdir --parents /etc/nomad.d -chmod 700 /etc/nomad.d +curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - +apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" +apt-get update +apt-get upgrade -y +apt-get install jq -y +apt-get install -y consul=${CONSUL_VERSION} +apt-get install -y nomad=${NOMAD_VERSION} + + +chown -R consul:consul /etc/consul.d +chmod -R 640 /etc/consul.d/* + + +chown -R nomad:nomad /etc/nomad.d +chmod -R 640 /etc/nomad.d/* + cat < /etc/nomad.d/nomad.hcl datacenter = "dc1" data_dir = "/opt/nomad" EOF -# Consul and Nomad should start automatically after boot. To enable this, create a systemd service for both of them. -cat < /etc/systemd/system/consul.service -[Unit] -Description="HashiCorp Consul - A service mesh solution" -Documentation=https://www.consul.io/ -Requires=network-online.target -After=network-online.target -ConditionFileNotEmpty=/etc/consul.d/consul.hcl - -[Service] -Type=exec -User=consul -Group=consul -ExecStart=/usr/local/bin/consul agent -config-dir=/etc/consul.d/ -ExecReload=/bin/kill --signal HUP $MAINPID -KillMode=process -KillSignal=SIGTERM -Restart=on-failure -LimitNOFILE=65536 - -[Install] -WantedBy=multi-user.target -EOF - -cat < /etc/systemd/system/nomad.service -[Unit] -Description=Nomad -Documentation=https://www.nomadproject.io/docs -Wants=network-online.target -After=network-online.target - -[Service] -ExecReload=/bin/kill -HUP $MAINPID -ExecStart=/usr/local/bin/nomad agent -config /etc/nomad.d -KillMode=process -KillSignal=SIGINT -LimitNOFILE=infinity -LimitNPROC=infinity -Restart=on-failure -RestartSec=2 -StartLimitBurst=3 -StartLimitIntervalSec=10 -TasksMax=infinity - -[Install] -WantedBy=multi-user.target -EOF - # Configure auto updates and security patches apt install unattended-upgrades -y @@ -115,17 +32,17 @@ EOF cat < /etc/apt/apt.conf.d/50unattended-upgrades Unattended-Upgrade::Allowed-Origins { - "\${distro_id}:\${distro_codename}"; - "\${distro_id}:\${distro_codename}-security"; + "\$${distro_id}:\$${distro_codename}"; + "\$${distro_id}:\$${distro_codename}-security"; // Extended Security Maintenance; doesn't necessarily exist for // every release and this system may not have it installed, but if // available, the policy for updates is such that unattended-upgrades // should also install from here by default. - "\${distro_id}ESMApps:\${distro_codename}-apps-security"; - "\${distro_id}ESM:\${distro_codename}-infra-security"; - "\${distro_id}:\${distro_codename}-updates"; - "\${distro_id}:\${distro_codename}-proposed"; -// "\${distro_id}:\${distro_codename}-backports"; + "\$${distro_id}ESMApps:\$${distro_codename}-apps-security"; + "\$${distro_id}ESM:\$${distro_codename}-infra-security"; + "\$${distro_id}:\$${distro_codename}-updates"; + "\$${distro_id}:\$${distro_codename}-proposed"; +// "\$${distro_id}:\$${distro_codename}-backports"; }; // Python regular expressions, matching packages to exclude from upgrading diff --git a/variables.tf b/variables.tf index 18c207a..43f6dc7 100644 --- a/variables.tf +++ b/variables.tf @@ -37,4 +37,16 @@ variable "hetzner_network_zone" { type = string description = "Hetzner Cloud Network Zone" default = "eu-central" +} + +variable "apt_consul_version" { + type = string + description = "Consul version to install" + default = "1.15.0-1" +} + +variable "apt_nomad_version" { + type = string + description = "Nomad version to install" + default = "1.5.0-1" } \ No newline at end of file From c247df9e840461c18eaa87760be59b30d2cf27b8 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 3 Mar 2023 20:35:33 +0100 Subject: [PATCH 3/3] added private key to module output --- README.md | 3 --- output.tf | 4 ++++ scripts/server_setup.sh | 11 ----------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9a4111d..e505391 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,6 @@ This module allows you to create a manually-scalable high-availability nomad cluster on Hetzner Cloud. You only need to provide a API token as variable and a default cluster with 3 servers and 1 client will be created. -## Dependencies -- [jq](https://stedolan.github.io/jq/) - ## Advanced Usage The module is mainly addressed to people who want to test the technology running terraform on their local PC, but it can be used in professional workflows as well. diff --git a/output.tf b/output.tf index 90cfc96..9e5b8a8 100755 --- a/output.tf +++ b/output.tf @@ -13,4 +13,8 @@ output "nomad_address" { output "network_id" { value = hcloud_network.network.id +} + +output "tls_private_key" { + value = tls_private_key.machines.private_key_pem } \ No newline at end of file diff --git a/scripts/server_setup.sh b/scripts/server_setup.sh index 93e786b..c5d64ce 100644 --- a/scripts/server_setup.sh +++ b/scripts/server_setup.sh @@ -55,15 +55,4 @@ systemctl enable nomad systemctl start consul systemctl start nomad -# consul acl bootstrap -format=json | jq -r -R 'fromjson? | .SecretID?' > /etc/consul.d/acl_master_token -# export CONSUL_HTTP_TOKEN=$(cat /etc/consul.d/acl_master_token) -# export CONSUL_HTTP_ADDR="http://127.0.0.1:8500" -# To check the cluster, run the following command on one of your servers -#consul members -#sleep 10 -#curl --request POST http://localhost:4646/v1/acl/bootstrap | jq -r '.SecretID' > nomad_token - -# Since we use ACLs (Access Control Lists) on Nomad, we have to get the bootstrap token first, before checking the status here as well. -#nomad server members - #reboot \ No newline at end of file