commit 6ae461b7ef20668751500891131d29225390c0f3 Author: NAGY Akos Date: Wed Dec 13 19:21:40 2023 +0200 add base diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f124295 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*example +*temp +*save +.vscode +ignition/*json +post.sh diff --git a/group_vars/all/main.yaml b/group_vars/all/main.yaml new file mode 100644 index 0000000..a632f8d --- /dev/null +++ b/group_vars/all/main.yaml @@ -0,0 +1,48 @@ +--- +kube_token: "" + +vcenter_hostname: "" +vcenter_username: "" +vcenter_password: "{{ ansible_password }}" + +datacenter: "" +cluster: "" +datastore: "" +resource_pool: "" +vm_folder_name: "srv" +vm_parent_folder_name: "" +vm_folder_name_full: "" +template_folder_name: "templates" +template_ folder_name_full: "" +parent_folder: "" + +ovf_file: "/home/user/Downloads/iso/fedora-coreos-39.20231119.3.0-vmware.x86_64.ova" + +vm_hostname_test: "srv-test02" +vm_hostname_template: "coreos-template" + +ProvisioningNetworkLabel: "VLAN1111" + +gateway: 10.2.4.1 + +kubehosts: + k3s-1: + ip: 10.2.4.111 + hostname: k3s-1 + +kubehosts_rest: + k3s-2: + ip: 10.2.4.112 + hostname: k3s-2 + # k3s-3: + # ip: 10.2.4.113 + # hostname: k3s-3 + # k3s-4: + # ip: 10.2.4.114 + # hostname: k3s-4 + # k3s-5: + # ip: 10.2.4.115 + # hostname: k3s-5 + # k3s-6: + # ip: 10.2.4.116 + # hostname: k3s-6 diff --git a/ignition/build.sh b/ignition/build.sh new file mode 100755 index 0000000..fce75ad --- /dev/null +++ b/ignition/build.sh @@ -0,0 +1,19 @@ +# https://devnonsense.com/posts/k3s-on-fedora-coreos-bare-metal/ +# https://www.murillodigital.com/tech_talk/k3s_in_coreos/ + +NODE_PREFIX="k3s-" +IP_PREFIX="10.3.4.111" + +FIRST_NODE_INDEX="1" +NODES_INDEX="2 3 4 5 6" + +TOKEN=$1 + +podman run -i --rm quay.io/coreos/butane:release --pretty --strict < template01.yaml > $NODE_PREFIX$FIRST_NODE_INDEX.json + +if [ ! -z $1 ]; then + echo "Token: $TOKEN" + for NODE in $NODES_INDEX; do + sed "s/SEDtokenSED/${TOKEN}/g;s/SEDipSED/${NODE}/g" template02.yaml | podman run -i --rm quay.io/coreos/butane:release --pretty --strict > $NODE_PREFIX$NODE.json + done +fi diff --git a/ignition/template01.yaml b/ignition/template01.yaml new file mode 100644 index 0000000..fcfa7fa --- /dev/null +++ b/ignition/template01.yaml @@ -0,0 +1,143 @@ +variant: fcos +version: 1.5.0 +systemd: + units: + - name: getty@tty1.service + dropins: + - name: autologin-core.conf + contents: | + [Service] + ExecStart= + ExecStart=-/usr/sbin/agetty --autologin core --noclear %I $TERM + - name: run-k3s-prereq-installer.service + enabled: true + contents: | + [Unit] + After=network-online.target + Wants=network-online.target + Before=systemd-user-sessions.service + OnFailure=emergency.target + OnFailureJobMode=replace-irreversibly + ConditionPathExists=!/var/lib/k3s-prereq-installed + [Service] + RemainAfterExit=yes + Type=oneshot + ExecStart=/usr/local/bin/run-k3s-prereq-installer + ExecStartPost=/usr/bin/touch /var/lib/k3s-prereq-installed + ExecStartPost=/usr/bin/systemctl --no-block reboot + StandardOutput=kmsg+console + StandardError=kmsg+console + [Install] + WantedBy=multi-user.target + - name: run-k3s-installer.service + enabled: true + contents: | + [Unit] + After=network-online.target + Wants=network-online.target + Before=systemd-user-sessions.service + OnFailure=emergency.target + OnFailureJobMode=replace-irreversibly + ConditionPathExists=/var/lib/k3s-prereq-installed + ConditionPathExists=!/var/lib/k3s-installed + [Service] + RemainAfterExit=yes + Type=oneshot + ExecStart=/usr/local/bin/run-k3s-installer + ExecStartPost=/usr/bin/touch /var/lib/k3s-installed + #ExecStartPost=/usr/bin/systemctl --no-block reboot + StandardOutput=kmsg+console + StandardError=kmsg+console + [Install] + WantedBy=multi-user.target + - name: install-open-vm-tools.service + enabled: true + contents: | + [Unit] + After=network-online.target + Wants=network-online.target + Before=systemd-user-sessions.service + OnFailure=emergency.target + OnFailureJobMode=replace-irreversibly + ConditionPathExists=!/var/lib/open-vm-tools-installed + [Service] + RemainAfterExit=yes + Type=oneshot + ExecStart=/usr/local/bin/install-open-vm-tools + ExecStartPost=/usr/bin/touch /var/lib/open-vm-tools-installed + ExecStartPost=/usr/bin/systemctl --no-block reboot + StandardOutput=kmsg+console + StandardError=kmsg+console + [Install] + WantedBy=multi-user.target +storage: + files: + - path: /usr/local/bin/run-k3s-prereq-installer + mode: 0755 + contents: + inline: | + #!/usr/bin/env sh + main() { + rpm-ostree install https://github.com/k3s-io/k3s-selinux/releases/download/v1.4.stable.1/k3s-selinux-1.4-1.coreos.noarch.rpm + return 0 + } + main + - path: /etc/hostname + mode: 0644 + contents: + inline: | + k3s-1 + - path: /etc/NetworkManager/system-connections/ens192.nmconnection + mode: 0600 + contents: + inline: | + [connection] + id=ens192 + type=ethernet + interface-name=ens192 + [ipv4] + address1=10.2.4.111/24,10.2.4.1 + dns=10.2.4.1; + dns-search=example.com + may-fail=false + method=manual + - path: /usr/local/bin/run-k3s-installer + mode: 0755 + contents: + inline: | + #!/usr/bin/env sh + main() { + export K3S_KUBECONFIG_MODE="644" + export INSTALL_K3S_EXEC=" --flannel-backend=none --disable-network-policy" + # export INSTALL_K3S_EXEC=" --flannel-backend=none --disable-network-policy" K3S_URL="https://10.2.4.111:6443" K3S_TOKEN="" + curl -sfL https://get.k3s.io | sh - + return 0 + } + main + - path: /etc/rancher/k3s/kubelet.config + mode: 0644 + contents: + inline: | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + shutdownGracePeriod: 60s + shutdownGracePeriodCriticalPods: 10s + - path: /usr/local/bin/install-open-vm-tools + mode: 0755 + contents: + inline: | + #!/usr/bin/env sh + main() { + while [ ! -f /var/lib/k3s-installed ] ; do + sleep 2 + done + rpm-ostree install open-vm-tools + return 0 + } + main +passwd: + users: + - name: core + password_hash: "" + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAEvf7cRxXo3dCYwLU0cxbxSgD4FabfYA2UFeZv1fwo7 akosfred diff --git a/ignition/template02.yaml b/ignition/template02.yaml new file mode 100644 index 0000000..55a8ffe --- /dev/null +++ b/ignition/template02.yaml @@ -0,0 +1,143 @@ +variant: fcos +version: 1.5.0 +systemd: + units: + - name: getty@tty1.service + dropins: + - name: autologin-core.conf + contents: | + [Service] + ExecStart= + ExecStart=-/usr/sbin/agetty --autologin core --noclear %I $TERM + - name: run-k3s-prereq-installer.service + enabled: true + contents: | + [Unit] + After=network-online.target + Wants=network-online.target + Before=systemd-user-sessions.service + OnFailure=emergency.target + OnFailureJobMode=replace-irreversibly + ConditionPathExists=!/var/lib/k3s-prereq-installed + [Service] + RemainAfterExit=yes + Type=oneshot + ExecStart=/usr/local/bin/run-k3s-prereq-installer + ExecStartPost=/usr/bin/touch /var/lib/k3s-prereq-installed + ExecStartPost=/usr/bin/systemctl --no-block reboot + StandardOutput=kmsg+console + StandardError=kmsg+console + [Install] + WantedBy=multi-user.target + - name: run-k3s-installer.service + enabled: true + contents: | + [Unit] + After=network-online.target + Wants=network-online.target + Before=systemd-user-sessions.service + OnFailure=emergency.target + OnFailureJobMode=replace-irreversibly + ConditionPathExists=/var/lib/k3s-prereq-installed + ConditionPathExists=!/var/lib/k3s-installed + [Service] + RemainAfterExit=yes + Type=oneshot + ExecStart=/usr/local/bin/run-k3s-installer + ExecStartPost=/usr/bin/touch /var/lib/k3s-installed + #ExecStartPost=/usr/bin/systemctl --no-block reboot + StandardOutput=kmsg+console + StandardError=kmsg+console + [Install] + WantedBy=multi-user.target + - name: install-open-vm-tools.service + enabled: true + contents: | + [Unit] + After=network-online.target + Wants=network-online.target + Before=systemd-user-sessions.service + OnFailure=emergency.target + OnFailureJobMode=replace-irreversibly + ConditionPathExists=!/var/lib/open-vm-tools-installed + [Service] + RemainAfterExit=yes + Type=oneshot + ExecStart=/usr/local/bin/install-open-vm-tools + ExecStartPost=/usr/bin/touch /var/lib/open-vm-tools-installed + ExecStartPost=/usr/bin/systemctl --no-block reboot + StandardOutput=kmsg+console + StandardError=kmsg+console + [Install] + WantedBy=multi-user.target +storage: + files: + - path: /usr/local/bin/run-k3s-prereq-installer + mode: 0755 + contents: + inline: | + #!/usr/bin/env sh + main() { + rpm-ostree install https://github.com/k3s-io/k3s-selinux/releases/download/v1.4.stable.1/k3s-selinux-1.4-1.coreos.noarch.rpm + return 0 + } + main + - path: /etc/hostname + mode: 0644 + contents: + inline: | + k3s-1 + - path: /etc/NetworkManager/system-connections/ens192.nmconnection + mode: 0600 + contents: + inline: | + [connection] + id=ens192 + type=ethernet + interface-name=ens192 + [ipv4] + address1=10.3.4.11SEDipSED/24,10.2.4.1 + dns=10.2.4.1; + dns-search=example.com + may-fail=false + method=manual + - path: /usr/local/bin/run-k3s-installer + mode: 0755 + contents: + inline: | + #!/usr/bin/env sh + main() { + export K3S_KUBECONFIG_MODE="644" + export K3S_URL="https://10.2.4.111:6443" K3S_TOKEN="SEDtokenSED" + curl -sfL https://get.k3s.io | sh - + return 0 + } + main + - path: /etc/rancher/k3s/kubelet.config + mode: 0644 + contents: + inline: | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + shutdownGracePeriod: 60s + shutdownGracePeriodCriticalPods: 10s + - path: /usr/local/bin/install-open-vm-tools + mode: 0755 + contents: + inline: | + #!/usr/bin/env sh + main() { + while [ ! -f /var/lib/k3s-installed ] ; do + sleep 2 + done + rpm-ostree install open-vm-tools + return 0 + } + main +passwd: + users: + - name: core + password_hash: "" + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAEvf7cRxXo3dCYwLU0cxbxSgD4FabfYA2UFeZv1fwo7 akosfred + diff --git a/inventory.all.yaml b/inventory.all.yaml new file mode 100644 index 0000000..e79a18c --- /dev/null +++ b/inventory.all.yaml @@ -0,0 +1,30 @@ +--- +all: + children: + initial_controller: + hosts: + k0s-1: + controller: + hosts: + k0s-2: + k0s-3: + worker: + hosts: + k0s-4: + k0s-5: + k0s-6: + hosts: + k0s-1: + ansible_host: 10.2.4.101 + k0s-2: + ansible_host: 10.2.4.102 + k0s-3: + ansible_host: 10.2.4.103 + k0s-4: + ansible_host: 10.2.4.104 + k0s-5: + ansible_host: 10.2.4.105 + k0s-6: + ansible_host: 10.2.4.106 + vars: + ansible_user: core \ No newline at end of file diff --git a/inventory.yaml b/inventory.yaml new file mode 100644 index 0000000..1686061 --- /dev/null +++ b/inventory.yaml @@ -0,0 +1,9 @@ +--- +all: + children: + initial_controller: + hosts: + k3s-1: + hosts: + k3s-1: + ansible_host: 10.2.4.111 diff --git a/playbook.yaml b/playbook.yaml new file mode 100644 index 0000000..d501a0d --- /dev/null +++ b/playbook.yaml @@ -0,0 +1,60 @@ +# ansible-playbook playbook.yaml --ask-pass + +- name: VM creator playbook + hosts: localhost + gather_facts: false + + tasks: + - name: Apply VM creator tasks + ansible.builtin.include_tasks: + file: tasks/01create_template.yaml + apply: + tags: + - createtemplate + tags: + - always + +- name: 1st node deploy playbook + hosts: localhost + gather_facts: false + + tasks: + - name: Apply node deploy tasks + ansible.builtin.include_tasks: + file: tasks/02create_first.yaml + apply: + tags: + - createfirst + tags: + - always + +- name: Show node info + hosts: k3s-1 + gather_facts: false + vars: + ansible_user: core + + tasks: + - name: Execute post raw commands + ansible.builtin.include_tasks: + file: tasks/02create_first_raw.yaml + apply: + tags: + - createfirst + - createfirstpost + tags: + - always + +- name: Deploy rest of nodes playbook + hosts: localhost + gather_facts: false + + tasks: + - name: Apply node deploy tasks + ansible.builtin.include_tasks: + file: tasks/03create_nodes.yaml + apply: + tags: + - createrest + tags: + - always diff --git a/tasks/01create_template.yaml b/tasks/01create_template.yaml new file mode 100644 index 0000000..b4866fd --- /dev/null +++ b/tasks/01create_template.yaml @@ -0,0 +1,56 @@ +--- +- name: Create a VM folder on given datacenter + community.vmware.vcenter_folder: + hostname: '{{ vcenter_hostname }}' + username: '{{ vcenter_username }}' + password: '{{ vcenter_password }}' + datacenter_name: "{{ datacenter }}" + folder_name: "{{ vm_folder_name }}" + parent_folder: "{{ vm_parent_folder_name }}" + folder_type: vm + state: present + register: vm_folder_creation_result + delegate_to: localhost + +- name: Create a VM folder on given datacenter + community.vmware.vcenter_folder: + hostname: '{{ vcenter_hostname }}' + username: '{{ vcenter_username }}' + password: '{{ vcenter_password }}' + datacenter_name: "{{ datacenter }}" + folder_name: "{{ template_folder_name }}" + parent_folder: "{{ vm_parent_folder_name }}" + folder_type: vm + state: present + register: vm_folder_creation_result + delegate_to: localhost + +- name: Create virtual machine for template + community.vmware.vmware_deploy_ovf: + hostname: "{{ vcenter_hostname }}" + username: "{{ vcenter_username }}" + password: "{{ vcenter_password }}" + validate_certs: false + folder: "/{{ datacenter }}/vm/{{ vm_parent_folder_name }}/{{ template_folder_name }}" + datacenter: "{{ datacenter }}" + cluster: "{{ cluster }}" + datastore: "{{ datastore }}" + resource_pool: "{{ resource_pool }}" + name: "{{ vm_hostname_template }}" + networks: "{u'VM Network':u'{{ ProvisioningNetworkLabel }}'}" + power_on: false + ovf: "{{ ovf_file }}" + +- name: Convert vm to template + community.vmware.vmware_guest: + hostname: "{{ vcenter_hostname }}" + username: "{{ vcenter_username }}" + password: "{{ vcenter_password }}" + validate_certs: false + folder: "/{{ datacenter }}/vm/{{ vm_parent_folder_name }}/{{ template_folder_name }}" + datacenter: "{{ datacenter }}" + cluster: "{{ cluster }}" + datastore: "{{ datastore }}" + resource_pool: "{{ resource_pool }}" + name: "{{ vm_hostname_template }}" + is_template: true diff --git a/tasks/02create_first.yaml b/tasks/02create_first.yaml new file mode 100644 index 0000000..f24ead5 --- /dev/null +++ b/tasks/02create_first.yaml @@ -0,0 +1,30 @@ +--- +- name: Create virtual machines from template + community.vmware.vmware_guest: + hostname: "{{ vcenter_hostname }}" + username: "{{ vcenter_username }}" + password: "{{ vcenter_password }}" + validate_certs: false + folder: "/{{ datacenter }}/vm/{{ vm_parent_folder_name }}/{{ vm_folder_name }}" + datacenter: "{{ datacenter }}" + cluster: "{{ cluster }}" + datastore: "{{ datastore }}" + resource_pool: "{{ resource_pool }}" + name: "test-{{ item.value.hostname }}" + template: "{{ vm_hostname_template }}" + state: poweredon + disk: + - size: 100gb + advanced_settings: + - key: "guestinfo.ignition.config.data" + value: "{{ lookup('template', 'ignition/{{ item.value.hostname }}.json') | b64encode }}" + - key: "guestinfo.ignition.config.data.encoding" + value: "base64" + wait_for_ip_address: true + wait_for_ip_address_timeout: 600 + with_dict: "{{ kubehosts }}" + register: deploy_vm + +# - name: Show debug info +# debug: +# msg: "{{ deploy_vm.results.instance.ipv4 }}" diff --git a/tasks/02create_first_raw.yaml b/tasks/02create_first_raw.yaml new file mode 100644 index 0000000..2f9c368 --- /dev/null +++ b/tasks/02create_first_raw.yaml @@ -0,0 +1,18 @@ +--- +- name: Read token + ansible.builtin.raw: "cat /var/lib/rancher/k3s/server/token" + become: true + register: token_result + +- name: Show token + ansible.builtin.debug: + msg: "{{ token_result.stdout_lines }}" + +- name: Read kubeconfig + ansible.builtin.raw: "cat /etc/rancher/k3s/k3s.yaml" + become: true + register: kubeconfig_result + +- name: Show kubeconfig + ansible.builtin.debug: + msg: "{{ kubeconfig_result.stdout }}" diff --git a/tasks/03create_nodes.yaml b/tasks/03create_nodes.yaml new file mode 100644 index 0000000..2b7cff7 --- /dev/null +++ b/tasks/03create_nodes.yaml @@ -0,0 +1,28 @@ +--- +- name: Create virtual machines from template + community.vmware.vmware_guest: + hostname: "{{ vcenter_hostname }}" + username: "{{ vcenter_username }}" + password: "{{ vcenter_password }}" + validate_certs: false + folder: "/{{ datacenter }}/vm/{{ vm_parent_folder_name }}/{{ vm_folder_name }}" + datacenter: "{{ datacenter }}" + cluster: "{{ cluster }}" + datastore: "{{ datastore }}" + resource_pool: "{{ resource_pool }}" + name: "test-{{ item.value.hostname }}" + template: "{{ vm_hostname_template }}" + state: poweredon + disk: + - size: 100gb + advanced_settings: + - key: "guestinfo.ignition.config.data" + value: "{{ lookup('template', 'ignition/{{ item.value.hostname }}.json') | b64encode }}" + - key: "guestinfo.ignition.config.data.encoding" + value: "base64" + with_dict: "{{ kubehosts_rest }}" + register: deploy_vm + +# - name: Show debug info +# debug: +# msg: "{{ deploy_vm.results.instance.ipv4 }}"