apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
  name: gpu
spec:
  consolidation:
    enabled: true
  requirements:
  - key: node.kubernetes.io/instance-type
    operator: In
    values: ["g4dn.xlarge", "g4dn.2xlarge", "g4dn.4xlarge", "g5.xlarge"]
  - key: karpenter.sh/capacity-type
    operator: In
    values: ["on-demand", "spot"]
#    values: ["on-demand"]
  taints:
  - key: nvidia.com/gpu
    value: "true"
    effect: NoSchedule
  providerRef:
    name: gpu-node
---
apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
  name: cpu
spec:
  consolidation:
    enabled: true
  requirements:
  - key: node.kubernetes.io/instance-type
    operator: In
    values: ["t4g.small", "t4g.medium", "t4g.large"]
  - key: kubernetes.io/arch
    operator: In
    values: ["amd64", "arm64"]
  - key: karpenter.sh/capacity-type
    operator: In
    values: ["on-demand", "spot"]
  taints:
  - key: wavtool.com/app-server-only
    value: "true"
    effect: NoSchedule
  providerRef:
    name: cpu-node
---
apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
  name: spot-big-cpu
spec:
  consolidation:
    enabled: true
  requirements:
  - key: karpenter.k8s.aws/instance-category
    operator: In
    values: ["c", "m", "r"]
  - key: karpenter.k8s.aws/instance-generation
    operator: In
    values: ["3", "4", "5", "6", "7"]
  - key: kubernetes.io/arch
    operator: In
    values: ["amd64"]
  - key: karpenter.sh/capacity-type
    operator: In
    values: ["spot"]
  taints:
  - key: wavtool.com/spot-big-cpu-only
    value: "true"
    effect: NoSchedule
  providerRef:
    name: cpu-node
---
apiVersion: karpenter.k8s.aws/v1alpha1
kind: AWSNodeTemplate
metadata:
  name: gpu-node
spec:
  subnetSelector:
    karpenter.sh/discovery: "wavtool"
  securityGroupSelector:
    karpenter.sh/discovery: "wavtool"
  amiFamily: "AL2"
  amiSelector:
    aws::ids: "ami-0fae6a7fa2f756810"
  userData: |
    #!/bin/bash
    sudo mkfs -F -t ext4 -m 0 -O ^has_journal /dev/nvme1n1
    sudo mount -o nobarrier,noatime /dev/nvme1n1 /mnt
    sudo mkdir -p /mnt/containerd /var/lib/containerd
    sudo mount --bind /mnt/containerd /var/lib/containerd
    #sudo mkdir -p /mnt/soci-snapshotter-grpc /var/lib/soci-snapshotter-grpc
    #sudo mount --bind /mnt/soci-snapshotter-grpc /var/lib/soci-snapshotter-grpc
    sudo cp -ax /var/lib/kubelet /mnt/kubelet
    sudo mount --bind /mnt/kubelet /var/lib/kubelet

    #sudo modprobe fuse
    #sudo systemctl start soci-snapshotter.service

    #sudo sed -i 's/default_runtime_name = "runc"/default_runtime_name = "nvidia"\nsnapshotter = "soci"\ndisable_snapshot_annotations = false/' /etc/eks/containerd/containerd-config.toml
    sudo sed -i 's/default_runtime_name = "runc"/default_runtime_name = "nvidia"/' /etc/eks/containerd/containerd-config.toml
    sudo cat >> /etc/eks/containerd/containerd-config.toml <<EOF
    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
    privileged_without_host_devices = false
    runtime_engine = ""
    runtime_root = ""
    runtime_type = "io.containerd.runc.v2"

    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
    BinaryName = "/usr/bin/nvidia-container-runtime"
    EOF

    # in above file
    #[proxy_plugins.soci]
    #type = "snapshot"
    #address = "/run/soci-snapshotter-grpc/soci-snapshotter-grpc.sock"

    mkdir -p ~ec2-user/.ssh/
    touch ~ec2-user/.ssh/authorized_keys
    cat >> ~ec2-user/.ssh/authorized_keys <<EOF
    ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEzkjmOXU4zuMXzrmmCp9Kk1+FOj4P3e0hQxC+A5qQ5/ m4burns@Marcs-MacBook-Air.local
    EOF
    chmod -R go-w ~ec2-user/.ssh/authorized_keys
    chown -R ec2-user ~ec2-user/.ssh
---
apiVersion: karpenter.k8s.aws/v1alpha1
kind: AWSNodeTemplate
metadata:
  name: cpu-node
spec:
  subnetSelector:
    karpenter.sh/discovery: "wavtool"
  securityGroupSelector:
    karpenter.sh/discovery: "wavtool"
  amiFamily: "AL2"
  userData: |
    #!/bin/bash

    mkdir -p ~ec2-user/.ssh/
    touch ~ec2-user/.ssh/authorized_keys
    cat >> ~ec2-user/.ssh/authorized_keys <<EOF
    ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEzkjmOXU4zuMXzrmmCp9Kk1+FOj4P3e0hQxC+A5qQ5/ m4burns@Marcs-MacBook-Air.local
    EOF
    chmod -R go-w ~ec2-user/.ssh/authorized_keys
    chown -R ec2-user ~ec2-user/.ssh