apiVersion: apps/v1
kind: Deployment
metadata:
  name: wavtool-server-$DEPLOY_SERVER_ENVIRONMENT
  namespace: production
  labels:
    app: wavtool-server-$DEPLOY_SERVER_ENVIRONMENT
spec:
  selector:
    matchLabels:
      app: wavtool-server-$DEPLOY_SERVER_ENVIRONMENT
  template:
    metadata:
      labels:
        app: wavtool-server-$DEPLOY_SERVER_ENVIRONMENT
    spec:
      terminationGracePeriodSeconds: 60
      tolerations:
      - effect: NoSchedule
        key: wavtool.com/app-server-only
        operator: Exists
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                app: wavtool-server-$DEPLOY_SERVER_ENVIRONMENT
            topologyKey: kubernetes.io/hostname
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchLabels:
                  app: wavtool-server-$DEPLOY_SERVER_ENVIRONMENT
              topologyKey: topology.kubernetes.io/zone
      containers:
      - name: wavtool-server
        imagePullPolicy: Always
        image: 035363416972.dkr.ecr.us-east-1.amazonaws.com/wavtool-server:$DEPLOY_TAG
        ports:
        - containerPort: 3001
        env:
        - name: SERVER_URI
          value: "https://$DEPLOY_SERVER_DOMAIN"
        - name: APP_URI
          value: "https://$DEPLOY_APP_DOMAIN"
        - name: LANDING_URI
          value: "https://$DEPLOY_LANDING_DOMAIN"
        - name: FUSIONAUTH_URI
          value: "http://wavtool-idp-fusionauth.fusionauth.svc.cluster.local:9011"
        - name: URL_SHORTENER_PREFIX
          value: "https://$DEPLOY_SHORT_DOMAIN"
        - name: COOKIE_DOMAIN
          value: ".wavtool.com"
        - name: COOKIE_SECURE
          value: "1"
        - name: COOKIE_SESSION_KEY
          value: "$DEPLOY_COOKIE_SESSION_KEY"
        - name: DATABASE_USER
          value: "wavtool_app"
        - name: DATABASE_HOST
          value: "database-2.ctreuprwt6kf.us-east-1.rds.amazonaws.com"
        - name: DATABASE_NAME
          value: "$DEPLOY_DATABASE_NAME"
        - name: DATABASE_PORT
          value: "5432"
        - name: PGSSLMODE
          value: "require"
        - name: ENVIRONMENT
          value: "$DEPLOY_SERVER_ENVIRONMENT"
        - name: NODE_ENV
          value: "production"
        - name: GITHUB_CLIENT_ID
          value: "ab6c7a93e0b085dba09d"
        - name: GOOGLE_CLIENT_ID
          value: "806921377054-c15bqaisrdk171q4pkeoum57j5ott5uk.apps.googleusercontent.com"
        - name: FACEBOOK_APP_ID
          value: "1131773567455385"
        - name: NOTION_FEEDBACK_DB_ID
          value: "34a3bc215f7641b98a7a2f95a9805dde"
        - name: CF_STATIC_PKEY_ID
          value: "K1HZJ564424GVA"
        - name: SHUTDOWN_TIMEOUT
          value: "60"
        - name: REDIS_SENTINEL_NAME
          value: "wavtoolmaster"
        - name: REDIS_SENTINEL_HOST
          value: "prod-redis.redis.svc.cluster.local"
        - name: REDIS_SESSION_PREFIX
          value: "$DEPLOY_REDIS_SESSION_PREFIX"
        - name: SESSION_PROVIDER
          value: "redis"
        - name: AWS_ACCESS_KEY_ID
          value: "AKIAQQO6S5OGBACV3HNW"
        - name: RELEASE
          value: "$DEPLOY_RELEASE"
        envFrom:
        - configMapRef:
            name: wavtool-server-config-$DEPLOY_SERVER_ENVIRONMENT
        - secretRef:
            name: wavtool-server-secret
        - secretRef:
            name: wavtool-server-secret-$DEPLOY_SERVER_ENVIRONMENT
        - secretRef:
            name: redis-secret
        resources:
          requests:
            cpu: 300m
            memory: 256Mi
        volumeMounts:
        - name: persistent-storage
          mountPath: /live-deps
          subPath: server_live_deps
        readinessProbe:
          httpGet:
            path: "/healthz"
            port: 3001
          periodSeconds: 5
          failureThreshold: 10
        livenessProbe:
          httpGet:
            path: "/healthz"
            port: 3001
          initialDelaySeconds: 60
          periodSeconds: 30
          failureThreshold: 2
        lifecycle:
          preStop:
            exec:
              command:
                - /bin/sh
                - -c
                - |
                  touch /tmp/healthcheck-unhealthy
                  sleep 30
      volumes:
      - name: persistent-storage
        persistentVolumeClaim:
          claimName: efs-claim
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: wavtool-queues-$DEPLOY_SERVER_ENVIRONMENT
  labels:
    app: wavtool-queues-$DEPLOY_SERVER_ENVIRONMENT
spec:
  selector:
    matchLabels:
      app: wavtool-queues-$DEPLOY_SERVER_ENVIRONMENT
  template:
    metadata:
      labels:
        app: wavtool-queues-$DEPLOY_SERVER_ENVIRONMENT
    spec:
      containers:
      - name: wavtool-queues
        imagePullPolicy: Always
        image: 035363416972.dkr.ecr.us-east-1.amazonaws.com/wavtool-server:$DEPLOY_TAG
        command:
          - /sbin/tini
          - node
          - ./built/src/queues.js
        env:
        - name: DATABASE_USER
          value: "wavtool_app"
        - name: DATABASE_HOST
          value: "database-2.ctreuprwt6kf.us-east-1.rds.amazonaws.com"
        - name: DATABASE_NAME
          value: "$DEPLOY_DATABASE_NAME"
        - name: DATABASE_PORT
          value: "5432"
        - name: ENVIRONMENT
          value: "$DEPLOY_SERVER_ENVIRONMENT"
        - name: NODE_ENV
          value: "production"
        - name: SHUTDOWN_TIMEOUT
          value: "60"
        - name: PGSSLMODE
          value: "require"
        - name: REDIS_SENTINEL_NAME
          value: "wavtoolmaster"
        - name: REDIS_SENTINEL_HOST
          value: "prod-redis.redis.svc.cluster.local"
        - name: AWS_ACCESS_KEY_ID
          value: "AKIAQQO6S5OGBACV3HNW"
        envFrom:
        - configMapRef:
            name: wavtool-server-config-$DEPLOY_SERVER_ENVIRONMENT
        - secretRef:
            name: wavtool-server-secret
        - secretRef:
            name: wavtool-server-secret-$DEPLOY_SERVER_ENVIRONMENT
        - secretRef:
            name: redis-secret
        resources:
          requests:
            cpu: 100m
            memory: 256Mi
---
apiVersion: v1
kind: Service
metadata:
  name: wavtool-server-$DEPLOY_SERVER_ENVIRONMENT
  namespace: production
spec:
  selector:
    app: wavtool-server-$DEPLOY_SERVER_ENVIRONMENT
  ports:
    - protocol: TCP
      port: 80
      targetPort: 3001
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: wavtool-$DEPLOY_SERVER_ENVIRONMENT
  namespace: production
  annotations:
    alb.ingress.kubernetes.io/scheme: internet-facing
    alb.ingress.kubernetes.io/subnets: subnet-0609787300b5d4a9a,subnet-08ede68b609cf7d64
    alb.ingress.kubernetes.io/group.name: wavtool
    alb.ingress.kubernetes.io/target-type: ip
    alb.ingress.kubernetes.io/target-group-attributes: deregistration_delay.timeout_seconds=60
    alb.ingress.kubernetes.io/load-balancer-attributes: access_logs.s3.enabled=true,access_logs.s3.bucket=wavtool-lb-access-logs,access_logs.s3.prefix=wavtool
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'
    alb.ingress.kubernetes.io/healthcheck-port: traffic-port
    alb.ingress.kubernetes.io/healthcheck-path: '/healthz'
    alb.ingress.kubernetes.io/healthcheck-interval-seconds: '5'
    alb.ingress.kubernetes.io/healthcheck-timeout-seconds: '4'
    alb.ingress.kubernetes.io/healthy-threshold-count: '2'
    alb.ingress.kubernetes.io/unhealthy-threshold-count: '2'
    alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:035363416972:certificate/547af941-c779-487e-9f85-64a66a7b5303
spec:
  ingressClassName: alb
  rules:
    - host: "$DEPLOY_SERVER_DOMAIN"
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: wavtool-server-$DEPLOY_SERVER_ENVIRONMENT
                port:
                  number: 80
---
apiVersion: keda.sh/v1alpha1
kind: ScaledJob
metadata:
  name: rvc-trainer-$DEPLOY_SERVER_ENVIRONMENT
  namespace: production
spec:
  jobTargetRef:
    parallelism: 1
    completions: 1
    activeDeadlineSeconds: 3600
    backoffLimit: 3
    template:
      spec:
        restartPolicy: Never
        nodeSelector:
          karpenter.sh/capacity-type: on-demand
          karpenter.k8s.aws/instance-family: g5
        serviceAccountName: composer-service-account
        hostIPC: true
        containers:
        - name: rvc
          image: 035363416972.dkr.ecr.us-east-1.amazonaws.com/minimal:latest
          command: [ "/opt/entrypoint.sh", "--train" ]
          env:
          - name: RVC_MODELS_S3_BUCKET
            value: "wavtool-user-rvc-models"
          - name: RVC_SAMPLES_S3_BUCKET
            value: "samples-test-1"
          - name: RVC_CACHE_DIR
            value: "/tmp/rvc_cache"
          - name: PGDATABASE
            value: "$DEPLOY_DATABASE_NAME"
          - name: PGHOST
            value: "database-2.ctreuprwt6kf.us-east-1.rds.amazonaws.com"
          - name: PGUSER
            value: "voice_conversion"
          - name: PGPASSWORD
            valueFrom:
              secretKeyRef:
                name: rvc-secret
                key: PGPASSWORD
          - name: PG_CONNECTION_STRING
            value: "postgresql://$(PGUSER):$(PGPASSWORD)@$(PGHOST)/$(PGDATABASE)"
          volumeMounts:
          - name: persistent-storage
            mountPath: /opt
            subPath: rvc_production_2023-11-21
          - name: memory-backed-temp
            mountPath: /tmp
          resources:
            limits:
              nvidia.com/gpu: 1
          readinessProbe:
            exec:
              command:
                - cat
                - /trainer_working
            periodSeconds: 5
        volumes:
        - name: persistent-storage
          persistentVolumeClaim:
            claimName: efs-claim
        - name: memory-backed-temp
          emptyDir:
            medium: Memory
            sizeLimit: 4Gi
  pollingInterval: 15
  successfulJobsHistoryLimit: 5
  failedJobsHistoryLimit: 5
  minReplicaCount: 0
  maxReplicaCount: 4
  rollout:
    strategy: gradual
  scalingStrategy:
    # https://github.com/kedacore/keda/pull/1227
    strategy: "accurate"
    pendingPodConditions:
      - "Ready"
      - "PodScheduled"
    multipleScalersCalculation: "sum"
  triggers:
    - type: postgresql
      metadata:
        connectionFromEnv: "PG_CONNECTION_STRING"
        query: "SELECT count(1) FROM voice_conversion_model_training_runs r JOIN voice_conversion_models vc ON r.model_id = vc.id WHERE r.state = 'pending' AND now() - r.created_at <= interval '1 hour' AND NOT vc.archived"
        targetQueryValue: "1"