Merge pull request #73 from wiseflat/dev/mgarcia/saas-litellm

feat(ansible): Create litellm saas role

Author: wiseflat

ansible/playbooks/saas/roles/litellm/README.md

@@ -0,0 +1 @@
+# Role: `litellm`

ansible/playbooks/saas/roles/litellm/defaults/main.yml

@@ -0,0 +1,23 @@
+---
+# Environments variables
+litellm_env:
+  - key: LITELLM_MASTER_KEY
+    value: sk-1234
+  - key: LITELLM_SALT_KEY
+    value: sk-1234
+  - key: DATABASE_URL
+    value: postgresql://llmproxy:dbpassword9090@db:5432/litellm
+  - key: STORE_MODEL_IN_DB
+    value: true
+  - key: OVHCLOUD_API_KEY
+    value: changeme
+
+# Default configuration
+litellm_config:
+  model_list:
+    - model_name: ovhcloud/gpt-oss-120b
+      litellm_params:
+        model: ovhcloud/gpt-oss-120b
+        api_key: os.environ/OVHCLOUD_API_KEY
+
+litellm_dbhost: localhost

ansible/playbooks/saas/roles/litellm/tasks/backup.yml

@@ -0,0 +1 @@
+---

ansible/playbooks/saas/roles/litellm/tasks/build.yml

@@ -0,0 +1,14 @@
+---
+- name: Include upstream variables
+  ansible.builtin.include_vars: upstream.yml
+
+- name: Set custom variables
+  ansible.builtin.set_fact:
+    image_version: "{{ latest_version }}"
+    image_name: "{{ image.name }}"
+    image_labels: "{{ image.labels }}"
+    image_build: "{{ image.build }}"
+
+- name: End playbook if no new version
+  ansible.builtin.meta: end_host
+  when: softwares[image.name] is defined and softwares[image.name].version == image_version

ansible/playbooks/saas/roles/litellm/tasks/destroy.yml

@@ -0,0 +1,11 @@
+---
+- name: Stop nomad job
+  ansible.builtin.include_role:
+    name: nomad
+    tasks_from: job_stop.yml
+
+- name: Remove software directory
+  ansible.builtin.file:
+    path: "{{ software_path }}"
+    state: absent
+  delegate_to: "{{ software.instance }}"

ansible/playbooks/saas/roles/litellm/tasks/main.yml

@@ -0,0 +1,50 @@
+---
+- name: Install mandatories packages
+  ansible.builtin.apt:
+    pkg:
+      - python3-psycopg2
+
+- name: Create postgresql database
+  community.postgresql.postgresql_db:
+    login_password: "{{ lookup('simple-stack-ui', type='secret', key=software.litellm_dbhost, subkey='passwd', missing='error') }}"
+    login_host: "{{ lookup('simple-stack-ui', type='secret', key=software.litellm_dbhost, subkey='service_name', missing='error') }}"
+    login_user: postgres
+    name: "{{ service_name }}"
+    encoding: UTF8
+    state: present
+
+- name: Create postgresql user
+  community.postgresql.postgresql_user:
+    login_password: "{{ lookup('simple-stack-ui', type='secret', key=software.litellm_dbhost, subkey='passwd', missing='error') }}"
+    login_host: "{{ lookup('simple-stack-ui', type='secret', key=software.litellm_dbhost, subkey='service_name', missing='error') }}"
+    login_user: postgres
+    login_db: "{{ service_name }}"
+    name: "{{ service_name }}"
+    password: "{{ lookup('simple-stack-ui', type='secret', key=domain, subkey='litellm_dbpasswd', missing='create', length=12) }}"
+    state: present
+
+- name: Create postgresql privileges
+  community.postgresql.postgresql_privs:
+    login_password: "{{ lookup('simple-stack-ui', type='secret', key=software.litellm_dbhost, subkey='passwd', missing='error') }}"
+    login_host: "{{ lookup('simple-stack-ui', type='secret', key=software.litellm_dbhost, subkey='service_name', missing='error') }}"
+    login_user: postgres
+    db: "{{ service_name }}"
+    roles: "{{ service_name }}"
+    type: schema
+    objs: public
+    privs: CREATE
+    state: present
+
+- name: Copy nomad job
+  ansible.builtin.template:
+    src: nomad.hcl
+    dest: "/var/tmp/{{ domain }}.nomad"
+    owner: root
+    group: root
+    mode: '0600'
+  become: true
+
+- name: Run nomad job
+  ansible.builtin.include_role:
+    name: nomad
+    tasks_from: job_run.yml

ansible/playbooks/saas/roles/litellm/tasks/restore.yml

@@ -0,0 +1 @@
+---

ansible/playbooks/saas/roles/litellm/templates/nomad.hcl

@@ -0,0 +1,78 @@
+job "{{ domain }}" {
+  region = "{{ fact_instance.region }}"
+  datacenters = ["{{ fact_instance.datacenter }}"]
+  type = "service"
+
+{% if software.constraints is defined and software.constraints.location is defined %}
+  constraint {
+    attribute    = "${meta.location}"
+    set_contains = "{{ software.constraints.location }}"
+  }
+{% endif %}
+
+  constraint {
+    attribute    = "${meta.instance}"
+    set_contains = "{{ software.instance }}"
+  }
+
+  group "{{ domain }}" {
+    count = 1
+
+    network {
+      port "litellm" {
+        to = 4000
+{% if software.static_port is defined %}
+        static = {{ software.static_port }}
+{% endif %}
+      }
+    }
+
+    service {
+      name = "{{ service_name }}"
+      port = "litellm"
+      provider = "nomad"
+      tags = [
+        {{ lookup('template', '../../traefik/templates/traefik_tag.j2') | indent(8) }}
+      ]
+      check {
+        type     = "http"
+        path     = "/health/liveliness"
+        interval = "30s"
+        timeout  = "10s"
+      }
+    }
+
+    task "{{ domain }}-milvus" {
+      driver = "docker"
+
+      template {
+        change_mode = "restart"
+        destination = "local/config.yaml"
+        perms = "644"
+        data = <<EOH
+{{ (software.litellm_config | to_nice_yaml) | default(litellm_config) }}
+  EOH
+      }
+
+      config {
+        image = "ghcr.io/berriai/litellm:v{{ softwares.litellm.version }}"
+        ports = ["litellm"]
+        args = [
+          "--config=/local/config.yaml"
+        ]
+      }
+
+      env {
+{% for env in (lookup('simple-stack-ui', type='secret', key=domain, subkey='litellm_env', missing='warn') | from_json) | default(litellm_env) %}
+       {{ env.key }} = "{{ env.value }}"
+{% endfor %}
+      }
+
+      resources {
+        cpu    = {{ size[software.size].cpu }}
+        memory = {{ size[software.size].memory }}
+        memory_max = {{ size[software.size].memory * 2 }}
+      }
+    }
+  }
+}

ansible/playbooks/saas/roles/litellm/vars/main.yml

@@ -0,0 +1,10 @@
+---
+image:
+  build: false
+  upstream:
+    source: github
+    user: BerriAI
+    repo: litellm
+    os: linux
+  labels: {}
+  name: litellm

ansible/playbooks/saas/roles/litellm/vars/upstream.yml

@@ -0,0 +1,3 @@
+---
+upstream_versions: "{{ (lookup('url', 'https://api.github.com/repos/' + image.upstream.user + '/' + image.upstream.repo + '/tags', headers={'Accept': 'application/vnd.github+json', 'Authorization': 'Bearer ' + lookup('ansible.builtin.env', 'GITHUB_API_TOKEN') })) }}"
+latest_version: "{{ upstream_versions | community.general.json_query('[].name') | select('match', '^(v|r)?\\d{1,4}(\\.\\d{1,4})(\\.\\d{1,4})(-stable)$') | community.general.version_sort | last | replace('v', '') }}"