使用Docker在8GB内存的VPS上部署 Ollama + 小参数LLM

df -h
lscpu
cat /proc/cpuinfo
free -h
cat /proc/meminfo

根据提供的硬件信息，分析是否可以在您的 VPS 上运行 Ollama 和 TinyLlama 模型：

硬件信息摘要

CPU:
 4 个 CPU 核心
 AMD EPYC 7642 48-Core Processor
 2.3 GHz

内存:
 总内存: 7.6 GiB
 可用内存: 5.5 GiB
 Swap: 255 MiB

硬盘:
 总大小: 59G
 可用大小: 31G

分析

CPU
您的 VPS 具有 4 个 CPU 核心，这是一个良好的基础，足以处理一些轻量级的机器学习模型和推理任务。虽然没有 GPU 可能会影响某些深度学习任务的效率，但使用 CPU 仍然是可行的，特别是对于小型模型如 TinyLlama。

内存
TinyLlama 模型在不同量化级别下的内存需求如下：
 TinyLlama: 3.4GB
 TinyLlama:Q6_K: 3.4GB
 TinyLlama:Q8_0: 3.67GB

您的系统总内存为 7.6 GiB，可用内存为 5.5 GiB，这足以运行 TinyLlama 模型。然而，您需要确保在运行模型时其他进程不会消耗过多的内存。此外，建议增加 swap 空间，以防止内存不足的情况。

硬盘
您的硬盘有 31G 的可用空间，这是足够的，因为模型和相关依赖项不会占用太多磁盘空间。TinyLlama 模型大约在 0.5G 到 2G 之间，这对您的可用空间影响不大。

结论
基于以上分析，您的 VPS 应该能够运行 Ollama 和 TinyLlama 模型，前提是适当地管理资源和优化配置。

docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama

docker exec -it ollama ollama run tinyllama

docker exec -it ollama ollama pull phi3

docker exec -it ollama ollama rm tinyllama

docker run -d -p 8380:8080 --add-host=host.docker.internal:host-gateway -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main

docker logs ollama
docker logs open-webui

docker stats ollama
docker stats open-webui

curl -X GET http://{VPS_IP}:11434/api/tags

{
 "models": [{
  "name": "phi3:latest",
  "model": "phi3:latest",
  "modified_at": "2024-08-08T06:06:59.799084299Z",
  "size": 2176178913,
  "digest": "4f222292793889a9a40a020799cfd28d53f3e01af25d48e06c5e708610fc47e9",
  "details": {
   "parent_model": "",
   "format": "gguf",
   "family": "phi3",
   "families": ["phi3"],
   "parameter_size": "3.8B",
   "quantization_level": "Q4_0"
  }
 }]
}

curl -X POST http://{VPS_IP}:11434/api/generate -d '{
  "model": "phi3",
  "prompt": "hi"
}'

{"model":"phi3","created_at":"2024-08-08T07:42:15.076509049Z","response":"hello","done":false}
{"model":"phi3","created_at":"2024-08-08T07:42:15.162735993Z","response":" there","done":false}
{"model":"phi3","created_at":"2024-08-08T07:42:15.245395284Z","response":"!","done":false}
{"model":"phi3","created_at":"2024-08-08T07:42:15.330822219Z","response":" How","done":false}
省略...

docker container stop ollama
docker container rm ollama

docker run -d -v ollama:/root/.ollama -p 127.0.0.1:11434:11434 --name ollama ollama/ollama
docker exec -it ollama ollama pull phi3

docker container ls | grep ollama
23b092ad82f3   ollama/ollama                        "/bin/ollama serve"      20 seconds ago   Up 20 seconds           127.0.0.1:11434->11434/tcp                  ollama

cat config.ini
[DefaultServer]
url = http://host.docker.internal:11434
queue_size = 5

cd ollama_proxy_server
docker build -t ollama_proxy_server:latest .
docker run -d --name ollama-proxy-server -p 8381:8080 --add-host=host.docker.internal:host-gateway ollama_proxy_server:latest

docker logs open-webui
ERROR [apps.ollama.main] Connection error: Cannot connect to host host.docker.internal:11434 ssl:default [Connection refused]

docker inspect -f '{{.NetworkSettings.Networks.bridge.Gateway}}' open-webui
172.17.0.1

再看下容器的hosts
docker exec -it open-webui /bin/bash
root@4ff3c3b6eb8f:/app/backend# cat /etc/hosts
127.0.0.1 localhost
::1 localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
172.17.0.1 host.docker.internal
172.17.0.5 4ff3c3b6eb8f

docker ps -q | xargs -n 1 docker inspect --format '{{.Name}}{{range $k,$v := .NetworkSettings.Networks}} - {{$k}}: {{$v.IPAddress}}{{end}}'
/ollama - bridge: 172.17.0.4
/ollama-proxy-server - bridge: 172.17.0.6
/open-webui - bridge: 172.17.0.5

cat config.ini
[DefaultServer]
url = http://ollama:11434
queue_size = 5

docker build -t ollama_proxy_server:latest .

FROM ollama/ollama
RUN apt-get update && apt-get install -y curl

version: '3.8'

services:
  ollama:
    build: .
    container_name: ollama
    volumes:
      - ollama:/root/.ollama
    expose:
      - "11434"
    networks:
      - ol-internal
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:11434"]
      interval: 10s
      timeout: 5s
      retries: 3

  open-webui:
    image: ghcr.io/open-webui/open-webui:main
    container_name: open-webui
    volumes:
      - open-webui:/app/backend/data
    ports:
      - "8380:8080"
    environment:
      - OLLAMA_BASE_URL=http://ollama:11434
    depends_on:
      ollama:
        condition: service_healthy

    networks:
      - ol-internal
      - ol-external

  ollama-proxy-server:
    image: ollama_proxy_server:latest
    container_name: ollama-proxy-server
    ports:
      - "8381:8080"
    depends_on:
      ollama:
        condition: service_healthy
    networks:
      - ol-internal
      - ol-external

volumes:
  ollama:
  open-webui:

networks:
  ol-internal:
    name: ol-internal
  ol-external:
    name: ol-external

curl -X GET http://{VPS_IP}:11434/api/tags

curl: (7) Failed connect to {VPS_IP}:11434; No route to host

curl -X GET -H "Authorization: Bearer {username}:{secret}" http://{VPS_IP}:8381/api/tags

curl -X POST -H "Authorization: Bearer {username}:{secret}" http://{VPS_IP}:8381/api/generate -d '{"model":"phi3","prompt": "hi"}'

{
 "models": [{
  "name": "phi3:latest",
  "model": "phi3:latest",
  "modified_at": "2024-08-09T06:13:03.416308442Z",
  "size": 2176178913,
  "digest": "4f222292793889a9a40a020799cfd28d53f3e01af25d48e06c5e708610fc47e9",
  "details": {
   "parent_model": "",
   "format": "gguf",
   "family": "phi3",
   "families": ["phi3"],
   "parameter_size": "3.8B",
   "quantization_level": "Q4_0"
  }
 }]
}