安装驱动
sudo ubuntu-drivers autoinstall
或手动安装
sudo apt install -y nvidia-driver-550 nvidia-utils-550
naidia-smi
进入容器
docker exec -it ollama bash
容器内下载7b模型
ollama pull qwen2:7b
拉去一个支持嵌入的模型
ollama pull nomic-embed-text
安装工具让容器可以用gpu
下载
https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/
nvidia-container-toolkit_1.16.1-1_amd64.deb
libnvidia-container1_1.16.1-1_amd64.deb
libnvidia-container-tools_1.16.1-1_amd64.deb
单独下载
https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/nvidia-container-toolkit-base_1.16.1-1_amd64.deb
nvidia-container-toolkit-base_1.16.1-1_amd64.deb
以上四个包
安装
sudo dpkg -i nvidia-container-toolkit-base_1.16.1-1_amd64.deb
sudo dpkg -i libnvidia-container1_*.deb
sudo dpkg -i libnvidia-container-tools_*.deb
sudo dpkg -i nvidia-container-toolkit_*.deb
配置容器使用gpu
sudo nvidia-ctk runtime configure --runtime=docker
重启docker
sudo systemctl restart docker
测试ollama
curl
配置compose
services:
ollama:
image: registry.cn-hangzhou.aliyuncs.com/leiuvn/ai:ollama-0.12.10
container_name: ollama
restart: unless-stopped
ports:
- "8011:11434"
volumes:
- /data/wwwroot/ai/ollama_data:/root/.ollama
- /data/wwwroot/ai/ollama_models:/root/.ollama/models
networks:
lbu_net:
ipv4_address: 172.20.0.27
mem_limit: 8g
cpus: 8.0
runtime: nvidia
environment:
- NVIDIA_VISIBLE_DEVICES=all
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:11434/api/tags"]
interval: 30s
timeout: 5s
retries: 3
open-webui:
image: registry.cn-hangzhou.aliyuncs.com/leiuvn/ai:open-webui-0.3.35-ollama
container_name: open-webui
restart: unless-stopped
ports:
- "8012:8080"
environment:
- OLLAMA_BASE_URL=http://ollama:11434
- ENABLE_OLLAMA_EMBEDDINGS=true
- OLLAMA_EMBEDDING_MODEL=nomic-embed-text # 指定嵌入模型
- DEFAULT_MODELS=qwen2:7b
- HF_ENDPOINT=https://hf-mirror.com
volumes:
- /data/wwwroot/ai/open_webui_data:/app/backend/data
depends_on:
- ollama
networks:
lbu_net:
ipv4_address: 172.20.0.28
mem_limit: 8g
cpus: 8.0
networks:
lbu_net:
external: true
手动下载模型
mkdir -p /data/wwwroot/ai/ollama_models/blobs
mkdir -p /data/wwwroot/ai/ollama_models/manifests/registry.ollama.ai/library/qwen2/7b
cd /data/wwwroot/ai/ollama_models/blobs
wget https://huggingface.co/Qwen/Qwen2-7B-Instruct-GGUF/resolve/main/qwen2-7b-instruct-q4_0.gguf
SHA=$(sha256sum qwen2-7b-instruct-q4_0.gguf | awk '{print $1}')
mv qwen2-7b-instruct-q4_0.gguf sha256:$SHA
cat > /data/wwwroot/ai/ollama_models/manifests/registry.ollama.ai/library/qwen2/7b/Modelfile << 'EOF'
FROM /data/wwwroot/ai/ollama_models/blobs/sha256:{{SHA}}
TEMPLATE "{{ .Prompt }}"
EOF
sed -i "s/{{SHA}}/$SHA/" /data/wwwroot/ai/ollama_models/manifests/registry.ollama.ai/library/qwen2/7b/Modelfile
docker run --rm -it \
-v /data/wwwroot/ai/ollama_models:/root/.ollama/models \
registry.cn-hangzhou.aliyuncs.com/leiuvn/ai:ollama-0.12.10 \
ollama create qwen2:7b -f /root/.ollama/models/manifests/registry.ollama.ai/library/qwen2/7b/Modelfile
微调模型
第一步
# 1. 下载模型https://ollama.com/library
docker run --rm -v /data/wwwroot/ai/ollama_models:/root/.ollama/models \
registry.cn-hangzhou.aliyuncs.com/leiuvn/ai:ollama-0.12.10 \
ollama pull qwen2:7b
ollama pull deepseek-r1:7b
# 2. 安装转换工具
pip install huggingface-hub transformers gguf
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp
pip install -r requirements.txt
# 3. 找到 GGUF 文件路径（宿主机）
ls /data/wwwroot/ai/ollama_models/blobs/sha256*
# 假设输出：sha256:1a2b3c...
# 4. 转为 HuggingFace 格式（用于 PyTorch）
python convert-gguf-to-hf.py \
--gguf /data/wwwroot/ai/ollama_models/blobs/sha256:1a2b3c... \
--output-dir /data/wwwroot/ai/hf_models/qwen2-7b-base
第二步
# 安装 Unsloth（支持 Qwen2，显存省 70%）
pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
pip install --no-deps trl peft accelerate bitsandbytes
# finetune.py
from unsloth import FastLanguageModel
import torch
from datasets import load_dataset
# 1. 加载模型
model, tokenizer = FastLanguageModel.from_pretrained(
"/data/wwwroot/ai/hf_models/qwen2-7b-base",
dtype=None,
load_in_4bit=True,
)
# 2. 添加 LoRA
model = FastLanguageModel.get_peft_model(
model,
r=16,
target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
lora_alpha=16,
lora_dropout=0,
bias="none",
use_gradient_checkpointing="unsloth",
)
# 3. 准备数据
dataset = load_dataset("json", data_files="/data/wwwroot/ai/finetune_data/train.jsonl", split="train")
# 4. 训练
from trl import SFTTrainer
trainer = SFTTrainer(
model=model,
tokenizer=tokenizer,
train_dataset=dataset,
dataset_text_field="prompt",
max_seq_length=2048,
args=TrainingArguments(
per_device_train_batch_size=2,
gradient_accumulation_steps=4,
warmup_steps=5,
max_steps=60,
learning_rate=2e-4,
fp16=not torch.cuda.is_bf16_supported(),
bf16=torch.cuda.is_bf16_supported(),
logging_steps=1,
output_dir="/data/wwwroot/ai/ft_output",
optim="adamw_8bit",
),
)
trainer.train()
# 5. 保存
model.save_pretrained("/data/wwwroot/ai/ft_output")
tokenizer.save_pretrained("/data/wwwroot/ai/ft_output")
第三步
cd llama.cpp
# 1. 转为 GGUF
python convert-hf-to-gguf.py /data/wwwroot/ai/ft_output --outfile qwen2-7b-ft.gguf
# 2. 量化（推荐 Q4_K_M，平衡）
./quantize qwen2-7b-ft.gguf qwen2-7b-ft-q4.gguf Q4_K_M
第四步
# 1. 计算 SHA256
SHA=$(sha256sum qwen2-7b-ft-q4.gguf | cut -d' ' -f1)
# 2. 放入 blobs
mkdir -p /data/wwwroot/ai/ollama_models/blobs
cp qwen2-7b-ft-q4.gguf /data/wwwroot/ai/ollama_models/blobs/sha256:$SHA
# 3. 创建 Modelfile
mkdir -p /data/wwwroot/ai/ollama_models/manifests/registry.ollama.ai/library/qwen2/7b-ft
cat > /data/wwwroot/ai/ollama_models/manifests/registry.ollama.ai/library/qwen2/7b-ft/Modelfile << EOF
FROM ./blobs/sha256:$SHA
PARAMETER num_ctx 4096
PARAMETER num_batch 4
PARAMETER num_gqa 8
EOF
# 4. 注册模型
docker run --rm -it \
-v /data/wwwroot/ai/ollama_models:/root/.ollama/models \
registry.cn-hangzhou.aliyuncs.com/leiuvn/ai:ollama-0.12.10 \
ollama create qwen2:7b-ft -f /root/.ollama/models/manifests/registry.ollama.ai/library/qwen2/7b-ft/Modelfile