Windows上玩转本地大模型：AI Max 395 + Ollama 从零到精通

# 检查系统识别到的GPU内存
(Get-CimInstance -ClassName Win32_VideoController | Where-Object {$_.Name -like "*Radeon*"}).AdapterRAM / 1GB

# 设置环境变量启用ROCm（如果自动检测失败）
[Environment]::SetEnvironmentVariable("OLLAMA_ROCM", "1", "User")

# 设置为高性能模式
powercfg /setactive 8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c

# 或者通过GUI：控制面板 → 电源选项 → 高性能

# 打开新的PowerShell窗口（必须新开窗口以加载环境变量）
ollama --version
# 应显示类似：ollama version 0.5.x

# 在管理员CMD/PowerShell中运行
OllamaSetup.exe /DIR="D:\Ollama"

# 设置环境变量
[Environment]::SetEnvironmentVariable("OLLAMA_MODELS", "D:\Ollama\Models", "User")

# 立即生效（当前会话）
$env:OLLAMA_MODELS = "D:\Ollama\Models"

# 安装NSSM（Non-Sucking Service Manager）
# 然后注册Ollama服务
nssm install Ollama "D:\Ollama\CLI\ollama.exe" serve
nssm set Ollama AppDirectory "D:\Ollama\CLI"
nssm set Ollama AppEnvironmentExtra "OLLAMA_MODELS=D:\Ollama\Models"
nssm start Ollama

# 方式1：GUI模式（开始菜单启动，会在系统托盘驻留）
# 方式2：命令行模式（适合调试）
ollama serve

# 下载Phi-3 Mini（2.3GB，适合快速测试）
ollama pull phi3:mini

# 运行测试
ollama run phi3:mini "Write a hello world in Python"

# 拉取模型（约40GB，根据网速需10-30分钟）
ollama pull llama3.3:70b

# 交互式运行
ollama run llama3.3:70b

FROM llama3.3:70b

# 设置上下文长度（根据任务调整，越长占用显存越多）
PARAMETER num_ctx 8192

# 批处理大小（影响延迟vs吞吐量平衡）
PARAMETER num_batch 512

# 温度设置
PARAMETER temperature 0.7

# 系统提示词
SYSTEM """你是一个专业的AI助手，请提供准确、有帮助的回答。"""

ollama create llama3.3-70b-fast -f D:\Ollama\Modelfiles\llama3.3-70b-optimized.modelfile

FROM ./gpt-oss-120b-q4.gguf

# 降低上下文长度以节省K/V缓存显存
PARAMETER num_ctx 4096

# 启用Flash Attention（减少显存占用）
PARAMETER flash_attn true

# 使用较小的批处理
PARAMETER num_batch 256

# 系统提示
SYSTEM """你是一个强大的大语言模型，能够处理复杂任务。请逐步思考并提供详细回答。"""

ollama create gpt-oss-120b-local -f .\gpt-oss-120b.modelfile
ollama run gpt-oss-120b-local

amdgpu: GPU reset begin!
VRAM is lost due to GPU reset!

# 在后台保持模型常驻（避免重复加载）
# 方法：通过API保持心跳

# 查看当前加载的模型
ollama ps

# 释放特定模型
ollama stop llama3.3:70b

# 清理所有模型缓存（遇到显存泄漏时）
ollama ps | ForEach-Object { ollama stop $_.NAME }

# 预加载常用模型到显存
$models = @("llama3.2", "llama3.3:70b", "qwen2.5:72b")
foreach ($model in $models) {
    Write-Host "Preloading $model..."
    # 发送空请求预热模型
    $body = @{model=$model; prompt="Hi"; stream=$false} | ConvertTo-Json
    Invoke-RestMethod -Uri "http://localhost:11434/api/generate" -Method Post -Body $body -ContentType "application/json" | Out-Null
}
Write-Host "All models preloaded into VRAM"

K/V缓存显存 ≈ 2 × num_layers × num_heads × head_dim × seq_len × batch_size × bytes_per_token

# 永久设置（推荐）
[Environment]::SetEnvironmentVariable("OLLAMA_KV_CACHE_TYPE", "q8_0", "User")

# 立即生效（当前PowerShell会话）
$env:OLLAMA_KV_CACHE_TYPE = "q8_0"

[Environment]::SetEnvironmentVariable("OLLAMA_KV_CACHE_TYPE", "q4_0", "User")

PARAMETER flash_attn true

# 将长文档分段，逐段处理
$chunks = Get-Content "long-document.txt" -Raw -Split "####"
foreach ($chunk in $chunks) {
    $body = @{
        model = "llama3.3:70b"
        prompt = "Summarize this section: $chunk"
        stream = $false
        options = @{
            num_ctx = 4096  # 每块使用较短上下文
        }
    } | ConvertTo-Json
    $summary = Invoke-RestMethod -Uri "http://localhost:11434/api/generate" -Method Post -Body $body
    Add-Content "summaries.txt" $summary.response
}

# 以管理员运行注册表编辑器
reg add "HKLM\SYSTEM\CurrentControlSet\Control\GraphicsDrivers" /v TdrLevel /t REG_DWORD /d 0 /f
reg add "HKLM\SYSTEM\CurrentControlSet\Control\GraphicsDrivers" /v TdrDelay /t REG_DWORD /d 60 /f

# 重启生效
Restart-Computer

# 注册表编辑（管理员）
reg add "HKLM\SYSTEM\CurrentControlSet\Control\Class\{4d36e968-e325-11ce-bfc1-08002be10318}\0000" /v EnableUlps /t REG_DWORD /d 0 /f

# 在Modelfile中
FROM llama3.3:70b

# 强制部分层在CPU（数字越大，越多层在GPU）
# 需要实验确定稳定值
PARAMETER num_gpu 60  # 假设70B模型有80层，60层在GPU，20层在CPU

# 设置环境变量
[Environment]::SetEnvironmentVariable("OLLAMA_NO_MMAP", "1", "User")

# 查看最新服务器日志
Get-Content "$env:LOCALAPPDATA\Ollama\server.log" -Tail 50

# 查看应用日志
Get-Content "$env:LOCALAPPDATA\Ollama\app.log" -Tail 30

# 设置调试日志
[Environment]::SetEnvironmentVariable("OLLAMA_DEBUG", "1", "User")

# 重新启动Ollama并观察详细输出
ollama serve 2>&1 | Tee-Object -FilePath "ollama-debug.log"

$headers = @{ "Content-Type" = "application/json" }
$body = @{
    model = "llama3.3:70b"
    prompt = "Explain quantum computing in simple terms"
    stream = $false
    options = @{
        temperature = 0.7
        num_predict = 500  # 最大生成token数
        top_p = 0.9
        stop = @("Human:", "Assistant:")
    }
} | ConvertTo-Json -Depth 3

$response = Invoke-RestMethod -Uri "http://localhost:11434/api/generate" -Method Post -Headers $headers -Body $body
$response.response

# 批量处理问答对
$inputFile = "questions.jsonl"
$outputFile = "answers.jsonl"

Get-Content $inputFile | ForEach-Object {
    $item = $_ | ConvertFrom-Json
    $body = @{
        model = "llama3.3:70b"
        prompt = $item.question
        stream = $false
    } | ConvertTo-Json
    
    $result = Invoke-RestMethod -Uri "http://localhost:11434/api/generate" -Method Post -Body $body -ContentType "application/json"
    
    $output = @{
        id = $item.id
        question = $item.question
        answer = $result.response
        eval_count = $result.eval_count
        eval_duration = $result.eval_duration
    } | ConvertTo-Json -Compress
    
    Add-Content $outputFile $output
}

docker run -d -p 3000:8080 `
  --add-host=host.docker.internal:host-gateway `
  -v open-webui:/app/backend/data `
  --name open-webui `
  --restart always `
  ghcr.io/open-webui/open-webui:main

{
  "models": [
    {
      "title": "Local Llama 3.3 70B",
      "provider": "ollama",
      "model": "llama3.3:70b",
      "apiBase": "http://localhost:11434"
    },
    {
      "title": "Local Code Specialist",
      "provider": "ollama",
      "model": "codellama:34b",
      "apiBase": "http://localhost:11434"
    }
  ]
}

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:11434/v1",
    api_key="ollama"  # 必填但可任意值
)

response = client.chat.completions.create(
    model="llama3.3:70b",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Write a Python script to analyze CSV data"}
    ],
    temperature=0.7,
    max_tokens=2000
)

print(response.choices[0].message.content)

from langchain_ollama import OllamaLLM
from langchain.chains import LLMChain
from langchain.prompts import PromptTemplate

llm = OllamaLLM(
    model="llama3.3:70b",
    base_url="http://localhost:11434",
    temperature=0.7,
    num_ctx=8192
)

template = """Question: {question}
Answer: Let's think step by step."""

prompt = PromptTemplate(template=template, input_variables=["question"])
chain = LLMChain(llm=llm, prompt=prompt)

result = chain.predict(question="What is the meaning of life?")
print(result)

# 监控文件夹，自动分类新文档
$watchPath = "D:\Documents\Incoming"
$model = "llama3.3:70b"

$watcher = New-Object System.IO.FileSystemWatcher
$watcher.Path = $watchPath
$watcher.Filter = "*.txt"
$watcher.Created = {
    $file = $Event.SourceEventArgs.FullPath
    $content = Get-Content $file -Raw
    
    $prompt = "Read this document and classify it into one of: Finance, Legal, Technical, Other. Respond with just the category name:`n`n$content"
    
    $body = @{
        model = $model
        prompt = $prompt
        stream = $false
        options = @{ num_predict = 50 }
    } | ConvertTo-Json
    
    $result = Invoke-RestMethod -Uri "http://localhost:11434/api/generate" -Method Post -Body $body -ContentType "application/json"
    $category = $result.response.Trim()
    
    # 移动文件到对应文件夹
    $dest = Join-Path $watchPath "..\$category"
    if (-not (Test-Path $dest)) { New-Item -ItemType Directory -Path $dest }
    Move-Item $file $dest -Force
    
    Write-Host "Classified $file as $category"
}

Register-ObjectEvent $watcher Created -Action $watcher.Created
Write-Host "Monitoring $watchPath for new documents..."
while ($true) { Start-Sleep -Seconds 1 }

# 手动添加PATH（根据实际安装路径调整）
$oldPath = [Environment]::GetEnvironmentVariable("Path", "User")
$newPath = $oldPath + ";C:\Users\<用户名>\AppData\Local\Programs\Ollama"
[Environment]::SetEnvironmentVariable("Path", $newPath, "User")
# 重启PowerShell

# 查找占用端口的进程
netstat -ano | findstr :11434
# 结束进程（将<PID>替换为实际进程ID）
taskkill /PID <PID> /F

# 从HuggingFace手动下载GGUF到D:\Models
# 创建Modelfile:
# FROM D:\Models\model.gguf
# 然后:
ollama create local-model -f D:\Models\Modelfile

# 检查日志
Get-Content "$env:LOCALAPPDATA\Ollama\server.log" | Select-String "rocm\|gpu\|cuda"

# 强制启用ROCm
$env:OLLAMA_ROCM = "1"
ollama serve

# 监控GPU使用率和显存（需要GPU-Z或内置计数器）
while ($true) {
    $gpu = Get-Counter "\GPU Engine(*)\Utilization Percentage" | Select-Object -ExpandProperty CounterSamples
    $vram = Get-Counter "\GPU Process Memory(*)\Local Usage" | Select-Object -ExpandProperty CounterSamples
    Write-Host "GPU: $($gpu | Where-Object {$_.InstanceName -like '*engtype_3D*'} | Select-Object -ExpandProperty CookedValue)%"
    Write-Host "VRAM: $([math]::Round($vram[0].CookedValue / 1GB, 2)) GB"
    Start-Sleep -Seconds 2
}

# 快速诊断
ollama --version                    # 查看版本
ollama list                         # 列出模型
ollama ps                           # 查看运行中模型
Get-Content "$env:LOCALAPPDATA\Ollama\server.log" -Tail 20  # 查看日志

# 环境变量设置
[Environment]::SetEnvironmentVariable("OLLAMA_MODELS", "D:\Ollama\Models", "User")
[Environment]::SetEnvironmentVariable("OLLAMA_KV_CACHE_TYPE", "q8_0", "User")
[Environment]::SetEnvironmentVariable("OLLAMA_ROCM", "1", "User")
[Environment]::SetEnvironmentVariable("OLLAMA_DEBUG", "1", "User")

# 模型管理
ollama pull llama3.3:70b           # 下载模型
ollama run llama3.3:70b            # 运行模型
ollama rm llama3.3:70b             # 删除模型
ollama stop llama3.3:70b            # 停止模型（释放显存）

# API测试
Invoke-RestMethod -Uri "http://localhost:11434/api/tags" -Method Get  # 列出模型