第9讲：部署与运维——把你的 AI 应用送上生产环境

一、整体部署架构┌──────────────────────────────────────────────────────────────┐ │ 用户层 │ │ 浏览器 ──→ CDN ──→ DNS ──→ 负载均衡器 │ │ │ │ ├──────────────────────────────┼───────────────────────────────┤ │ ▼ │ │ Kubernetes 集群 │ │ │ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ │ LLM 网关 │ │ RAG 服务 │ │ FC 服务 │ │ 前端 │ │ │ │ (2-4 副本)│ │ (2-4 副本)│ │ (2-4 副本)│ │ (2 副本) │ │ │ └────┬─────┘ └────┬─────┘ └────┬─────┘ └──────────┘ │ │ │ │ │ │ ├───────┴──────────────┴──────────────┴────────────────────────┤ │ 数据层 │ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ │ Redis │ │ PostgreSQL│ │ Milvus │ │ MinIO │ │ │ │ (缓存) │ │ (业务数据)│ │ (向量库) │ │ (对象存储)│ │ │ └──────────┘ └──────────┘ └──────────┘ └──────────┘ │ │ │ ├──────────────────────────────────────────────────────────────┤ │ 监控层 │ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ │ Prometheus│ │ Grafana │ │ Loki │ │ Jaeger │ │ │ │ (指标) │ │ (看板) │ │ (日志) │ │ (链路) │ │ │ └──────────┘ └──────────┘ └──────────┘ └──────────┘ │ └──────────────────────────────────────────────────────────────┘二、容器化与 Dockerfile# deploy/Dockerfile.gateway FROM golang:1.22-alpine AS builder WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . . RUN CGO_ENABLED0 GOOSlinux go build -ldflags-s -w \ -o /app/gateway ./cmd/gateway # ─── 运行时镜像 ─── FROM alpine:3.19 RUN apk add --no-cache ca-certificates tzdata curl # 非 root 用户运行 RUN addgroup -S app adduser -S app -G app USER app WORKDIR /app COPY --frombuilder /app/gateway . EXPOSE 8080 HEALTHCHECK --interval10s --timeout3s --retries3 \ CMD curl -f http://localhost:8080/health || exit 1 ENTRYPOINT [./gateway] # ─── 多阶段构建说明 ─── # 构建docker build -t llm-gateway:latest -f deploy/Dockerfile.gateway . # 运行docker run -p 8080:8080 llm-gateway:latest# deploy/Dockerfile.rag FROM python:3.11-slim AS builder WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY . . # ─── 运行时镜像 ─── FROM python:3.11-slim RUN groupadd -r app useradd -r -g app app USER app WORKDIR /app COPY --frombuilder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages COPY --frombuilder /app . EXPOSE 9090 HEALTHCHECK --interval10s --timeout3s --retries3 \ CMD python -c import urllib.request; urllib.request.urlopen(http://localhost:9090/health) CMD [uvicorn, main:app, --host, 0.0.0.0, --port, 9090]三、Kubernetes 部署清单# deploy/k8s/namespace.yaml apiVersion: v1 kind: Namespace metadata: name: ai-platform --- # deploy/k8s/configmap.yaml apiVersion: v1 kind: ConfigMap metadata: name: gateway-config namespace: ai-platform data: config.yaml: | server: port: 8080 read_timeout: 120s write_timeout: 300s llm: providers: openai: model: gpt-4 timeout: 60s max_retries: 3 claude: model: claude-3-sonnet timeout: 90s redis: address: redis-service:6379 pool_size: 100 rate_limit: anonymous: 5 basic: 20 premium: 100 --- # deploy/k8s/secret.yaml apiVersion: v1 kind: Secret metadata: name: gateway-secrets namespace: ai-platform type: Opaque stringData: OPENAI_API_KEY: sk-your-key-here CLAUDE_API_KEY: sk-ant-your-key-here JWT_SECRET: your-jwt-secret-here DATABASE_URL: postgresql://user:passwordpostgres-service:5432/aiplatform --- # deploy/k8s/deployment-gateway.yaml apiVersion: apps/v1 kind: Deployment metadata: name: llm-gateway namespace: ai-platform spec: replicas: 3 strategy: type: RollingUpdate rollingUpdate: maxSurge: 1 maxUnavailable: 0 selector: matchLabels: app: llm-gateway template: metadata: labels: app: llm-gateway annotations: prometheus.io/scrape: true prometheus.io/port: 8080 prometheus.io/path: /metrics spec: affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: app operator: In values: - llm-gateway topologyKey: kubernetes.io/hostname containers: - name: gateway image: registry.example.com/llm-gateway:v1.0.0 imagePullPolicy: Always ports: - containerPort: 8080 protocol: TCP envFrom: - secretRef: name: gateway-secrets volumeMounts: - name: config mountPath: /app/config readOnly: true resources: requests: cpu: 500m memory: 512Mi limits: cpu: 2000m memory: 2048Mi livenessProbe: httpGet: path: /health port: 8080 initialDelaySeconds: 30 periodSeconds: 10 readinessProbe: httpGet: path: /ready port: 8080 initialDelaySeconds: 5 periodSeconds: 5 startupProbe: httpGet: path: /health port: 8080 failureThreshold: 30 periodSeconds: 10 volumes: - name: config configMap: name: gateway-config --- # deploy/k8s/service-gateway.yaml apiVersion: v1 kind: Service metadata: name: llm-gateway-service namespace: ai-platform spec: type: ClusterIP ports: - port: 80 targetPort: 8080 protocol: TCP name: http selector: app: llm-gateway --- # deploy/k8s/hpa-gateway.yaml apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: llm-gateway-hpa namespace: ai-platform spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: llm-gateway minReplicas: 2 maxReplicas: 20 metrics: - type: Resource resource: name: cpu target: type: Utilization averageUtilization: 70 - type: Resource resource: name: memory target: type: Utilization averageUtilization: 75 behavior: scaleDown: stabilizationWindowSeconds: 300 policies: - type: Percent value: 10 periodSeconds: 60 scaleUp: stabilizationWindowSeconds: 0 policies: - type: Pods value: 4 periodSeconds: 60 - type: Percent value: 100 periodSeconds: 60 --- # deploy/k8s/ingress.yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: ai-platform-ingress namespace: ai-platform annotations: nginx.ingress.kubernetes.io/proxy-read-timeout: 300 nginx.ingress.kubernetes.io/proxy-send-timeout: 300 nginx.ingress.kubernetes.io/proxy-buffering: off nginx.ingress.kubernetes.io/proxy-buffer-size: 128k cert-manager.io/cluster-issuer: letsencrypt-prod spec: ingressClassName: nginx tls: - hosts: - api.aiplatform.com secretName: ai-platform-tls rules: - host: api.aiplatform.com http: paths: - path: /v1/chat pathType: Prefix backend: service: name: llm-gateway-service port: number: 80 - path: /v1/rag pathType: Prefix backend: service: name: rag-service port: number: 80四、CI/CD 流水线# .github/workflows/deploy.yml name: Deploy AI Platform on: push: branches: [main, release/*] pull_request: branches: [main] env: REGISTRY: registry.example.com CLUSTER_NAME: ai-platform-prod CLUSTER_ZONE: asia-east1-a jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkoutv4 - name: Setup Go uses: actions/setup-gov5 with: go-version: 1.22 - name: Run unit tests run: go test ./... -v -coverprofilecoverage.out - name: Run integration tests run: go test ./tests/... -v -tagsintegration - name: Security scan uses: aquasecurity/trivy-actionmaster with: scan-type: fs severity: CRITICAL,HIGH build-and-push: needs: test runs-on: ubuntu-latest steps: - uses: actions/checkoutv4 - name: Set up Docker Buildx uses: docker/setup-buildx-actionv3 - name: Login to Registry uses: docker/login-actionv3 with: registry: ${{ env.REGISTRY }} username: ${{ secrets.REGISTRY_USERNAME }} password: ${{ secrets.REGISTRY_PASSWORD }} - name: Extract version id: version run: echo tag$(git rev-parse --short HEAD) $GITHUB_OUTPUT - name: Build and push gateway uses: docker/build-push-actionv5 with: context: . file: deploy/Dockerfile.gateway push: true tags: | ${{ env.REGISTRY }}/llm-gateway:${{ steps.version.outputs.tag }} ${{ env.REGISTRY }}/llm-gateway:latest cache-from: typegha cache-to: typegha,modemax - name: Scan image uses: aquasecurity/trivy-actionmaster with: image-ref: ${{ env.REGISTRY }}/llm-gateway:${{ steps.version.outputs.tag }} severity: CRITICAL,HIGH deploy-staging: needs: build-and-push runs-on: ubuntu-latest environment: staging steps: - uses: actions/checkoutv4 - name: Configure kubectl uses: google-github-actions/get-gke-credentialsv2 with: cluster_name: ${{ env.CLUSTER_NAME }}-staging location: ${{ env.CLUSTER_ZONE }} - name: Update image tag run: | sed -i s|image:.*|image: ${{ env.REGISTRY }}/llm-gateway:${{ steps.version.outputs.tag }}| \ deploy/k8s/deployment-gateway.yaml - name: Deploy to staging run: | kubectl apply -f deploy/k8s/ -n ai-platform-staging kubectl rollout status deployment/llm-gateway -n ai-platform-staging --timeout5m - name: Smoke test run: | sleep 30 curl -f https://staging.aiplatform.com/health curl -f -X POST https://staging.aiplatform.com/v1/chat/stream \ -H Content-Type: application/json \ -d {messages:[{role:user,content:ping}]} \ --max-time 30 deploy-production: needs: deploy-staging runs-on: ubuntu-latest environment: production steps: - uses: actions/checkoutv4 - name: Configure kubectl uses: google-github-actions/get-gke-credentialsv2 with: cluster_name: ${{ env.CLUSTER_NAME }} location: ${{ env.CLUSTER_ZONE }} - name: Canary deploy (10%) run: | kubectl set image deployment/llm-gateway-canary \ gateway${{ env.REGISTRY }}/llm-gateway:${{ steps.version.outputs.tag }} \ -n ai-platform kubectl scale deployment/llm-gateway-canary --replicas1 -n ai-platform - name: Wait and monitor canary run: | echo Monitoring canary for 5 minutes... sleep 300 # 检查错误率 ERROR_RATE$(curl -s http://prometheus:9090/api/v1/query?queryrate(llm_requests_failed[5m])/rate(llm_requests_total[5m]) | jq .data.result[0].value[1] | tr -d ) if (( $(echo $ERROR_RATE 0.01 | bc -l) )); then echo Error rate too high: $ERROR_RATE, rolling back kubectl scale deployment/llm-gateway-canary --replicas0 -n ai-platform exit 1 fi - name: Full rollout run: | kubectl set image deployment/llm-gateway \ gateway${{ env.REGISTRY }}/llm-gateway:${{ steps.version.outputs.tag }} \ -n ai-platform kubectl rollout status deployment/llm-gateway -n ai-platform --timeout10m kubectl scale deployment/llm-gateway-canary --replicas0 -n ai-platform - name: Post-deploy verification run: | sleep 60 kubectl run smoke-test --imagecurlimages/curl --rm -it --restartNever \ -- -f https://api.aiplatform.com/health五、配置管理与环境差异化// deploy/config.go package deploy import ( fmt os strconv time ) // Config 应用配置 type Config struct { Env string // development/staging/production Version string Server ServerConfig LLM LLMConfig Redis RedisConfig Database DatabaseConfig Auth AuthConfig Monitor MonitorConfig } type ServerConfig struct { Port int ReadTimeout time.Duration WriteTimeout time.Duration GracePeriod time.Duration } type LLMConfig struct { DefaultProvider string Providers map[string]ProviderConfig CacheEnabled bool CacheTTL time.Duration } type ProviderConfig struct { Model string APIKey string BaseURL string Timeout time.Duration MaxRetries int } type RedisConfig struct { Address string Password string PoolSize int } type DatabaseConfig struct { URL string MaxConnections int MaxIdleConns int ConnMaxLifetime time.Duration } type AuthConfig struct { JWTSecret string TokenDuration time.Duration } type MonitorConfig struct { Enabled bool SamplingRate float64 AuditLogPath string } // LoadConfig 加载配置优先级环境变量 配置文件 默认值 func LoadConfig(configPath string) (*Config, error) { cfg : Config{ Env: getEnv(APP_ENV, development), Version: getEnv(APP_VERSION, dev), } // 服务器配置 cfg.Server ServerConfig{ Port: getEnvInt(SERVER_PORT, 8080), ReadTimeout: getEnvDuration(SERVER_READ_TIMEOUT, 120*time.Second), WriteTimeout: getEnvDuration(SERVER_WRITE_TIMEOUT, 300*time.Second), GracePeriod: getEnvDuration(SERVER_GRACE_PERIOD, 30*time.Second), } // LLM 配置 cfg.LLM LLMConfig{ DefaultProvider: getEnv(LLM_DEFAULT_PROVIDER, openai), CacheEnabled: getEnvBool(LLM_CACHE_ENABLED, true), CacheTTL: getEnvDuration(LLM_CACHE_TTL, 5*time.Minute), Providers: map[string]ProviderConfig{ openai: { Model: getEnv(OPENAI_MODEL, gpt-4), APIKey: getEnv(OPENAI_API_KEY, ), BaseURL: getEnv(OPENAI_BASE_URL, https://api.openai.com/v1), Timeout: getEnvDuration(OPENAI_TIMEOUT, 60*time.Second), MaxRetries: getEnvInt(OPENAI_MAX_RETRIES, 3), }, claude: { Model: getEnv(CLAUDE_MODEL, claude-3-sonnet), APIKey: getEnv(CLAUDE_API_KEY, ), BaseURL: getEnv(CLAUDE_BASE_URL, https://api.anthropic.com/v1), Timeout: getEnvDuration(CLAUDE_TIMEOUT, 90*time.Second), MaxRetries: getEnvInt(CLAUDE_MAX_RETRIES, 2), }, }, } // Redis 配置 cfg.Redis RedisConfig{ Address: getEnv(REDIS_ADDRESS, localhost:6379), Password: getEnv(REDIS_PASSWORD, ), PoolSize: getEnvInt(REDIS_POOL_SIZE, 100), } // 数据库配置 cfg.Database DatabaseConfig{ URL: getEnv(DATABASE_URL, postgresql://localhost:5432/aiplatform), MaxConnections: getEnvInt(DB_MAX_CONNECTIONS, 50), MaxIdleConns: getEnvInt(DB_MAX_IDLE_CONNS, 10), ConnMaxLifetime: getEnvDuration(DB_CONN_MAX_LIFETIME, 30*time.Minute), } // 认证配置 cfg.Auth AuthConfig{ JWTSecret: getEnv(JWT_SECRET, change-me-in-production), TokenDuration: getEnvDuration(JWT_TOKEN_DURATION, 24*time.Hour), } // 监控配置 cfg.Monitor MonitorConfig{ Enabled: getEnvBool(MONITOR_ENABLED, true), SamplingRate: getEnvFloat(MONITOR_SAMPLING_RATE, 1.0), AuditLogPath: getEnv(AUDIT_LOG_PATH, /var/log/audit.log), } // 环境差异调整 cfg.applyEnvironmentOverrides() return cfg, nil } func (cfg *Config) applyEnvironmentOverrides() { switch cfg.Env { case development: cfg.Server.GracePeriod 5 * time.Second cfg.Monitor.SamplingRate 1.0 case staging: cfg.LLM.CacheEnabled true cfg.Monitor.SamplingRate 0.5 case production: cfg.Server.GracePeriod 45 * time.Second cfg.Monitor.SamplingRate 0.1 } } // ─── 辅助函数 ─── func getEnv(key, defaultVal string) string { if val : os.Getenv(key); val ! { return val } return defaultVal } func getEnvInt(key string, defaultVal int) int { if val : os.Getenv(key); val ! { if i, err : strconv.Atoi(val); err nil { return i } } return defaultVal } func getEnvBool(key string, defaultVal bool) bool { if val : os.Getenv(key); val ! { if b, err : strconv.ParseBool(val); err nil { return b } } return defaultVal } func getEnvDuration(key string, defaultVal time.Duration) time.Duration { if val : os.Getenv(key); val ! { if d, err : time.ParseDuration(val); err nil { return d } } return defaultVal } func getEnvFloat(key string, defaultVal float64) float64 { if val : os.Getenv(key); val ! { if f, err : strconv.ParseFloat(val, 64); err nil { return f } } return defaultVal }六、健康检查与优雅关闭// deploy/graceful.go package deploy import ( context log net/http os os/signal syscall time ) // GracefulServer 优雅启停服务器 type GracefulServer struct { server *http.Server shutdownCh chan struct{} } // NewGracefulServer 创建优雅服务器 func NewGracefulServer(handler http.Handler, cfg ServerConfig) *GracefulServer { return GracefulServer{ server: http.Server{ Addr: fmt.Sprintf(:%d, cfg.Port), Handler: handler, ReadTimeout: cfg.ReadTimeout, WriteTimeout: cfg.WriteTimeout, }, shutdownCh: make(chan struct{}), } } // Start 启动服务器阻塞直到收到退出信号 func (gs *GracefulServer) Start() { // 监听系统信号 sigCh : make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) // 启动服务器 go func() { log.Printf([Server] Starting on %s, gs.server.Addr) if err : gs.server.ListenAndServe(); err ! nil err ! http.ErrServerClosed { log.Fatalf([Server] Failed to start: %v, err) } }() // 等待信号 sig : -sigCh log.Printf([Server] Received signal: %v, shutting down..., sig) // 优雅关闭 gs.Shutdown() } // Shutdown 优雅关闭 func (gs *GracefulServer) Shutdown() { ctx, cancel : context.WithTimeout(context.Background(), 30*time.Second) defer cancel() if err : gs.server.Shutdown(ctx); err ! nil { log.Printf([Server] Shutdown error: %v, err) } close(gs.shutdownCh) log.Println([Server] Shutdown complete) } // ─── 就绪检查 ─── // ReadinessChecker 就绪检查器 type ReadinessChecker struct { checks []ReadinessCheck } type ReadinessCheck struct { Name string Check func() error } func NewReadinessChecker() *ReadinessChecker { return ReadinessChecker{} } func (rc *ReadinessChecker) AddCheck(name string, check func() error) { rc.checks append(rc.checks, ReadinessCheck{Name: name, Check: check}) } func (rc *ReadinessChecker) Ready() error { for _, check : range rc.checks { if err : check.Check(); err ! nil { return fmt.Errorf(%s: %w, check.Name, err) } } return nil } // HealthHandler 健康检查处理器 func HealthHandler(checker *ReadinessChecker) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { w.Header().Set(Content-Type, application/json) if r.URL.Path /ready { if err : checker.Ready(); err ! nil { w.WriteHeader(http.StatusServiceUnavailable) w.Write([]byte(fmt.Sprintf({status:not_ready,reason:%s}, err.Error()))) return } } w.WriteHeader(http.StatusOK) w.Write([]byte({status:ok})) } }七、成本优化// deploy/cost.go package deploy import ( sort sync time ) // CostOptimizer 成本优化器 type CostOptimizer struct { metrics *CostMetrics rules []CostRule mu sync.RWMutex } // CostMetrics 成本指标 type CostMetrics struct { DailyCost float64 MonthlyProjection float64 CostByModel map[string]float64 CostByEndpoint map[string]float64 TokenUsage map[string]int64 } // CostRule 成本优化规则 type CostRule struct { Name string Condition func(metrics *CostMetrics) bool Action func() string SavingsDesc string } // NewCostOptimizer 创建成本优化器 func NewCostOptimizer() *CostOptimizer { co : CostOptimizer{ metrics: CostMetrics{ CostByModel: make(map[string]float64), CostByEndpoint: make(map[string]float64), TokenUsage: make(map[string]int64), }, } co.registerRules() return co } func (co *CostOptimizer) registerRules() { co.rules []CostRule{ { Name: model_downgrade, Condition: func(m *CostMetrics) bool { return m.DailyCost 100 m.CostByModel[gpt-4] m.DailyCost*0.7 }, Action: func() string { return 建议将部分流量切换到 gpt-3.5-turbo可节省约 70% 成本 }, SavingsDesc: 每月可节省约 $2000, }, { Name: cache_enable, Condition: func(m *CostMetrics) bool { // 如果相同 Prompt 重复率高 return true }, Action: func() string { return 启用语义缓存相同或相似问题直接从缓存返回 }, SavingsDesc: 预计减少 30% LLM 调用, }, { Name: batch_processing, Condition: func(m *CostMetrics) bool { return m.DailyCost 500 }, Action: func() string { return 非实时场景使用批量 API价格约为实时 API 的 50% }, SavingsDesc: 批量处理可节省 50% 成本, }, { Name: context_window, Condition: func(m *CostMetrics) bool { avgTokens : float64(m.TokenUsage[total]) / float64(max(1, m.TokenUsage[requests])) return avgTokens 4000 }, Action: func() string { return 压缩历史对话只保留最近的 3 轮对话 }, SavingsDesc: 减少 40% Token 消耗, }, ] } // Analyze 分析成本并提供优化建议 func (co *CostOptimizer) Analyze() []string { co.mu.RLock() defer co.mu.RUnlock() var suggestions []string for _, rule : range co.rules { if rule.Condition(co.metrics) { suggestions append(suggestions, rule.Action() (rule.SavingsDesc)) } } if len(suggestions) 0 { suggestions append(suggestions, 当前成本在合理范围内) } return suggestions } // ─── 弹性伸缩策略 ─── // ScalingStrategy 弹性伸缩策略 type ScalingStrategy struct { // 基于 QPS TargetQPS int CurrentReplicas int MaxReplicas int MinReplicas int // 基于成本 MaxDailyCost float64 CurrentDailyCost float64 // 调度策略 Schedule map[string]int // hour - replicas } // OptimizeReplicas 优化副本数 func (ss *ScalingStrategy) OptimizeReplicas(currentQPS int) int { // 基于 QPS 计算所需副本 neededByLoad : currentQPS / ss.TargetQPS if neededByLoad ss.MinReplicas { neededByLoad ss.MinReplicas } if neededByLoad ss.MaxReplicas { neededByLoad ss.MaxReplicas } // 基于成本限制 costPerReplica : ss.CurrentDailyCost / float64(ss.CurrentReplicas) maxReplicasByCost : int(ss.MaxDailyCost / costPerReplica) if maxReplicasByCost neededByLoad { neededByLoad maxReplicasByCost } // 基于时间表 hour : time.Now().Hour() if scheduled, ok : ss.Schedule[fmt.Sprintf(%d, hour)]; ok { if scheduled neededByLoad { neededByLoad scheduled } } return neededByLoad }八、SLA 保障// deploy/sla.go package deploy import ( fmt sync time ) // SLAConfig SLA 配置 type SLAConfig struct { UptimePercent float64 // 99.9% MaxLatencyP99 time.Duration MaxErrorRate float64 MaxResponseTime time.Duration ThroughputMin int // QPS } // SLAMonitor SLA 监控器 type SLAMonitor struct { config SLAConfig metrics *SLAMetrics breaches []SLABreach mu sync.RWMutex } // SLAMetrics SLA 指标 type SLAMetrics struct { TotalRequests int64 SuccessfulReqs int64 FailedReqs int64 Latencies []time.Duration UptimeChecks int64 UptimeSuccesses int64 } // SLABreach SLA 违规 type SLABreach struct { Time time.Time Indicator string Actual string Expected string Severity string } // NewSLAMonitor 创建 SLA 监控器 func NewSLAMonitor(config SLAConfig) *SLAMonitor { return SLAMonitor{ config: config, metrics: SLAMetrics{}, } } // RecordRequest 记录请求 func (sm *SLAMonitor) RecordRequest(latency time.Duration, success bool) { sm.mu.Lock() defer sm.mu.Unlock() sm.metrics.TotalRequests if success { sm.metrics.SuccessfulReqs } else { sm.metrics.FailedReqs } sm.metrics.Latencies append(sm.metrics.Latencies, latency) // 检查 SLA 违规 sm.checkBreaches(latency, success) } // RecordUptimeCheck 记录存活检查 func (sm *SLAMonitor) RecordUptimeCheck(success bool) { sm.mu.Lock() defer sm.mu.Unlock() sm.metrics.UptimeChecks if success { sm.metrics.UptimeSuccesses } } func (sm *SLAMonitor) checkBreaches(latency time.Duration, success bool) { // 错误率检查 if !success { rate : float64(sm.metrics.FailedReqs) / float64(sm.metrics.TotalRequests) if rate sm.config.MaxErrorRate sm.metrics.TotalRequests 100 { sm.breaches append(sm.breaches, SLABreach{ Time: time.Now(), Indicator: error_rate, Actual: fmt.Sprintf(%.2f%%, rate*100), Expected: fmt.Sprintf( %.2f%%, sm.config.MaxErrorRate*100), Severity: critical, }) } } // 延迟检查 if latency sm.config.MaxLatencyP99 { sm.breaches append(sm.breaches, SLABreach{ Time: time.Now(), Indicator: latency_p99, Actual: latency.String(), Expected: fmt.Sprintf( %s, sm.config.MaxLatencyP99), Severity: warning, }) } } // Report 生成 SLA 报告 func (sm *SLAMonitor) Report() string { sm.mu.RLock() defer sm.mu.RUnlock() uptime : 0.0 if sm.metrics.UptimeChecks 0 { uptime float64(sm.metrics.UptimeSuccesses) / float64(sm.metrics.UptimeChecks) * 100 } errorRate : 0.0 if sm.metrics.TotalRequests 0 { errorRate float64(sm.metrics.FailedReqs) / float64(sm.metrics.TotalRequests) * 100 } report : fmt.Sprintf( SLA Report Uptime: %.3f%% (target: %.1f%%) Error Rate: %.3f%% (target: %.2f%%) Total Reqs: %d Failed Reqs: %d Breaches: %d , uptime, sm.config.UptimePercent, errorRate, sm.config.MaxErrorRate*100, sm.metrics.TotalRequests, sm.metrics.FailedReqs, len(sm.breaches), ) if len(sm.breaches) 0 { report \nRecent Breaches:\n for _, b : range sm.breaches[len(sm.breaches)-5:] { report fmt.Sprintf( [%s] %s: actual%s expected%s\n, b.Severity, b.Indicator, b.Actual, b.Expected) } } return report }九、生产坑位#坑位现象解决方案1启动顺序依赖​RAG 服务先启动但向量库还没就绪Init Container 等待依赖 Readiness Probe2配置漂移​不同环境的配置不一致导致线上事故GitOps ConfigMap 版本管理 配置校验3滚动更新中断连接​旧 Pod 销毁时正在处理的请求丢失PreStop Hook 优雅排空 客户端重试4资源争抢​LLM 调用占满 CPU健康检查超时被杀资源配额 优先级调度 独立的健康检查端口5日志淹没​单 Pod 每天产生 GB 级日志日志级别动态调整 采样 轮转归档6冷启动延迟​扩容后的新 Pod 首次 LLM 调用特别慢预热机制 最小副本保底 就绪探针延迟十、本讲小结┌─────────────────────────────────────────────────────────────┐ │ 第9讲 核心收获 │ │ │ │ ✅ 掌握了 Docker 多阶段构建减小镜像体积 80% │ │ ✅ 编写了完整的 Kubernetes 部署清单Deploy/Svc/HPA/Ingress│ │ ✅ 搭建了 CI/CD 流水线测试→构建→预发→金丝雀→全量 │ │ ✅ 实现了环境差异化配置管理Dev/Staging/Prod │ │ ✅ 实现了优雅启停信号处理 健康检查 就绪检查 │ │ ✅ 掌握了成本优化策略模型降级/缓存/批处理/上下文压缩 │ │ ✅ 构建了 SLA 监控体系可用性/延迟/错误率/吞吐量 │ │ ✅ 了解了 6 个常见坑位及其解决方案 │ │ │ │ 核心理念部署不是终点运维才是真正的开始 │ └─────────────────────────────────────────────────────────────┘下一讲预告《总结与展望——从 CRUD 到 Agent 的全栈进化之路》全系列回顾、最佳实践总结、未来趋势展望、下一步学习路线图。