多gpu下如何分配卡给多个容器(建议容器和gpu:x绑定)

Posted on Views:

01/02绑定gpu0|03/04绑定gpu1

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
services:
  ocrserver01:
    image: zzz
    container_name: ocrserver01
    #runtime: nvidia
    ports:
      - "10001:5001"
    networks:
      - apps
    volumes:
      - /etc/localtime:/etc/localtime
    environment:
      #NVIDIA_VISIBLE_DEVICES: "0" # 部分服务可能不支持
      CUDA_VISIBLE_DEVICES: "0"
    shm_size: 8g
    command: /bin/bash docker/run.sh
    stdin_open: true
    tty: true
    restart: always

  ocrserver02:
    image: zzz
    container_name: ocrserver02
    #runtime: nvidia
    ports:
      - "10002:5001"
    networks:
      - apps
    volumes:
      - /etc/localtime:/etc/localtime
    environment:
      #NVIDIA_VISIBLE_DEVICES: "0"
      CUDA_VISIBLE_DEVICES: "0"
    shm_size: 8g
    command: /bin/bash docker/run.sh
    stdin_open: true
    tty: true
    restart: always

  ocrserver03:
    image: zzz
    container_name: ocrserver03
    #runtime: nvidia
    ports:
      - "10003:5001"
    networks:
      - apps
    volumes:
      - /etc/localtime:/etc/localtime
    environment:
      #NVIDIA_VISIBLE_DEVICES: "1"
      CUDA_VISIBLE_DEVICES: "1"
    shm_size: 8g
    command: /bin/bash docker/run.sh
    stdin_open: true
    tty: true
    restart: always

  ocrserver04:
    image: zzz
    container_name: ocrserver04
    #runtime: nvidia
    ports:
      - "10004:5001"
    networks:
      - apps
    volumes:
      - /etc/localtime:/etc/localtime
    environment:
      #NVIDIA_VISIBLE_DEVICES: "1"
      CUDA_VISIBLE_DEVICES: "1"
    shm_size: 8g
    command: /bin/bash docker/run.sh
    stdin_open: true
    tty: true
    restart: always

networks:
  apps:
    external: true

70d5fd5f21a37f9a3498d9c387fb8a4e.png

1.1当前配置是否正确

你的目标：

容器 GPU
ocrserver01 GPU0
ocrserver02 GPU0
ocrserver03 GPU1
ocrserver04 GPU1

当前配置：

1
2
3
4
5
ocrserver01 -> CUDA_VISIBLE_DEVICES: "0"
ocrserver02 -> CUDA_VISIBLE_DEVICES: "0"

ocrserver03 -> CUDA_VISIBLE_DEVICES: "1"
ocrserver04 -> CUDA_VISIBLE_DEVICES: "1"

这是正确的会形成：

1
2
3
4
5
6
7
GPU0:
  ocrserver01
  ocrserver02

GPU1:
  ocrserver03
  ocrserver04

1.2推荐生产级写法

  • NVIDIA_VISIBLE_DEVICES 物理层可见性
  • CUDA_VISIBLE_DEVICES 程序运行时可见性
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
services:
  ocrserver01:
    image: xxx
    container_name: ocrserver01
    runtime: nvidia
    ports:
      - "10001:5001"
    environment:
      NVIDIA_VISIBLE_DEVICES: "0"
      CUDA_VISIBLE_DEVICES: "0"
    shm_size: 8g
    command: /bin/bash docker/run.sh
    restart: always
    stdin_open: true
    tty: true
    networks:
      - apps
    volumes:
      - /etc/localtime:/etc/localtime:ro
	
  ocrserver02:
    image: xxx
    container_name: ocrserver02
    runtime: nvidia
    ports:
      - "10002:5001"
    environment:
      NVIDIA_VISIBLE_DEVICES: "0"
      CUDA_VISIBLE_DEVICES: "0"
    shm_size: 8g
    command: /bin/bash docker/run.sh
    restart: always
    stdin_open: true
    tty: true
    networks:
      - apps
    volumes:
      - /etc/localtime:/etc/localtime:ro

  ocrserver03:
    image: xxx
    container_name: ocrserver03
    runtime: nvidia
    ports:
      - "10003:5001"
    environment:
      NVIDIA_VISIBLE_DEVICES: "1"
      CUDA_VISIBLE_DEVICES: "1"
    shm_size: 8g
    command: /bin/bash docker/run.sh
    restart: always
    stdin_open: true
    tty: true
    networks:
      - apps
    volumes:
      - /etc/localtime:/etc/localtime:ro

  ocrserver04:
    image: xxx
    container_name: ocrserver04
    runtime: nvidia
    ports:
      - "10004:5001"
    environment:
      NVIDIA_VISIBLE_DEVICES: "1"
      CUDA_VISIBLE_DEVICES: "1"
    shm_size: 8g
    command: /bin/bash docker/run.sh
    restart: always
    stdin_open: true
    tty: true
    networks:
      - apps
    volumes:
      - /etc/localtime:/etc/localtime:ro

networks:
  apps:
    external: true

1.3.验证 GPU 是否正确绑定

宿主机

查看：

1
nvidia-smi

会看到：

1
2
3
4
5
6
7
8
9
GPU0:
  python
  ocrserver01
  ocrserver02

GPU1:
  python
  ocrserver03
  ocrserver04

容器内验证

进入：

1
docker exec -it ocrserver03 bash

执行：

1
nvidia-smi

理论上只会看到：

1
GPU1

而不是全部 GPU。

生产环境建议

你现在：

1
1 GPU 跑 2 OCR 实例

是合理的，因为 OCR：

  • 通常显存占用不高
  • 推理偏 IO/CPU
  • GPU utilization 不一定满

所以：1卡多实例，通常比：1卡1实例吞吐更高。

nginx负载

flowchart TB

Nginx/LB
    --> OCR01
    --> OCR02
    --> OCR03
    --> OCR04

OCR01 --> GPU0
OCR02 --> GPU0

OCR03 --> GPU1
OCR04 --> GPU1

1.1tengine-lb

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
services:
  tengine:
    image: axizdkr/tengine:alpine3.19.1
    container_name: ngx
    restart: always
    ports:
      - "80:80"
    volumes:
      - ./logs:/var/log/nginx
      - ./nginx.conf:/etc/nginx/nginx.conf
      #- ./conf.d/:/etc/nginx/conf.d/
    networks:
      - apps

networks:
  apps:
    external: true

1.2.nginx.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log error;
pid        /var/run/nginx.pid;

events {
    worker_connections  102400;
    use epoll;
    multi_accept on;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    # log
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                     '$status $body_bytes_sent "$http_referer" '
                     '"$http_user_agent" "$http_x_forwarded_for" "$upstream_addr" "$upstream_response_time" "$request_time" ';
    access_log  /var/log/nginx/access.log  main;

    #send/tcp
    sendfile        on;
    tcp_nopush     on;
    tcp_nodelay on; 

    # Hide web server information
    server_tokens off;
    server_info off;
    server_tag off;

    #client
    reset_timedout_connection on;
    client_header_timeout       10s;
    client_body_timeout         12s;
    client_max_body_size        256m;

    #send
    send_timeout                15s;

    #proxy
    proxy_connect_timeout       15s;
    proxy_send_timeout          120s;
    proxy_read_timeout          120s;

    #keepalive
    keepalive_timeout  35;

    #gzip
    gzip on;
    gzip_types text/plain application/xml;

    # redirect server error pages to the static page
    error_page 404             /404.html;
    error_page 500 502 503 504 /50x.html;

    # ocr识别服务
    upstream ocrserver{ 
      server ocrserver01:5001  weight=100;
      server ocrserver02:5001  weight=100;
      server ocrserver03:5001  weight=100;
      server ocrserver04:5001  weight=100;
      check interval=3000 rise=2 fall=3 timeout=2000 type=tcp;
    }

    server {
        listen       80;
        server_name  —;

        #charset koi8-r;
        #access_log  logs/host.access.log  main;

        location ~* /predict {
                        proxy_pass http://ocrserver;
                        proxy_set_header Host $host;
                        proxy_set_header X-Real-IP $remote_addr;
                        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        }
      }
}