From 961f543a93dc9249dbb2cdefd93064e6574e7c18 Mon Sep 17 00:00:00 2001 From: Bihan Rana Date: Fri, 21 Aug 2026 14:00:16 +0545 Subject: [PATCH] [Docs] Service groups and Replica IP interpolation Co-authored-by: Cursor --- mkdocs/docs/concepts/services.md | 51 +++++++++++++-------- mkdocs/docs/examples/inference/dynamo.md | 8 ++-- mkdocs/docs/examples/inference/sglang.md | 28 +++++------ mkdocs/docs/examples/inference/vllm.md | 8 ++-- mkdocs/docs/reference/dstack.yml/service.md | 4 +- 5 files changed, 57 insertions(+), 42 deletions(-) diff --git a/mkdocs/docs/concepts/services.md b/mkdocs/docs/concepts/services.md index 7cd6034cb..2c8a5689b 100644 --- a/mkdocs/docs/concepts/services.md +++ b/mkdocs/docs/concepts/services.md @@ -291,7 +291,8 @@ Setting the minimum number of replicas to `0` allows the service to scale down t ??? info "Replica groups" - A service can include multiple replica groups. Each group can define its own `commands`, `resources` requirements, and `scaling` rules. + A service can define multiple replica groups. Each group has its own `replicas` count (or range), + `resources`, `commands`, and `scaling` rules.
@@ -303,8 +304,8 @@ Setting the minimum number of replicas to `0` allows the service to scale down t env: - MODEL_ID=deepseek-ai/DeepSeek-R1-Distill-Llama-8B - replicas: - - count: 1..2 + groups: + - replicas: 1..2 scaling: metric: rps target: 10 @@ -317,7 +318,7 @@ Setting the minimum number of replicas to `0` allows the service to scale down t resources: gpu: 48GB - - count: 1..4 + - replicas: 1..4 scaling: metric: rps target: 5 @@ -336,7 +337,21 @@ Setting the minimum number of replicas to `0` allows the service to scale down t
- > Properties such as `regions`, `port`, `image`, `env` and some other cannot be configured per replica group. This support is coming soon. + [`groups`](../reference/dstack.yml/service.md#groups) and top-level [`replicas`](../reference/dstack.yml/service.md#replicas) are mutually exclusive. + + > Properties such as `regions`, `port`, `env` and some other cannot be configured per replica group. This support is coming soon. + +### Accessing replica IPs + +Commands in any group can reference the internal IP address of any replica in the run via +`${{ groups[i].replicas[j].IP_ADDRESS }}`, where `i` is the index of the group in `groups` and `j` is +the index of the replica within that group. + +> Only replicas guaranteed at start can be referenced: +> +> - `replicas: 2` → replica indexes `0` and `1` can be referenced +> - `replicas: 1..4` → replica index `0` can be referenced +> - `replicas: 0..4` → no replica indexes can be referenced ### PD disaggregation @@ -363,9 +378,9 @@ Below is an example for running `zai-org/GLM-4.5-Air-FP8` on `H200`: - HF_TOKEN - MODEL_ID=zai-org/GLM-4.5-Air-FP8 - replicas: - - count: 1 - # For now replica group with router must have count: 1 + groups: + - replicas: 1 + # For now the router group must have replicas: 1 commands: - pip install smg - | @@ -379,7 +394,7 @@ Below is an example for running `zai-org/GLM-4.5-Air-FP8` on `H200`: router: type: sglang - - count: 1..4 + - replicas: 1..4 scaling: metric: rps target: 3 @@ -394,7 +409,7 @@ Below is an example for running `zai-org/GLM-4.5-Air-FP8` on `H200`: resources: gpu: H200 - - count: 1..8 + - replicas: 1..8 scaling: metric: rps target: 2 @@ -437,8 +452,8 @@ Below is an example for running `zai-org/GLM-4.5-Air-FP8` on `H200`: - HF_TOKEN - MODEL_ID=zai-org/GLM-4.5-Air-FP8 - replicas: - - count: 1 + groups: + - replicas: 1 docker: true commands: - apt-get update @@ -460,7 +475,7 @@ Below is an example for running `zai-org/GLM-4.5-Air-FP8` on `H200`: router: type: dynamo - - count: 1..4 + - replicas: 1..4 scaling: metric: rps target: 3 @@ -489,7 +504,7 @@ Below is an example for running `zai-org/GLM-4.5-Air-FP8` on `H200`: resources: gpu: H200 - - count: 1..8 + - replicas: 1..8 scaling: metric: rps target: 2 @@ -551,8 +566,8 @@ env: - RDMA_DEVICES=bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7 - NCCL_IB_DISABLE=1 -replicas: - - count: 1 +groups: + - replicas: 1 commands: - pip install smg - | @@ -565,7 +580,7 @@ replicas: router: type: sglang - - count: 1..2 + - replicas: 1..2 scaling: metric: rps target: 300 @@ -591,7 +606,7 @@ replicas: cpu: 96.. memory: 512GB.. - - count: 1..4 + - replicas: 1..4 scaling: metric: rps target: 300 diff --git a/mkdocs/docs/examples/inference/dynamo.md b/mkdocs/docs/examples/inference/dynamo.md index 32a9a1e6e..b73cc809f 100644 --- a/mkdocs/docs/examples/inference/dynamo.md +++ b/mkdocs/docs/examples/inference/dynamo.md @@ -24,8 +24,8 @@ env: - HF_TOKEN - MODEL_ID=zai-org/GLM-4.5-Air-FP8 -replicas: - - count: 1 +groups: + - replicas: 1 docker: true commands: - apt-get update @@ -47,7 +47,7 @@ replicas: router: type: dynamo - - count: 1..4 + - replicas: 1..4 scaling: metric: rps target: 3 @@ -76,7 +76,7 @@ replicas: resources: gpu: H200 - - count: 1..8 + - replicas: 1..8 scaling: metric: rps target: 2 diff --git a/mkdocs/docs/examples/inference/sglang.md b/mkdocs/docs/examples/inference/sglang.md index 5bf25ed5d..94789a2af 100644 --- a/mkdocs/docs/examples/inference/sglang.md +++ b/mkdocs/docs/examples/inference/sglang.md @@ -152,9 +152,9 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ - HF_TOKEN - MODEL_ID=zai-org/GLM-4.5-Air-FP8 - replicas: - - count: 1 - # For now replica group with router must have count: 1 + groups: + - replicas: 1 + # For now replica group with router must have replicas: 1 commands: - pip install smg - | @@ -168,7 +168,7 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ router: type: sglang - - count: 1..4 + - replicas: 1..4 scaling: metric: rps target: 3 @@ -184,7 +184,7 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ resources: gpu: H200 - - count: 1..8 + - replicas: 1..8 scaling: metric: rps target: 2 @@ -225,9 +225,9 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ - HF_TOKEN - MODEL_ID=zai-org/GLM-4.5-Air-FP8 - replicas: - - count: 1 - # For now replica group with router must have count: 1 + groups: + - replicas: 1 + # For now replica group with router must have replicas: 1 python: "3.12" commands: - pip install smg @@ -244,7 +244,7 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ resources: cpu: 4 - - count: 1..4 + - replicas: 1..4 scaling: metric: rps target: 3 @@ -262,7 +262,7 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ resources: gpu: H200 - - count: 1..8 + - replicas: 1..8 scaling: metric: rps target: 2 @@ -309,8 +309,8 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ - RDMA_DEVICES=bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7 - NCCL_IB_DISABLE=1 - replicas: - - count: 1 + groups: + - replicas: 1 commands: - pip install smg - | @@ -323,7 +323,7 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ router: type: sglang - - count: 1..2 + - replicas: 1..2 scaling: metric: rps target: 300 @@ -349,7 +349,7 @@ To run SGLang with [PD disaggregation](https://docs.sglang.io/advanced_features/ cpu: 96.. memory: 512GB.. - - count: 1..4 + - replicas: 1..4 scaling: metric: rps target: 300 diff --git a/mkdocs/docs/examples/inference/vllm.md b/mkdocs/docs/examples/inference/vllm.md index fe1575ed8..0bc271c8d 100644 --- a/mkdocs/docs/examples/inference/vllm.md +++ b/mkdocs/docs/examples/inference/vllm.md @@ -140,8 +140,8 @@ env: - HF_TOKEN - MODEL_ID=zai-org/GLM-4.5-Air-FP8 -replicas: - - count: 1 +groups: + - replicas: 1 python: "3.12" commands: - pip install smg @@ -158,7 +158,7 @@ replicas: resources: cpu: 4 - - count: 1..4 + - replicas: 1..4 scaling: metric: rps target: 3 @@ -173,7 +173,7 @@ replicas: resources: gpu: H200 - - count: 1..8 + - replicas: 1..8 scaling: metric: rps target: 2 diff --git a/mkdocs/docs/reference/dstack.yml/service.md b/mkdocs/docs/reference/dstack.yml/service.md index 5f3aa3bd1..ad4e5585a 100644 --- a/mkdocs/docs/reference/dstack.yml/service.md +++ b/mkdocs/docs/reference/dstack.yml/service.md @@ -82,9 +82,9 @@ The `service` configuration type allows running [services](../../concepts/servic required: true -### `replicas` +### `groups` -#### `replicas[n]` +#### `groups[n]` #SCHEMA# dstack._internal.core.models.configurations.ReplicaGroup overrides: