这有帮助吗？
version: "3.6"
services:
apiserver:
image: allegroai/clearml:latest
restart: unless-stopped
volumes:
- /opt/clearml/logs:/var/log/clearml
- /opt/clearml/config:/opt/clearml/config
- /opt/clearml/data/fileserver:/mnt/fileserver
environment:
CLEARML_MONGODB_SERVICE_HOST: mongo
CLEARML_MONGODB_SERVICE_PORT: 27017
CLEARML_ELASTICSEARCH_SERVICE_HOST: elasticsearch
CLEARML_ELASTICSEARCH_SERVICE_PORT: 9200
CLEARML_REDIS_SERVICE_HOST: redis
CLEARML_REDIS_SERVICE_PORT: 6379
ports:
- "8008:8008"
depends_on:
- mongo
- elasticsearch
- redis
webserver:
image: allegroai/clearml-webserver:latest
restart: unless-stopped
ports:
- "8081:80"
environment:
CLEARML_API_HOST: http://localhost:8008
fileserver:
image: allegroai/clearml-fileserver:latest
restart: unless-stopped
volumes:
- /opt/clearml/data/fileserver:/mnt/fileserver
ports:
- "8081:8081"
mongo:
image: mongo:4.4
restart: unless-stopped
volumes:
- /opt/clearml/data/mongo:/data/db
command: --setParameter internalQueryMaxBlockingSortMemoryUsageBytes=196100200
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.17.6
restart: unless-stopped
environment:
ES_JAVA_OPTS: "-Xms512m -Xmx2048m"
bootstrap.memory_lock: "true"
cluster.name: "clearml"
discovery.type: "single-node"
http.publish_host: "$CLEARML_HOST_IP"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- /opt/clearml/data/elastic:/usr/share/elasticsearch/data
redis:
image: redis:6
restart: unless-stopped
volumes:
- /opt/clearml/data/redis:/data
networks:
default:
name: clearml_networkmkdir -p /opt/clearml/{logs,config,data/{fileserver,mongo,elastic,redis}}
# 设置服务器的公网 IP
export CLEARML_HOST_IP=<your-server-ip>
docker-compose up -d# 安装 SDK
pip install clearml
# 使用托管服务器进行配置
clearml-init
# 在提示输入 API 主机时输入： https://api.clear.ml
# 从此处获取凭证： https://app.clear.ml/settings/workspace-configurationhttp://<server-ip>:8081http://<server-ip>:8008ssh root@<server-ip> -p 22pip install clearmlclearml-initfrom clearml import Task
Task.set_credentials(
api_host="http://<server-ip>:8008",
web_host="http://<server-ip>:8081",
files_host="http://<server-ip>:8081",
key="YOUR_ACCESS_KEY",
secret="YOUR_SECRET_KEY"
)from clearml import Task
# 初始化任务 —— 这会自动捕获所有内容
task = Task.init(project_name="MyProject", task_name="experiment-001")
# 您现有的训练代码 —— 无需更改
import torch
import torch.nn as nn
model = nn.Linear(10, 1)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
for epoch in range(10):
loss = torch.tensor(1.0 / (epoch + 1))
# 如果使用标准框架，ClearML 会自动检测并记录 loss
print(f"Epoch {epoch}, Loss: {loss.item():.4f}")
task.close()from clearml import Task, Logger
task = Task.init(project_name="MyProject", task_name="manual-logging-demo")
logger = task.get_logger()
for epoch in range(50):
train_loss = 1.0 / (epoch + 1)
val_accuracy = 0.95 - 0.5 / (epoch + 1)
# 记录标量
logger.report_scalar("Loss", "train", value=train_loss, iteration=epoch)
logger.report_scalar("Accuracy", "validation", value=val_accuracy, iteration=epoch)
# 记录学习率
logger.report_scalar("Learning Rate", "lr", value=0.001 * 0.9**epoch, iteration=epoch)
print("训练完成！")
task.close()from clearml import Task
task = Task.init(project_name="HPO-Demo", task_name="run-001")
# 连接超参数 —— 自动记录并可远程覆盖
params = {
"learning_rate": 0.001,
"batch_size": 32,
"num_layers": 4,
"dropout": 0.3,
"optimizer": "adam",
"epochs": 100,
}
params = task.connect(params) # 现在可以被 ClearML HPO 覆盖
print(f"Training with lr={params['learning_rate']}, batch={params['batch_size']}")from clearml import Dataset
# 创建版本化数据集
dataset = Dataset.create(
dataset_name="my-training-data",
dataset_project="MyProject",
dataset_version="1.0",
)
# 添加文件
dataset.add_files(path="/data/images/", recursive=True)
dataset.add_files(path="/data/labels.csv")
# 上传到 ClearML 服务器
dataset.upload()
dataset.finalize()
print(f"Dataset ID: {dataset.id}")
# 稍后：在实验中使用该数据集
dataset = Dataset.get(dataset_name="my-training-data", dataset_version="1.0")
local_path = dataset.get_local_copy()
print(f"Dataset at: {local_path}")from clearml import Task, OutputModel, InputModel
import torch
task = Task.init(project_name="ModelRegistry", task_name="training-run")
# 训练后，注册模型
model = torch.nn.Linear(100, 10)
torch.save(model.state_dict(), "my_model.pt")
# 注册输出模型
output_model = OutputModel(task=task, name="MyModel-v1")
output_model.update_weights("my_model.pt")
output_model.publish() # 标记为可用
print(f"Model registered: {output_model.id}")
# 部署时：按名称加载模型
input_model = InputModel(model_id="<model-id-from-dashboard>")
local_model_path = input_model.get_local_copy()
state_dict = torch.load(local_model_path)from clearml.automation import PipelineController
def step_preprocess(dataset_id: str) -> str:
"""数据预处理步骤。"""
from clearml import Task, Dataset
task = Task.init(task_name="step-preprocess")
# ... 预处理逻辑
return "processed_data_id"
def step_train(data_id: str, lr: float = 0.001) -> str:
"""模型训练步骤。"""
from clearml import Task
task = Task.init(task_name="step-train")
# ... 训练逻辑
return "model_id"
def step_evaluate(model_id: str) -> float:
"""模型评估步骤。"""
from clearml import Task
task = Task.init(task_name="step-evaluate")
# ... 评估逻辑
return 0.95
# 构建流水线
pipe = PipelineController(
name="ML-Training-Pipeline",
project="MyPipelines",
version="1.0"
)
pipe.add_function_step(
name="preprocess",
function=step_preprocess,
function_kwargs={"dataset_id": "raw-data-id"},
function_return=["processed_id"],
)
pipe.add_function_step(
name="train",
parents=["preprocess"],
function=step_train,
function_kwargs={"data_id": "${preprocess.processed_id}"},
function_return=["model_id"],
execution_queue="gpu-queue", # 在 GPU 工作节点上运行
)
pipe.add_function_step(
name="evaluate",
parents=["train"],
function=step_evaluate,
function_kwargs={"model_id": "${train.model_id}"},
function_return=["accuracy"],
)
pipe.start()
pipe.wait()
print("流水线完成！")# 安装 agent
pip install clearml-agent
# 配置（使用与 SDK 相同的凭证）
clearml-agent init
# 在 GPU 上启动工作者
clearml-agent daemon --queue "gpu-queue" --gpus 0,1
# 使用 Docker 隔离启动工作者（推荐）
clearml-agent daemon \
--queue "gpu-queue" \
--docker pytorch/pytorch:2.1.0-cuda12.1-cudnn8-runtime \
--gpus allfrom clearml.automation import (
HyperParameterOptimizer,
UniformParameterRange,
DiscreteParameterValues,
GridSearch,
)
optimizer = HyperParameterOptimizer(
base_task_id="<task-id-to-optimize>",
hyper_parameters:[
UniformParameterRange("General/learning_rate", min_value=1e-5, max_value=1e-2, step_size=1e-5),
DiscreteParameterValues("General/batch_size", values=[16, 32, 64, 128]),
DiscreteParameterValues("General/optimizer", values=["adam", "sgd", "adamw"]),
],
objective_metric_title="Accuracy",
objective_metric_series="validation",
objective_metric_sign="max", # 最大化验证准确率
max_number_of_concurrent_tasks=4,
optimizer_class=GridSearch,
execution_queue="gpu-queue",
total_max_jobs=50,
)
optimizer.start()
top_exps = optimizer.get_top_experiments(top_k=3)
print("最佳实验：", top_exps)from clearml import Task
task = Task.init(project_name="Production", task_name="monitoring")
# 为便于筛选设置任务标签
task.add_tags(["production", "v2.1", "gpu"])
# 自动记录系统指标 —— 只需初始化任务即可
# ClearML 自动捕获：CPU、内存、GPU 利用率、GPU 显存
# 添加自定义标量监控
logger = task.get_logger()
import time
for i in range(100):
gpu_util = 85 + (i % 10)
logger.report_scalar("GPU", "utilization_%", value=gpu_util, iteration=i)
time.sleep(1)