first commit
This commit is contained in:
Binary file not shown.
@@ -0,0 +1,147 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
bench_latency.py
|
||||
- 加载训练好的 Transformer 模型(best_model.pt)与 scaler.npz(用于确定列与F)
|
||||
- 构造单次前向输入(从指定 CSV 取最后 L 行并标准化,或随机生成)
|
||||
- 多次重复前向,统计耗时(均值/最佳/p50/p90/p99),支持 CUDA 同步与 autocast
|
||||
|
||||
用法示例:
|
||||
uv run chaos_pdl/bench_latency.py \
|
||||
--model_path outputs/chaos/best_model.pt \
|
||||
--scaler_path outputs/chaos/scaler.npz \
|
||||
--csv_path chaos_pdl/data/metrics-20250920-000328_angles.csv \
|
||||
--seq_len 180 --iters 200 --warmup 50 --dtype fp32
|
||||
|
||||
如导入问题,可用模块方式:
|
||||
uv run -m chaos_pdl.bench_latency ...
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import time
|
||||
from typing import List
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
|
||||
try:
|
||||
from chaos_pdl.pendulum_transformer import TimeSeriesTransformer
|
||||
except ModuleNotFoundError:
|
||||
from pendulum_transformer import TimeSeriesTransformer
|
||||
|
||||
|
||||
def read_last_window(csv_path: str, expect_columns: List[str], seq_len: int):
|
||||
df = pd.read_csv(csv_path)
|
||||
if "frame_index" in df.columns:
|
||||
df = df.sort_values("frame_index").drop(columns=["frame_index"]) # 与训练一致
|
||||
df_num = df.select_dtypes(include=[np.number]).copy()
|
||||
has = list(df_num.columns)
|
||||
if set(has) != set(expect_columns):
|
||||
raise ValueError(
|
||||
f"CSV 数值列与训练不一致\n 期望: {expect_columns}\n 实际: {has}\n"
|
||||
)
|
||||
df_num = df_num[expect_columns].reset_index(drop=True)
|
||||
if len(df_num) < seq_len:
|
||||
raise ValueError(f"数据行不足:需要至少 {seq_len} 行,实际 {len(df_num)}")
|
||||
arr = df_num.values.astype(np.float32)
|
||||
return arr[-seq_len:, :]
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--model_path", type=str, default="./outputs/chaos/best_model.pt")
|
||||
ap.add_argument("--scaler_path", type=str, default="./outputs/chaos/scaler.npz")
|
||||
ap.add_argument("--csv_path", type=str, default="", help="用于构造输入的 CSV(可选)")
|
||||
ap.add_argument("--seq_len", type=int, default=180)
|
||||
ap.add_argument("--iters", type=int, default=200)
|
||||
ap.add_argument("--warmup", type=int, default=50)
|
||||
ap.add_argument("--batch", type=int, default=1)
|
||||
ap.add_argument("--dtype", type=str, default="fp32", choices=["fp32", "bf16", "fp16"])
|
||||
ap.add_argument("--use_autocast", action="store_true", help="使用 autocast 进行混合精度(GPU 推荐)")
|
||||
args = ap.parse_args()
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
if device.type == "cuda":
|
||||
print(f"Device: cuda | {torch.cuda.get_device_name(0)}")
|
||||
else:
|
||||
print("Device: cpu")
|
||||
|
||||
sc = np.load(args.scaler_path, allow_pickle=True)
|
||||
mean = sc["mean"].astype(np.float32)
|
||||
std = sc["std"].astype(np.float32)
|
||||
columns = list(sc["columns"].tolist())
|
||||
F = len(columns)
|
||||
|
||||
# 加载模型
|
||||
ckpt = torch.load(args.model_path, map_location=device)
|
||||
cfg = ckpt["config"]
|
||||
horizon = int(cfg["horizon"])
|
||||
model = TimeSeriesTransformer(
|
||||
in_features=F,
|
||||
d_model=cfg["d_model"],
|
||||
nhead=cfg["nhead"],
|
||||
num_layers=cfg["num_layers"],
|
||||
dim_feedforward=cfg["dim_ff"],
|
||||
dropout=cfg["dropout"],
|
||||
horizon=horizon,
|
||||
).to(device)
|
||||
model.load_state_dict(ckpt["model_state"])
|
||||
model.eval()
|
||||
|
||||
# 构造输入(标准化后)
|
||||
if args.csv_path:
|
||||
arr = read_last_window(args.csv_path, columns, args.seq_len)
|
||||
else:
|
||||
# 随机输入(标准化空间)
|
||||
arr = np.random.randn(args.seq_len, F).astype(np.float32)
|
||||
|
||||
# 标准化
|
||||
arr_norm = (arr - mean) / std
|
||||
xb_np = np.repeat(arr_norm[None, :, :], args.batch, axis=0) # [B,L,F]
|
||||
xb = torch.from_numpy(xb_np).to(device)
|
||||
|
||||
# dtype/autocast
|
||||
dtype_map = {"fp32": torch.float32, "bf16": torch.bfloat16, "fp16": torch.float16}
|
||||
amp_dtype = dtype_map[args.dtype]
|
||||
use_amp = args.use_autocast and (device.type == "cuda") and (amp_dtype != torch.float32)
|
||||
|
||||
# 预热
|
||||
with torch.no_grad():
|
||||
for _ in range(max(0, args.warmup)):
|
||||
if use_amp:
|
||||
with torch.autocast(device_type="cuda", dtype=amp_dtype):
|
||||
_ = model(xb)
|
||||
else:
|
||||
_ = model(xb)
|
||||
if device.type == "cuda":
|
||||
torch.cuda.synchronize()
|
||||
|
||||
# 正式测时
|
||||
times = []
|
||||
with torch.no_grad():
|
||||
for _ in range(args.iters):
|
||||
if device.type == "cuda":
|
||||
torch.cuda.synchronize()
|
||||
t0 = time.perf_counter()
|
||||
if use_amp:
|
||||
with torch.autocast(device_type="cuda", dtype=amp_dtype):
|
||||
_ = model(xb)
|
||||
else:
|
||||
_ = model(xb)
|
||||
if device.type == "cuda":
|
||||
torch.cuda.synchronize()
|
||||
t1 = time.perf_counter()
|
||||
times.append((t1 - t0) * 1000.0) # ms
|
||||
|
||||
times = np.array(times, dtype=np.float64)
|
||||
def pct(p):
|
||||
return float(np.percentile(times, p))
|
||||
|
||||
print("Latency (ms) for single forward:")
|
||||
print(f" batch={args.batch}, L={args.seq_len}, F={F}, H={horizon}, dtype={args.dtype}, autocast={use_amp}")
|
||||
print(f" mean={times.mean():.3f} | std={times.std():.3f} | min={times.min():.3f}")
|
||||
print(f" p50={pct(50):.3f} | p90={pct(90):.3f} | p99={pct(99):.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
uv run ./chaos_pdl/normalize_csv_batch_2.py ./chaos_pdl/data/raw/*.csv -o ./chaos_pdl/data/ --font "chaos_pdl/PingFang SC.ttf"
|
||||
|
||||
uv run chaos_pdl/pendulum_transformer.py --csv_paths chaos_pdl/data/*.csv
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,335 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
eval_model.py
|
||||
- 加载训练好的 Transformer 模型(best_model.pt)和标准化参数(scaler.npz)
|
||||
- 对单个 CSV 进行标准化、滑窗切片并评测:wRMSE / MSE / MAE(original)
|
||||
- 可选输出最后一个窗口的预测与真值(反标准化)到 CSV
|
||||
|
||||
使用示例:
|
||||
uv run chaos_pdl/eval_model.py \
|
||||
--csv_path chaos_pdl/data/achieve2/raw/IMG_1119_out_metrics.csv \
|
||||
--model_path outputs/chaos/best_model.pt \
|
||||
--scaler_path outputs/chaos/scaler.npz \
|
||||
--seq_len 180 --stride 1 \
|
||||
--save_dir outputs/chaos
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
|
||||
# 复用训练脚本中的模块与评测函数
|
||||
# 兼容两种导入方式:
|
||||
try:
|
||||
# 优先按包路径导入(若将项目安装为包或以工作区根为 sys.path)
|
||||
from chaos_pdl.pendulum_transformer import (
|
||||
TimeSeriesTransformer,
|
||||
WeightedRMSELoss,
|
||||
WindowedTSDataset,
|
||||
)
|
||||
except ModuleNotFoundError:
|
||||
# 退化为同目录导入(直接运行本脚本时更稳)
|
||||
from pendulum_transformer import (
|
||||
TimeSeriesTransformer,
|
||||
WeightedRMSELoss,
|
||||
WindowedTSDataset,
|
||||
)
|
||||
|
||||
|
||||
def build_windows_indices(T: int, seq_len: int, horizon: int, stride: int) -> np.ndarray:
|
||||
"""与训练一致:生成起点索引,确保 s+seq_len+horizon 不越界。"""
|
||||
max_start = T - seq_len - horizon
|
||||
if max_start < 0:
|
||||
return np.array([], dtype=np.int64)
|
||||
return np.arange(0, max_start + 1, stride, dtype=np.int64)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def evaluate_losses(model, loader, device, weighted_rmse: WeightedRMSELoss):
|
||||
model.eval()
|
||||
total_wrmse, total_mse = 0.0, 0.0
|
||||
count = 0
|
||||
mse_fn = torch.nn.MSELoss()
|
||||
for xb, yb in loader:
|
||||
xb = xb.to(device)
|
||||
yb = yb.to(device)
|
||||
pred = model(xb)
|
||||
total_wrmse += weighted_rmse(pred, yb).item() * xb.size(0)
|
||||
total_mse += mse_fn(pred, yb).item() * xb.size(0)
|
||||
count += xb.size(0)
|
||||
return total_wrmse / max(1, count), total_mse / max(1, count)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def mae_in_original_scale(model, loader, device, mean: np.ndarray, std: np.ndarray):
|
||||
model.eval()
|
||||
total_mae, count = 0.0, 0
|
||||
mean_t = torch.from_numpy(mean).to(device).view(1, 1, -1)
|
||||
std_t = torch.from_numpy(std).to(device).view(1, 1, -1)
|
||||
for xb, yb in loader:
|
||||
xb = xb.to(device)
|
||||
yb = yb.to(device)
|
||||
pred = model(xb)
|
||||
pred_denorm = pred * std_t + mean_t
|
||||
yb_denorm = yb * std_t + mean_t
|
||||
mae = torch.abs(pred_denorm - yb_denorm).mean()
|
||||
total_mae += mae.item() * xb.size(0)
|
||||
count += xb.size(0)
|
||||
return total_mae / max(1, count)
|
||||
|
||||
|
||||
def read_single_csv_align(path: str, expect_columns: List[str]) -> pd.DataFrame:
|
||||
"""
|
||||
读取单个 CSV:
|
||||
- 若含 frame_index,则按其排序并移除;
|
||||
- 仅保留数值列;
|
||||
- 校验列集合与 scaler 中记录的列一致,并按预期顺序重排。
|
||||
"""
|
||||
df = pd.read_csv(path)
|
||||
if "frame_index" in df.columns:
|
||||
df = df.sort_values("frame_index").drop(columns=["frame_index"]) # 对齐训练逻辑
|
||||
df_num = df.select_dtypes(include=[np.number]).copy()
|
||||
has = list(df_num.columns)
|
||||
if set(has) != set(expect_columns):
|
||||
raise ValueError(
|
||||
f"CSV 数值列与训练时不一致\n 期望: {expect_columns}\n 实际: {has}\n"
|
||||
"请确保列集合一致(名称与类型),且来源一致。"
|
||||
)
|
||||
# 按 scaler 的列顺序重排
|
||||
df_num = df_num[expect_columns]
|
||||
return df_num.reset_index(drop=True)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--csv_path", type=str, required=True, help="待评测的 CSV 文件路径")
|
||||
ap.add_argument("--model_path", type=str, default="./outputs/chaos/best_model.pt")
|
||||
ap.add_argument("--scaler_path", type=str, default="./outputs/chaos/scaler.npz")
|
||||
ap.add_argument("--seq_len", type=int, default=180, help="历史窗口长度 L(需与训练一致)")
|
||||
ap.add_argument("--stride", type=int, default=1, help="评测滑窗步长")
|
||||
ap.add_argument("--batch_size", type=int, default=1024)
|
||||
ap.add_argument("--save_dir", type=str, default="", help="若提供,则保存最后一个窗口的预测/真值 CSV")
|
||||
ap.add_argument(
|
||||
"--plot_path",
|
||||
type=str,
|
||||
default="",
|
||||
help="每步RMSE(原尺度)曲线图的保存路径(.png)。若未提供且有save_dir,则保存到 save_dir/per_step_rmse.png",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--plot_heatmap",
|
||||
action="store_true",
|
||||
help="输出每步×每特征的RMSE热力图(原尺度),保存到 save_dir/per_step_feature_rmse.png 与 CSV",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--plot_feature",
|
||||
type=str,
|
||||
default="",
|
||||
help="指定一个特征名,对最后一个窗口画出预测与真值曲线;默认取第一个列名",
|
||||
)
|
||||
|
||||
args = ap.parse_args()
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"Device: {device}")
|
||||
|
||||
# 读取 scaler(包含列顺序)
|
||||
sc = np.load(args.scaler_path, allow_pickle=True)
|
||||
mean = sc["mean"]
|
||||
std = sc["std"]
|
||||
columns = list(sc["columns"].tolist())
|
||||
print(f"Scaler loaded. Columns={columns}")
|
||||
|
||||
# 读取并对齐 CSV
|
||||
df = read_single_csv_align(args.csv_path, columns)
|
||||
values = df.values.astype(np.float32)
|
||||
T, F = values.shape
|
||||
print(f"Data loaded. T={T}, F={F}")
|
||||
|
||||
# 加载模型
|
||||
ckpt = torch.load(args.model_path, map_location=device)
|
||||
cfg = ckpt["config"]
|
||||
loss_cfg = ckpt.get("loss_cfg", {"mode": "exp", "gamma": 0.99, "alpha": 0.02, "min_w": 0.05})
|
||||
horizon = int(cfg["horizon"]) # 评测时以 ckpt 中的 horizon 为准
|
||||
model = TimeSeriesTransformer(
|
||||
in_features=F,
|
||||
d_model=cfg["d_model"],
|
||||
nhead=cfg["nhead"],
|
||||
num_layers=cfg["num_layers"],
|
||||
dim_feedforward=cfg["dim_ff"],
|
||||
dropout=cfg["dropout"],
|
||||
horizon=horizon,
|
||||
).to(device)
|
||||
model.load_state_dict(ckpt["model_state"])
|
||||
model.eval()
|
||||
|
||||
# 构造评测窗口
|
||||
starts = build_windows_indices(T, args.seq_len, horizon, args.stride)
|
||||
if len(starts) == 0:
|
||||
raise ValueError(
|
||||
f"数据不足:T={T},需要至少 seq_len({args.seq_len})+horizon({horizon}) 帧。"
|
||||
)
|
||||
print(f"Eval windows: {len(starts)} (L={args.seq_len}, H={horizon}, stride={args.stride})")
|
||||
|
||||
# 标准化(使用训练好的 mean/std)
|
||||
values_norm = (values - mean) / std
|
||||
|
||||
# DataLoader
|
||||
ds = WindowedTSDataset(values_norm, starts, args.seq_len, horizon)
|
||||
loader = torch.utils.data.DataLoader(ds, batch_size=args.batch_size, shuffle=False)
|
||||
|
||||
# 构造损失
|
||||
weighted_rmse = WeightedRMSELoss(
|
||||
horizon=horizon,
|
||||
mode=loss_cfg.get("mode", "exp"),
|
||||
gamma=loss_cfg.get("gamma", 0.99),
|
||||
alpha=loss_cfg.get("alpha", 0.02),
|
||||
min_w=loss_cfg.get("min_w", 0.05),
|
||||
).to(device)
|
||||
|
||||
# 评测
|
||||
val_wrmse, val_mse = evaluate_losses(model, loader, device, weighted_rmse)
|
||||
val_mae_orig = mae_in_original_scale(model, loader, device, mean, std)
|
||||
print(
|
||||
f"[Eval] wRMSE={val_wrmse:.6f} | MSE={val_mse:.6f} | MAE(original)={val_mae_orig:.6f}"
|
||||
)
|
||||
|
||||
# 统计每个预测步 t 的原尺度 RMSE,并可视化;可选统计每步×每特征的RMSE热力图
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
model.eval()
|
||||
H = horizon
|
||||
# 累积每步的 MSE(原尺度)
|
||||
step_sse = np.zeros(H, dtype=np.float64)
|
||||
step_count = 0 # 累积的样本数(B*F)按 batch 聚合
|
||||
# 若需要热力图:累积 [H,F] 的平方误差和,以及样本计数(按窗口计数)
|
||||
per_feat_sse = None # shape [H,F]
|
||||
window_count = 0
|
||||
mean_t = torch.from_numpy(mean).to(device).view(1, 1, -1)
|
||||
std_t = torch.from_numpy(std).to(device).view(1, 1, -1)
|
||||
with torch.no_grad():
|
||||
for xb, yb in loader:
|
||||
xb = xb.to(device)
|
||||
yb = yb.to(device)
|
||||
pred = model(xb)
|
||||
pred_den = pred * std_t + mean_t
|
||||
yb_den = yb * std_t + mean_t
|
||||
err = pred_den - yb_den # [B,H,F]
|
||||
# 对 batch 和 feature 求均方,先累加平方和,最后再除以总数
|
||||
sse_h = (err ** 2).sum(dim=(0, 2)).detach().cpu().numpy() # [H]
|
||||
step_sse += sse_h
|
||||
step_count += (xb.size(0) * xb.size(2))
|
||||
|
||||
if args.plot_heatmap:
|
||||
sse_hf = (err ** 2).sum(dim=0).detach().cpu().numpy() # [H,F] 按 batch 合并
|
||||
if per_feat_sse is None:
|
||||
per_feat_sse = sse_hf.astype(np.float64)
|
||||
else:
|
||||
per_feat_sse += sse_hf
|
||||
window_count += xb.size(0)
|
||||
|
||||
step_mse = step_sse / max(1, step_count)
|
||||
step_rmse = np.sqrt(np.maximum(step_mse, 1e-12))
|
||||
|
||||
# 保存数据到 CSV(若 save_dir 提供)
|
||||
if args.save_dir:
|
||||
os.makedirs(args.save_dir, exist_ok=True)
|
||||
csv_path = os.path.join(args.save_dir, "per_step_rmse.csv")
|
||||
pd.DataFrame({
|
||||
"step": np.arange(1, H + 1),
|
||||
"rmse": step_rmse,
|
||||
"mse": step_mse,
|
||||
}).to_csv(csv_path, index=False)
|
||||
print(f"Saved: {csv_path}")
|
||||
|
||||
# 绘图路径
|
||||
plot_path = args.plot_path
|
||||
if not plot_path and args.save_dir:
|
||||
plot_path = os.path.join(args.save_dir, "per_step_rmse.png")
|
||||
if plot_path:
|
||||
plt.figure(figsize=(8, 4.5))
|
||||
plt.plot(np.arange(1, H + 1), step_rmse, marker="o", lw=1.5)
|
||||
plt.xlabel("Prediction step (t)")
|
||||
plt.ylabel("RMSE (original scale)")
|
||||
plt.title("Per-step RMSE vs prediction step")
|
||||
plt.grid(True, alpha=0.3)
|
||||
plt.tight_layout()
|
||||
plt.savefig(plot_path, dpi=150)
|
||||
print(f"Saved: {plot_path}")
|
||||
|
||||
# 输出热力图与CSV
|
||||
if args.plot_heatmap and per_feat_sse is not None and args.save_dir:
|
||||
os.makedirs(args.save_dir, exist_ok=True)
|
||||
per_feat_mse = per_feat_sse / max(1, window_count) # [H,F] 每步每特征 MSE(按窗口平均)
|
||||
per_feat_rmse = np.sqrt(np.maximum(per_feat_mse, 1e-12))
|
||||
# CSV:行为 step,列为各特征
|
||||
csv_hm = os.path.join(args.save_dir, "per_step_feature_rmse.csv")
|
||||
pd.DataFrame(per_feat_rmse, columns=columns, index=np.arange(1, H + 1)).to_csv(csv_hm)
|
||||
print(f"Saved: {csv_hm}")
|
||||
# 热力图
|
||||
plt.figure(figsize=(min(12, 1.5 + 0.35 * len(columns) + 6), 5))
|
||||
im = plt.imshow(per_feat_rmse, aspect="auto", origin="lower", cmap="viridis")
|
||||
plt.colorbar(im, label="RMSE")
|
||||
plt.yticks(ticks=np.arange(0, H, max(1, H // 10)), labels=(np.arange(1, H + 1)[::max(1, H // 10)]))
|
||||
plt.xticks(ticks=np.arange(len(columns)), labels=columns, rotation=45, ha="right")
|
||||
plt.xlabel("Feature")
|
||||
plt.ylabel("Prediction step (t)")
|
||||
plt.title("Per-step per-feature RMSE (original scale)")
|
||||
plt.tight_layout()
|
||||
hm_path = os.path.join(args.save_dir, "per_step_feature_rmse.png")
|
||||
plt.savefig(hm_path, dpi=150)
|
||||
print(f"Saved: {hm_path}")
|
||||
except Exception as e:
|
||||
print(f"[warn] 每步RMSE绘图失败:{e}")
|
||||
|
||||
# 可选:保存最后一个窗口的预测与真值(反标准化)
|
||||
if args.save_dir:
|
||||
os.makedirs(args.save_dir, exist_ok=True)
|
||||
last_start = int(starts[-1])
|
||||
x_last = values_norm[last_start : last_start + args.seq_len]
|
||||
y_true = values[last_start + args.seq_len : last_start + args.seq_len + horizon]
|
||||
with torch.no_grad():
|
||||
pred = (
|
||||
model(torch.from_numpy(x_last).unsqueeze(0).to(device))
|
||||
.cpu()
|
||||
.numpy()[0]
|
||||
)
|
||||
pred_denorm = pred * std + mean
|
||||
# 保存为两份 CSV:pred 和 true,行是步长 t=1..H,列为各特征名
|
||||
pred_df = pd.DataFrame(pred_denorm, columns=columns)
|
||||
true_df = pd.DataFrame(y_true, columns=columns)
|
||||
pred_path = os.path.join(args.save_dir, "last_window_pred.csv")
|
||||
true_path = os.path.join(args.save_dir, "last_window_true.csv")
|
||||
pred_df.to_csv(pred_path, index=False)
|
||||
true_df.to_csv(true_path, index=False)
|
||||
print(f"Saved: {pred_path}\nSaved: {true_path}")
|
||||
|
||||
# 可选:对某个特征画最后一个窗口的预测-真值曲线
|
||||
try:
|
||||
import matplotlib.pyplot as plt
|
||||
feat = args.plot_feature or columns[0]
|
||||
if feat not in columns:
|
||||
print(f"[warn] plot_feature='{feat}' 不在列中,使用默认 {columns[0]}")
|
||||
feat = columns[0]
|
||||
fi = columns.index(feat)
|
||||
plt.figure(figsize=(7, 4))
|
||||
plt.plot(np.arange(1, horizon + 1), pred_denorm[:, fi], label="pred", marker="o", lw=1.5)
|
||||
plt.plot(np.arange(1, horizon + 1), y_true[:, fi], label="true", marker="o", lw=1.5)
|
||||
plt.xlabel("Prediction step (t)")
|
||||
plt.ylabel(feat)
|
||||
plt.title(f"Last window prediction vs truth: {feat}")
|
||||
plt.grid(True, alpha=0.3)
|
||||
plt.legend()
|
||||
plt.tight_layout()
|
||||
line_path = os.path.join(args.save_dir, f"last_window_{feat}_pred_vs_true.png")
|
||||
plt.savefig(line_path, dpi=150)
|
||||
print(f"Saved: {line_path}")
|
||||
except Exception as e:
|
||||
print(f"[warn] 最后窗口曲线绘图失败:{e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,249 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
make_demo_video.py
|
||||
- 读取单个 CSV + 模型与标准化,进行滚动 1-step 预测,生成 10 秒预测示意视频
|
||||
- 画面:展示绿色/红色摆的真实位置(实线)与上一步预测到当前帧的预测位置(虚线)
|
||||
|
||||
使用示例:
|
||||
uv run -m chaos_pdl.make_demo_video \
|
||||
--csv_path chaos_pdl/data/metrics-20250920-000328_angles.csv \
|
||||
--model_path outputs/chaos/best_model.pt \
|
||||
--scaler_path outputs/chaos/scaler.npz \
|
||||
--seq_len 180 \
|
||||
--duration 10 \
|
||||
--fps 30 \
|
||||
--out_path outputs/chaos/demo.mp4
|
||||
|
||||
若系统未安装 ffmpeg,会自动回退为 GIF:outputs/chaos/demo.gif
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from typing import List, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.animation import FuncAnimation, FFMpegWriter, PillowWriter
|
||||
|
||||
try:
|
||||
from chaos_pdl.pendulum_transformer import TimeSeriesTransformer, WindowedTSDataset
|
||||
except ModuleNotFoundError:
|
||||
from pendulum_transformer import TimeSeriesTransformer, WindowedTSDataset
|
||||
|
||||
|
||||
def read_align_csv(path: str, expect_columns: List[str]) -> pd.DataFrame:
|
||||
df = pd.read_csv(path)
|
||||
if "frame_index" in df.columns:
|
||||
df = df.sort_values("frame_index").drop(columns=["frame_index"]) # 对齐训练逻辑
|
||||
df_num = df.select_dtypes(include=[np.number]).copy()
|
||||
has = list(df_num.columns)
|
||||
if set(has) != set(expect_columns):
|
||||
raise ValueError(
|
||||
f"CSV 数值列与训练不一致\n 期望: {expect_columns}\n 实际: {has}\n"
|
||||
)
|
||||
return df_num[expect_columns].reset_index(drop=True)
|
||||
|
||||
|
||||
def build_one_step_pred_sequence(values_norm: np.ndarray, model: TimeSeriesTransformer, device: torch.device,
|
||||
seq_len: int, frame_idx_list: np.ndarray, batch_size: int = 512) -> np.ndarray:
|
||||
"""
|
||||
对于每个目标帧 t,使用窗口 [t-L, t)(即以 t-1 为最后一帧)做前向,取 1-step 预测(pred[:, 0, :])。
|
||||
返回形状 [N, F] 的反标准化前的预测(在外部再去反标准化)。
|
||||
这里直接返回标准化空间预测(保留与训练一致的数值),由外层负责反标准化。
|
||||
"""
|
||||
model.eval()
|
||||
starts = frame_idx_list - seq_len # 每个 t 对应窗口起点(最后一帧是 t-1)
|
||||
assert np.all(starts >= 0)
|
||||
# 构造批量窗口
|
||||
preds_list = []
|
||||
with torch.no_grad():
|
||||
for i in range(0, len(starts), batch_size):
|
||||
bs = starts[i:i + batch_size]
|
||||
# 堆叠为 [B,L,F]
|
||||
batch = np.stack([values_norm[s:s + seq_len] for s in bs], axis=0).astype(np.float32)
|
||||
xb = torch.from_numpy(batch).to(device)
|
||||
out = model(xb) # [B,H,F]
|
||||
preds_list.append(out[:, 0, :].cpu().numpy()) # 取 t+1 的第一步
|
||||
return np.concatenate(preds_list, axis=0) # [N,F]
|
||||
|
||||
|
||||
def angle_from_sin_cos(sin_v: float, cos_v: float) -> float:
|
||||
return float(np.arctan2(sin_v, cos_v))
|
||||
|
||||
|
||||
def endpoint_from_angle(theta: float, length: float = 1.0) -> Tuple[float, float]:
|
||||
# 以 (0,0) 为锚点,x=sin(theta), y=-cos(theta)(让角度0指向正下)
|
||||
return (length * np.sin(theta), -length * np.cos(theta))
|
||||
|
||||
|
||||
def draw_frame(ax, true_row: np.ndarray, pred_row: np.ndarray, col_idx: dict, title: str):
|
||||
ax.clear()
|
||||
ax.set_aspect('equal', 'box')
|
||||
ax.set_xlim(-1.2, 1.2)
|
||||
ax.set_ylim(-1.2, 1.2)
|
||||
ax.grid(True, alpha=0.2)
|
||||
|
||||
# 真实角度
|
||||
g_theta = angle_from_sin_cos(true_row[col_idx['green_angle_sin']], true_row[col_idx['green_angle_cos']])
|
||||
r_theta = angle_from_sin_cos(true_row[col_idx['red_angle_sin']], true_row[col_idx['red_angle_cos']])
|
||||
gx, gy = endpoint_from_angle(g_theta)
|
||||
rx, ry = endpoint_from_angle(r_theta)
|
||||
|
||||
# 预测角度(上一步预测到当前帧)
|
||||
pg_theta = angle_from_sin_cos(pred_row[col_idx['green_angle_sin']], pred_row[col_idx['green_angle_cos']])
|
||||
pr_theta = angle_from_sin_cos(pred_row[col_idx['red_angle_sin']], pred_row[col_idx['red_angle_cos']])
|
||||
pgx, pgy = endpoint_from_angle(pg_theta)
|
||||
prx, pry = endpoint_from_angle(pr_theta)
|
||||
|
||||
# 真实(实线)
|
||||
ax.plot([0, gx], [0, gy], color='tab:green', lw=3, label='green-true')
|
||||
ax.plot([0, rx], [0, ry], color='tab:red', lw=3, label='red-true')
|
||||
ax.scatter([gx, rx], [gy, ry], color=['tab:green', 'tab:red'], s=40)
|
||||
|
||||
# 预测(虚线半透明)
|
||||
ax.plot([0, pgx], [0, pgy], color='tab:green', lw=2, ls='--', alpha=0.7, label='green-pred')
|
||||
ax.plot([0, prx], [0, pry], color='tab:red', lw=2, ls='--', alpha=0.7, label='red-pred')
|
||||
ax.scatter([pgx, prx], [pgy, pry], color=['tab:green', 'tab:red'], s=30, alpha=0.7)
|
||||
|
||||
ax.legend(loc='upper right')
|
||||
ax.set_title(title)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--csv_path', type=str, required=True, help='输入 CSV(需要包含训练使用的列)')
|
||||
ap.add_argument('--model_path', type=str, default='outputs/chaos/best_model.pt')
|
||||
ap.add_argument('--scaler_path', type=str, default='outputs/chaos/scaler.npz')
|
||||
ap.add_argument('--seq_len', type=int, default=180)
|
||||
ap.add_argument('--duration', type=int, default=10, help='视频秒数')
|
||||
ap.add_argument('--fps', type=int, default=30)
|
||||
ap.add_argument('--out_path', type=str, default='outputs/chaos/demo.mp4')
|
||||
ap.add_argument('--start_from_end', action='store_true', help='从序列末尾倒推所需时长;不设则从 seq_len 处开始')
|
||||
ap.add_argument('--batch_size', type=int, default=512)
|
||||
ap.add_argument('--orient', type=str, default='down', choices=['down', 'right'],
|
||||
help='坐标映射:down -> (x=sin(theta), y=-cos(theta)); right -> (x=cos(theta), y=sin(theta))')
|
||||
ap.add_argument('--debug_stats', action='store_true', help='打印窗口内若干关键列的最小/最大值,排查坐标恒正等问题')
|
||||
|
||||
args = ap.parse_args()
|
||||
|
||||
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
|
||||
sc = np.load(args.scaler_path, allow_pickle=True)
|
||||
mean = sc['mean'].astype(np.float32)
|
||||
std = sc['std'].astype(np.float32)
|
||||
columns = list(sc['columns'].tolist())
|
||||
col_idx = {c: i for i, c in enumerate(columns)}
|
||||
|
||||
# 加载模型
|
||||
ckpt = torch.load(args.model_path, map_location=device)
|
||||
cfg = ckpt['config']
|
||||
H = int(cfg['horizon'])
|
||||
model = TimeSeriesTransformer(
|
||||
in_features=len(columns),
|
||||
d_model=cfg['d_model'],
|
||||
nhead=cfg['nhead'],
|
||||
num_layers=cfg['num_layers'],
|
||||
dim_feedforward=cfg['dim_ff'],
|
||||
dropout=cfg['dropout'],
|
||||
horizon=H,
|
||||
).to(device)
|
||||
model.load_state_dict(ckpt['model_state'])
|
||||
model.eval()
|
||||
|
||||
# 数据
|
||||
df = read_align_csv(args.csv_path, columns)
|
||||
values = df.values.astype(np.float32)
|
||||
T = len(values)
|
||||
if T < args.seq_len + 2:
|
||||
raise ValueError(f'数据太短:需要至少 {args.seq_len + 2} 行,当前 {T}')
|
||||
|
||||
# 选择时间范围
|
||||
N_frames = args.duration * args.fps
|
||||
if args.start_from_end: # 从末尾倒推
|
||||
end_t = T - 1
|
||||
start_t = max(args.seq_len, end_t - N_frames + 1)
|
||||
else:
|
||||
start_t = args.seq_len
|
||||
end_t = min(T - 1, start_t + N_frames - 1)
|
||||
frame_ts = np.arange(start_t, end_t + 1, dtype=np.int64)
|
||||
N_frames = len(frame_ts)
|
||||
|
||||
# 标准化
|
||||
values_norm = (values - mean) / std
|
||||
|
||||
# 为每个 t 进行 1-step 预测:窗口是 [t-L, t),目标是 t
|
||||
preds_norm = build_one_step_pred_sequence(values_norm, model, device, args.seq_len, frame_ts, args.batch_size)
|
||||
preds = preds_norm * std + mean # 反标准化
|
||||
|
||||
# 真实对应行
|
||||
truths = values[frame_ts]
|
||||
|
||||
# 可选调试:打印该时段关键列范围
|
||||
if args.debug_stats:
|
||||
def rng(name):
|
||||
i = col_idx[name]
|
||||
return float(truths[:, i].min()), float(truths[:, i].max())
|
||||
keys = [
|
||||
'green_angle_sin','green_angle_cos','red_angle_sin','red_angle_cos'
|
||||
]
|
||||
avail = [k for k in keys if k in col_idx]
|
||||
for k in avail:
|
||||
lo, hi = rng(k)
|
||||
print(f"[debug] {k}: min={lo:.4f}, max={hi:.4f}")
|
||||
|
||||
# 准备动画
|
||||
fig, ax = plt.subplots(figsize=(5, 5))
|
||||
|
||||
def update(i):
|
||||
t = frame_ts[i]
|
||||
title = f't={int(t)} (frame {i+1}/{N_frames})'
|
||||
# 根据 orient 选择坐标映射
|
||||
if args.orient == 'down':
|
||||
draw_frame(ax, truths[i], preds[i], col_idx, title)
|
||||
else: # 'right'
|
||||
def angle_from_sc(row, which):
|
||||
return float(np.arctan2(row[col_idx[f'{which}_angle_sin']], row[col_idx[f'{which}_angle_cos']]))
|
||||
def xy_from_theta(theta):
|
||||
return (np.cos(theta), np.sin(theta))
|
||||
ax.clear()
|
||||
ax.set_aspect('equal', 'box')
|
||||
ax.set_xlim(-1.2, 1.2)
|
||||
ax.set_ylim(-1.2, 1.2)
|
||||
ax.grid(True, alpha=0.2)
|
||||
# 真实
|
||||
g_t = angle_from_sc(truths[i], 'green'); r_t = angle_from_sc(truths[i], 'red')
|
||||
gx, gy = xy_from_theta(g_t); rx, ry = xy_from_theta(r_t)
|
||||
# 预测
|
||||
pg_t = angle_from_sc(preds[i], 'green'); pr_t = angle_from_sc(preds[i], 'red')
|
||||
pgx, pgy = xy_from_theta(pg_t); prx, pry = xy_from_theta(pr_t)
|
||||
ax.plot([0, gx],[0, gy], color='tab:green', lw=3, label='green-true')
|
||||
ax.plot([0, rx],[0, ry], color='tab:red', lw=3, label='red-true')
|
||||
ax.scatter([gx, rx],[gy, ry], color=['tab:green','tab:red'], s=40)
|
||||
ax.plot([0, pgx],[0, pgy], color='tab:green', lw=2, ls='--', alpha=0.7, label='green-pred')
|
||||
ax.plot([0, prx],[0, pry], color='tab:red', lw=2, ls='--', alpha=0.7, label='red-pred')
|
||||
ax.scatter([pgx, prx],[pgy, pry], color=['tab:green','tab:red'], s=30, alpha=0.7)
|
||||
ax.legend(loc='upper right')
|
||||
ax.set_title(title)
|
||||
return []
|
||||
|
||||
anim = FuncAnimation(fig, update, frames=N_frames, interval=1000.0 / args.fps, blit=False)
|
||||
|
||||
# 存储
|
||||
os.makedirs(os.path.dirname(args.out_path), exist_ok=True)
|
||||
try:
|
||||
writer = FFMpegWriter(fps=args.fps, bitrate=1800)
|
||||
anim.save(args.out_path, writer=writer)
|
||||
print(f'Saved video: {args.out_path}')
|
||||
except Exception as e:
|
||||
# 回退为 GIF
|
||||
gif_path = os.path.splitext(args.out_path)[0] + '.gif'
|
||||
print(f'[warn] 写入 mp4 失败(可能缺少 ffmpeg):{e}\n回退为 GIF -> {gif_path}')
|
||||
writer = PillowWriter(fps=args.fps)
|
||||
anim.save(gif_path, writer=writer)
|
||||
print(f'Saved GIF: {gif_path}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
批量处理 CSV:
|
||||
- 从 CSV(支持多个路径或通配符)读取中国语系 GB2312 编码数据;
|
||||
- 对黄色/红色圆心序列分别做最小二乘圆拟合;
|
||||
- 计算角度的 sin/cos 以及按拟合圆归一化后的坐标(pos_sin/pos_cos);
|
||||
- 导出 UTF-8 CSV 到指定输出目录;
|
||||
- 可选保存拟合示意图 PNG。
|
||||
|
||||
用法示例:
|
||||
1) 处理一个/多个文件:
|
||||
python circle_fit_batch.py data/3.csv data/4.csv -o outputs
|
||||
|
||||
2) 使用通配符批量处理:
|
||||
python circle_fit_batch.py "data/*.csv" -o outputs
|
||||
|
||||
3) 指定不同编码或禁用绘图:
|
||||
python circle_fit_batch.py "data/*.csv" -o outputs --encoding gb2312 --no-plot
|
||||
|
||||
4) 指定中文字体文件用于图例(可选):
|
||||
python circle_fit_batch.py "data/*.csv" -o outputs --font "/path/to/PingFang Regular.ttf"
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
from typing import Tuple, List
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
# 后端设为无界面,以便在服务器/批处理环境下保存图像
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib import font_manager
|
||||
|
||||
REQUIRED_COLS = [
|
||||
"帧序号",
|
||||
"黄色与水平线夹角(度)",
|
||||
"红色与水平线夹角(度)",
|
||||
"黄色外接圆圆心X",
|
||||
"黄色外接圆圆心Y",
|
||||
"红色相关外接圆圆心X",
|
||||
"红色相关外接圆圆心Y",
|
||||
]
|
||||
|
||||
|
||||
def fit_circle_lstsq(x: np.ndarray, y: np.ndarray) -> Tuple[float, float, float]:
|
||||
"""最小二乘拟合圆:解 x^2 + y^2 = 2 a x + 2 b y + c
|
||||
返回 (cx, cy, r)
|
||||
"""
|
||||
x = np.asarray(x, dtype=np.float64)
|
||||
y = np.asarray(y, dtype=np.float64)
|
||||
A = np.c_[2 * x, 2 * y, np.ones_like(x)]
|
||||
b = x * x + y * y
|
||||
sol, *_ = np.linalg.lstsq(A, b, rcond=None)
|
||||
a, b_, c = sol
|
||||
cx, cy = a, b_
|
||||
r = math.sqrt(max(cx * cx + cy * cy + c, 0.0))
|
||||
return float(cx), float(cy), float(r)
|
||||
|
||||
|
||||
def deg_to_cos_sin(deg: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""角度(度) -> (cos, sin)"""
|
||||
rad = np.radians(deg.astype(np.float64))
|
||||
return np.cos(rad), np.sin(rad)
|
||||
|
||||
|
||||
def expand_inputs(inputs: List[str]) -> List[str]:
|
||||
files: List[str] = []
|
||||
for pat in inputs:
|
||||
matched = glob.glob(pat)
|
||||
if matched:
|
||||
files.extend(matched)
|
||||
else:
|
||||
# 若未匹配到且参数本身是一个存在的文件,也加入
|
||||
if os.path.isfile(pat):
|
||||
files.append(pat)
|
||||
# 去重并排序,便于稳定输出
|
||||
files = sorted(set(files))
|
||||
return files
|
||||
|
||||
|
||||
def ensure_output_dir(out_dir: str) -> None:
|
||||
if not os.path.isdir(out_dir):
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
|
||||
def process_one_csv(csv_path: str, out_dir: str, encoding: str, plot: bool, font_path: str = None, overwrite: bool = False) -> None:
|
||||
base = os.path.splitext(os.path.basename(csv_path))[0]
|
||||
out_csv = os.path.join(out_dir, f"{base}_angles.csv")
|
||||
out_png = os.path.join(out_dir, f"{base}_circle_fit.png")
|
||||
|
||||
if not overwrite and os.path.exists(out_csv):
|
||||
print(f"[SKIP] 输出已存在(使用 --overwrite 覆盖):{out_csv}")
|
||||
return
|
||||
|
||||
try:
|
||||
df = pd.read_csv(csv_path, encoding=encoding)
|
||||
except Exception as e:
|
||||
print(f"[ERROR] 读取失败:{csv_path} ({e})")
|
||||
return
|
||||
|
||||
missing = [c for c in REQUIRED_COLS if c not in df.columns]
|
||||
if missing:
|
||||
print(f"[ERROR] 列缺失:{csv_path} 缺少 {missing}")
|
||||
return
|
||||
|
||||
# 提取列
|
||||
yellow_x = df["黄色外接圆圆心X"].astype(float).to_numpy()
|
||||
yellow_y = df["黄色外接圆圆心Y"].astype(float).to_numpy()
|
||||
red_x = df["红色相关外接圆圆心X"].astype(float).to_numpy()
|
||||
red_y = df["红色相关外接圆圆心Y"].astype(float).to_numpy()
|
||||
|
||||
# 圆拟合
|
||||
ycx, ycy, yr = fit_circle_lstsq(yellow_x, yellow_y)
|
||||
rcx, rcy, rr = fit_circle_lstsq(red_x, red_y)
|
||||
|
||||
if yr <= 0 or rr <= 0 or not np.isfinite([yr, rr]).all():
|
||||
print(f"[ERROR] 拟合半径非法(<=0 或 非数),文件:{csv_path}")
|
||||
return
|
||||
|
||||
# 角度 -> cos/sin
|
||||
yellow_angle_deg = pd.to_numeric(df["黄色与水平线夹角(度)"], errors="coerce").to_numpy()
|
||||
red_angle_deg = pd.to_numeric(df["红色与水平线夹角(度)"], errors="coerce").to_numpy()
|
||||
yellow_angle_cos, yellow_angle_sin = deg_to_cos_sin(yellow_angle_deg)
|
||||
red_angle_cos, red_angle_sin = deg_to_cos_sin(red_angle_deg)
|
||||
|
||||
# 位置按拟合圆归一化(cosx=沿X方向的归一化偏移;cosy=沿Y方向的归一化偏移)
|
||||
yellow_cosx = (yellow_x - ycx) / yr
|
||||
yellow_cosy = (yellow_y - ycy) / yr
|
||||
red_cosx = (red_x - rcx) / rr
|
||||
red_cosy = (red_y - rcy) / rr
|
||||
|
||||
out_df = pd.DataFrame({
|
||||
"frame_index": df["帧序号"],
|
||||
"yellow_angle_sin": yellow_angle_sin,
|
||||
"yellow_angle_cos": yellow_angle_cos,
|
||||
"red_angle_sin": red_angle_sin,
|
||||
"red_angle_cos": red_angle_cos,
|
||||
"yellow_pos_sin": yellow_cosy,
|
||||
"yellow_pos_cos": yellow_cosx,
|
||||
"red_pos_sin": red_cosy,
|
||||
"red_pos_cos": red_cosx,
|
||||
})
|
||||
|
||||
try:
|
||||
out_df.to_csv(out_csv, index=False, encoding="utf-8")
|
||||
print(f"[OK] 导出:{out_csv} | 黄圆 (cx,cy,r)=({ycx:.6g},{ycy:.6g},{yr:.6g}) 红圆 (cx,cy,r)=({rcx:.6g},{rcy:.6g},{rr:.6g})")
|
||||
except Exception as e:
|
||||
print(f"[ERROR] 写出 CSV 失败:{out_csv} ({e})")
|
||||
|
||||
if plot:
|
||||
# 字体(可选)
|
||||
if font_path:
|
||||
try:
|
||||
font_manager.fontManager.addfont(font_path)
|
||||
# 若传入中文字体,则优先用它显示中文
|
||||
font_name = os.path.splitext(os.path.basename(font_path))[0]
|
||||
plt.rcParams["font.sans-serif"] = [font_name]
|
||||
except Exception as e:
|
||||
print(f"[WARN] 字体加载失败:{font_path} ({e})")
|
||||
|
||||
fig, ax = plt.subplots()
|
||||
ax.set_aspect("equal", "box")
|
||||
ax.scatter(yellow_x, yellow_y, s=1, label="黄色")
|
||||
ax.scatter(red_x, red_y, s=1, label="红色")
|
||||
|
||||
circle1 = plt.Circle((ycx, ycy), yr, color="orange", fill=False, label="拟合黄色圆")
|
||||
circle2 = plt.Circle((rcx, rcy), rr, color="red", fill=False, label="拟合红色圆")
|
||||
ax.add_artist(circle1)
|
||||
ax.add_artist(circle2)
|
||||
ax.legend()
|
||||
|
||||
try:
|
||||
fig.savefig(out_png, dpi=150, bbox_inches="tight")
|
||||
print(f"[OK] 拟合图:{out_png}")
|
||||
finally:
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser(description="批量对 CSV 进行圆拟合与角度/位置归一化导出")
|
||||
p.add_argument("inputs", nargs="+", help="输入 CSV 路径或通配符(可多个),例如 data/3.csv 或 \"data/*.csv\"")
|
||||
p.add_argument("-o", "--output-dir", required=True, help="输出目录,将在其中生成 *_angles.csv 与 *_circle_fit.png")
|
||||
p.add_argument("--encoding", default="gb2312", help="输入 CSV 编码(默认 gb2312)")
|
||||
p.add_argument("--no-plot", action="store_true", help="不保存拟合示意图 PNG")
|
||||
p.add_argument("--font", default=None, help="可选:中文字体文件路径(用于图例/标签显示中文)")
|
||||
p.add_argument("--overwrite", action="store_true", help="若输出文件存在则覆盖")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
files = expand_inputs(args.inputs)
|
||||
if not files:
|
||||
print("[ERROR] 未找到任何输入文件。请检查路径/通配符。", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
ensure_output_dir(args.output_dir)
|
||||
|
||||
print(f"共 {len(files)} 个文件,将输出到:{args.output_dir}")
|
||||
for fp in files:
|
||||
print(f"==> 处理:{fp}")
|
||||
process_one_csv(
|
||||
csv_path=fp,
|
||||
out_dir=args.output_dir,
|
||||
encoding=args.encoding,
|
||||
plot=(not args.no_plot),
|
||||
font_path=args.font,
|
||||
overwrite=args.overwrite,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
批量处理 CSV:
|
||||
- 从 CSV(支持多个路径或通配符)读取中国语系 GB2312 编码数据;
|
||||
- 对黄色/红色圆心序列分别做最小二乘圆拟合;
|
||||
- 计算角度的 sin/cos 以及按拟合圆归一化后的坐标(pos_sin/pos_cos);
|
||||
- 导出 UTF-8 CSV 到指定输出目录;
|
||||
- 可选保存拟合示意图 PNG。
|
||||
|
||||
用法示例:
|
||||
1) 处理一个/多个文件:
|
||||
python circle_fit_batch.py data/3.csv data/4.csv -o outputs
|
||||
|
||||
2) 使用通配符批量处理:
|
||||
python circle_fit_batch.py "data/*.csv" -o outputs
|
||||
|
||||
3) 指定不同编码或禁用绘图:
|
||||
python circle_fit_batch.py "data/*.csv" -o outputs --encoding gb2312 --no-plot
|
||||
|
||||
4) 指定中文字体文件用于图例(可选):
|
||||
python circle_fit_batch.py "data/*.csv" -o outputs --font "/path/to/PingFang Regular.ttf"
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
from typing import Tuple, List
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
# 后端设为无界面,以便在服务器/批处理环境下保存图像
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib import font_manager
|
||||
|
||||
REQUIRED_COLS = [
|
||||
"frameIndex",
|
||||
"time",
|
||||
"greenRectRotation",
|
||||
"blueRectRotation",
|
||||
"greenRectCenterX",
|
||||
"greenRectCenterY",
|
||||
"blueRectCenterX",
|
||||
"blueRectCenterY",
|
||||
]
|
||||
|
||||
|
||||
def fit_circle_lstsq(x: np.ndarray, y: np.ndarray) -> Tuple[float, float, float]:
|
||||
"""最小二乘拟合圆:解 x^2 + y^2 = 2 a x + 2 b y + c
|
||||
返回 (cx, cy, r)
|
||||
"""
|
||||
x = np.asarray(x, dtype=np.float64)
|
||||
y = np.asarray(y, dtype=np.float64)
|
||||
A = np.c_[2 * x, 2 * y, np.ones_like(x)]
|
||||
b = x * x + y * y
|
||||
sol, *_ = np.linalg.lstsq(A, b, rcond=None)
|
||||
a, b_, c = sol
|
||||
cx, cy = a, b_
|
||||
r = math.sqrt(max(cx * cx + cy * cy + c, 0.0))
|
||||
return float(cx), float(cy), float(r)
|
||||
|
||||
|
||||
def deg_to_cos_sin(deg: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""角度(度) -> (cos, sin)"""
|
||||
rad = np.radians(deg.astype(np.float64))
|
||||
return np.cos(rad), np.sin(rad)
|
||||
|
||||
|
||||
def expand_inputs(inputs: List[str]) -> List[str]:
|
||||
files: List[str] = []
|
||||
for pat in inputs:
|
||||
matched = glob.glob(pat)
|
||||
if matched:
|
||||
files.extend(matched)
|
||||
else:
|
||||
# 若未匹配到且参数本身是一个存在的文件,也加入
|
||||
if os.path.isfile(pat):
|
||||
files.append(pat)
|
||||
# 去重并排序,便于稳定输出
|
||||
files = sorted(set(files))
|
||||
return files
|
||||
|
||||
def ensure_output_dir(out_dir: str) -> None:
|
||||
if not os.path.isdir(out_dir):
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
|
||||
def process_one_csv(csv_path: str, out_dir: str, encoding: str, plot: bool, font_path: str = None, overwrite: bool = False) -> None:
|
||||
base = os.path.splitext(os.path.basename(csv_path))[0]
|
||||
out_csv = os.path.join(out_dir, f"{base}_angles.csv")
|
||||
out_png = os.path.join(out_dir, f"{base}_circle_fit.png")
|
||||
|
||||
if not overwrite and os.path.exists(out_csv):
|
||||
print(f"[SKIP] 输出已存在(使用 --overwrite 覆盖):{out_csv}")
|
||||
return
|
||||
|
||||
try:
|
||||
df = pd.read_csv(csv_path, encoding=encoding)
|
||||
except Exception as e:
|
||||
print(f"[ERROR] 读取失败:{csv_path} ({e})")
|
||||
return
|
||||
|
||||
missing = [c for c in REQUIRED_COLS if c not in df.columns]
|
||||
if missing:
|
||||
print(f"[ERROR] 列缺失:{csv_path} 缺少 {missing}")
|
||||
return
|
||||
|
||||
# 将必要列转为数值,无法解析的置为 NaN
|
||||
for c in REQUIRED_COLS:
|
||||
df[c] = pd.to_numeric(df[c], errors="coerce")
|
||||
|
||||
# 逐帧过滤:若任一必要列缺失/非数,则剔除该帧
|
||||
valid_mask = df[REQUIRED_COLS].notna().all(axis=1)
|
||||
dropped = int(len(df) - valid_mask.sum())
|
||||
df_valid = df.loc[valid_mask].reset_index(drop=True)
|
||||
if dropped > 0:
|
||||
print(f"[INFO] 已过滤 {dropped} 帧(必要列缺失),保留 {len(df_valid)} 帧:{csv_path}")
|
||||
|
||||
# 至少需要 2 帧来计算 deltaTime
|
||||
if len(df_valid) < 2:
|
||||
print(f"[ERROR] 有效帧不足2帧,无法计算deltaTime:{csv_path}")
|
||||
return
|
||||
|
||||
# 基于过滤后的时间序列计算 deltaTime;输出与其它列对齐(从第2帧开始)
|
||||
time = df_valid["time"].to_numpy(dtype=float)
|
||||
delta_time = np.diff(time)
|
||||
|
||||
# 从第2帧开始提取所有数据(与 delta_time 对齐)
|
||||
green_x = df_valid["greenRectCenterX"].to_numpy(dtype=float)[1:]
|
||||
green_y = df_valid["greenRectCenterY"].to_numpy(dtype=float)[1:]
|
||||
red_x = df_valid["blueRectCenterX"].to_numpy(dtype=float)[1:]
|
||||
red_y = df_valid["blueRectCenterY"].to_numpy(dtype=float)[1:]
|
||||
|
||||
# 圆拟合
|
||||
ycx, ycy, yr = fit_circle_lstsq(green_x, green_y)
|
||||
rcx, rcy, rr = fit_circle_lstsq(red_x, red_y)
|
||||
|
||||
if yr <= 0 or rr <= 0 or not np.isfinite([yr, rr]).all():
|
||||
print(f"[ERROR] 拟合半径非法(<=0 或 非数),文件:{csv_path}")
|
||||
return
|
||||
|
||||
# 角度 -> cos/sin (从第2帧开始)
|
||||
green_angle_deg = df_valid["greenRectRotation"].to_numpy(dtype=float)[1:]
|
||||
red_angle_deg = df_valid["blueRectRotation"].to_numpy(dtype=float)[1:]
|
||||
green_angle_cos, green_angle_sin = deg_to_cos_sin(green_angle_deg)
|
||||
red_angle_cos, red_angle_sin = deg_to_cos_sin(red_angle_deg)
|
||||
|
||||
# 位置按拟合圆归一化(cosx=沿X方向的归一化偏移;cosy=沿Y方向的归一化偏移)
|
||||
green_cosx = (green_x - ycx) / yr
|
||||
green_cosy = (green_y - ycy) / yr
|
||||
red_cosx = (red_x - rcx) / rr
|
||||
red_cosy = (red_y - rcy) / rr
|
||||
|
||||
out_df = pd.DataFrame({
|
||||
"frame_index": df_valid["frameIndex"].to_numpy()[1:], # 从第2帧开始
|
||||
"delta_time": delta_time,
|
||||
"green_angle_sin": green_angle_sin,
|
||||
"green_angle_cos": green_angle_cos,
|
||||
"red_angle_sin": red_angle_sin,
|
||||
"red_angle_cos": red_angle_cos,
|
||||
"green_pos_sin": green_cosy,
|
||||
"green_pos_cos": green_cosx,
|
||||
"red_pos_sin": red_cosy,
|
||||
"red_pos_cos": red_cosx,
|
||||
})
|
||||
|
||||
try:
|
||||
out_df.to_csv(out_csv, index=False, encoding="utf-8")
|
||||
print(f"[OK] 导出:{out_csv} | 黄圆 (cx,cy,r)=({ycx:.6g},{ycy:.6g},{yr:.6g}) 红圆 (cx,cy,r)=({rcx:.6g},{rcy:.6g},{rr:.6g})")
|
||||
except Exception as e:
|
||||
print(f"[ERROR] 写出 CSV 失败:{out_csv} ({e})")
|
||||
|
||||
if plot:
|
||||
# 字体(可选)
|
||||
if font_path:
|
||||
try:
|
||||
font_manager.fontManager.addfont(font_path)
|
||||
# 若传入中文字体,则优先用它显示中文
|
||||
font_name = os.path.splitext(os.path.basename(font_path))[0]
|
||||
plt.rcParams["font.sans-serif"] = [font_name]
|
||||
except Exception as e:
|
||||
print(f"[WARN] 字体加载失败:{font_path} ({e})")
|
||||
|
||||
fig, ax = plt.subplots()
|
||||
ax.set_aspect("equal", "box")
|
||||
ax.scatter(green_x, green_y, s=1, label="黄色")
|
||||
ax.scatter(red_x, red_y, s=1, label="红色")
|
||||
|
||||
circle1 = plt.Circle((ycx, ycy), yr, color="orange", fill=False, label="拟合黄色圆")
|
||||
circle2 = plt.Circle((rcx, rcy), rr, color="red", fill=False, label="拟合红色圆")
|
||||
ax.add_artist(circle1)
|
||||
ax.add_artist(circle2)
|
||||
ax.legend()
|
||||
|
||||
try:
|
||||
fig.savefig(out_png, dpi=150, bbox_inches="tight")
|
||||
print(f"[OK] 拟合图:{out_png}")
|
||||
finally:
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser(description="批量对 CSV 进行圆拟合与角度/位置归一化导出")
|
||||
p.add_argument("inputs", nargs="+", help="输入 CSV 路径或通配符(可多个),例如 data/3.csv 或 \"data/*.csv\"")
|
||||
p.add_argument("-o", "--output-dir", required=True, help="输出目录,将在其中生成 *_angles.csv 与 *_circle_fit.png")
|
||||
p.add_argument("--encoding", default="utf-8", help="输入 CSV 编码(默认 utf-8)")
|
||||
p.add_argument("--no-plot", action="store_true", help="不保存拟合示意图 PNG")
|
||||
p.add_argument("--font", default=None, help="可选:中文字体文件路径(用于图例/标签显示中文)")
|
||||
p.add_argument("--overwrite", action="store_true", help="若输出文件存在则覆盖")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
files = expand_inputs(args.inputs)
|
||||
if not files:
|
||||
print("[ERROR] 未找到任何输入文件。请检查路径/通配符。", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
ensure_output_dir(args.output_dir)
|
||||
|
||||
print(f"共 {len(files)} 个文件,将输出到:{args.output_dir}")
|
||||
for fp in files:
|
||||
print(f"==> 处理:{fp}")
|
||||
process_one_csv(
|
||||
csv_path=fp,
|
||||
out_dir=args.output_dir,
|
||||
encoding=args.encoding,
|
||||
plot=(not args.no_plot),
|
||||
font_path=args.font,
|
||||
overwrite=args.overwrite,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,448 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
train_ts_transformer_multi.py
|
||||
- 同时读取多个 CSV(列名需一致;若存在 frame_index 会按其排序并移除)
|
||||
- Transformer Encoder 进行多步预测(默认输入 240 帧,预测 30 帧)
|
||||
- 训练/验证集“均匀交错抽样”划分,可设置 val_gap 减少邻近泄漏
|
||||
- 标准化仅用训练窗口拟合,避免验证泄漏
|
||||
- 损失:按预测步 t 递减权重的加权 RMSE(支持指数/线性衰减)
|
||||
- 保存 best_model.pt 与 scaler.npz
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import math
|
||||
import random
|
||||
from typing import List, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
|
||||
|
||||
def set_seed(seed: int = 42):
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
|
||||
class WindowedTSDataset(Dataset):
|
||||
def __init__(self, X_norm: np.ndarray, starts: np.ndarray, seq_len: int, horizon: int):
|
||||
self.X = X_norm.astype(np.float32) # [T_total, F]
|
||||
self.starts = starts.astype(np.int64) # 全局起点索引(已处理多文件偏移)
|
||||
self.seq_len = seq_len
|
||||
self.horizon = horizon
|
||||
|
||||
def __len__(self):
|
||||
return len(self.starts)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
s = int(self.starts[idx])
|
||||
x = self.X[s: s + self.seq_len] # [L, F]
|
||||
y = self.X[s + self.seq_len: s + self.seq_len + self.horizon] # [H, F]
|
||||
return torch.from_numpy(x), torch.from_numpy(y)
|
||||
|
||||
|
||||
class PositionalEncoding(nn.Module):
|
||||
def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 10000):
|
||||
super().__init__()
|
||||
self.dropout = nn.Dropout(p=dropout)
|
||||
pe = torch.zeros(max_len, d_model)
|
||||
position = torch.arange(0, max_len, dtype=torch.float32).unsqueeze(1)
|
||||
div_term = torch.exp(
|
||||
torch.arange(0, d_model, 2, dtype=torch.float32) * (-math.log(10000.0) / d_model)
|
||||
)
|
||||
pe[:, 0::2] = torch.sin(position * div_term)
|
||||
pe[:, 1::2] = torch.cos(position * div_term)
|
||||
self.register_buffer("pe", pe.unsqueeze(1)) # [max_len, 1, d_model]
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
# x: [S, N, E]
|
||||
x = x + self.pe[: x.size(0)]
|
||||
return self.dropout(x)
|
||||
|
||||
|
||||
class TimeSeriesTransformer(nn.Module):
|
||||
def __init__(self, in_features: int, d_model: int, nhead: int, num_layers: int,
|
||||
dim_feedforward: int, dropout: float, horizon: int):
|
||||
super().__init__()
|
||||
self.in_features = in_features
|
||||
self.horizon = horizon
|
||||
self.input_proj = nn.Linear(in_features, d_model)
|
||||
self.pos_enc = PositionalEncoding(d_model, dropout=dropout)
|
||||
enc_layer = nn.TransformerEncoderLayer(
|
||||
d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward,
|
||||
dropout=dropout, batch_first=False, norm_first=True
|
||||
)
|
||||
self.encoder = nn.TransformerEncoder(enc_layer, num_layers=num_layers)
|
||||
self.head = nn.Sequential(
|
||||
nn.LayerNorm(d_model),
|
||||
nn.Linear(d_model, horizon * in_features),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
# x: [B, L, F]
|
||||
B, L, F = x.shape
|
||||
h = self.input_proj(x).transpose(0, 1) # [L, B, d]
|
||||
h = self.pos_enc(h)
|
||||
h = self.encoder(h) # [L, B, d]
|
||||
h_last = h[-1] # [B, d]
|
||||
y = self.head(h_last).view(B, self.horizon, self.in_features) # [B, H, F]
|
||||
return y
|
||||
|
||||
|
||||
def build_windows_indices(T: int, seq_len: int, horizon: int, stride: int) -> np.ndarray:
|
||||
max_start = T - seq_len - horizon
|
||||
if max_start < 0:
|
||||
return np.array([], dtype=np.int64)
|
||||
return np.arange(0, max_start + 1, stride, dtype=np.int64)
|
||||
|
||||
|
||||
def interleaved_split(starts: np.ndarray, val_ratio: float, val_gap: int = 0) -> Tuple[np.ndarray, np.ndarray]:
|
||||
assert 0.0 < val_ratio < 1.0
|
||||
total = len(starts)
|
||||
if total == 0:
|
||||
return np.array([], dtype=np.int64), np.array([], dtype=np.int64)
|
||||
every = max(int(round(1.0 / val_ratio)), 2)
|
||||
# 安全裁剪 val_gap:若 2*val_gap+1 >= every,会把所有索引都覆盖掉
|
||||
safe_gap = int(min(max((every - 2) // 2, 0), val_gap))
|
||||
if safe_gap != val_gap:
|
||||
print(f"[warn] val_gap={val_gap} 过大,按交错周期 every={every} 自动裁剪为 {safe_gap},以避免训练集为空。")
|
||||
idx_all = np.arange(total, dtype=np.int64)
|
||||
val_mask = (idx_all % every) == 0
|
||||
val_idx = idx_all[val_mask]
|
||||
train_mask = ~val_mask
|
||||
if safe_gap > 0:
|
||||
drop = set()
|
||||
for v in val_idx:
|
||||
lo = max(0, v - safe_gap)
|
||||
hi = min(total - 1, v + safe_gap)
|
||||
for t in range(lo, hi + 1):
|
||||
drop.add(t)
|
||||
if drop:
|
||||
train_mask[list(drop)] = False
|
||||
train_idx = idx_all[train_mask]
|
||||
return starts[train_idx], starts[val_idx]
|
||||
|
||||
|
||||
def fit_scaler_from_windows(X: np.ndarray, train_starts: np.ndarray, seq_len: int) -> Tuple[np.ndarray, np.ndarray]:
|
||||
chunks = [X[s:s + seq_len] for s in train_starts]
|
||||
if not chunks:
|
||||
raise ValueError("训练窗口为空,无法拟合标准化。")
|
||||
arr = np.concatenate(chunks, axis=0) # [N_train*L, F]
|
||||
mean = arr.mean(axis=0)
|
||||
std = arr.std(axis=0)
|
||||
std[std < 1e-6] = 1e-6
|
||||
return mean, std
|
||||
|
||||
|
||||
class WeightedRMSELoss(nn.Module):
|
||||
"""
|
||||
对预测步 t 使用递减权重 w_t;支持:
|
||||
- 指数衰减: w_t = gamma^t
|
||||
- 线性衰减: w_t = 1 - alpha * t, 直到 >= min_w
|
||||
返回整体 RMSE 标量(可反传)
|
||||
"""
|
||||
def __init__(self, horizon: int, mode: str = "exp",
|
||||
gamma: float = 0.97, alpha: float = 0.0, min_w: float = 0.05):
|
||||
super().__init__()
|
||||
self.horizon = horizon
|
||||
self.mode = mode
|
||||
self.gamma = gamma
|
||||
self.alpha = alpha
|
||||
self.min_w = min_w
|
||||
self.register_buffer("w", self._make_weights())
|
||||
|
||||
def _make_weights(self) -> torch.Tensor:
|
||||
if self.mode == "exp":
|
||||
w = torch.tensor([self.gamma ** t for t in range(self.horizon)], dtype=torch.float32)
|
||||
elif self.mode == "linear":
|
||||
w = torch.tensor([max(1.0 - self.alpha * t, self.min_w) for t in range(self.horizon)], dtype=torch.float32)
|
||||
else:
|
||||
raise ValueError("mode 必须是 'exp' 或 'linear'")
|
||||
# 归一化到均值为 1(或 sum==H),让数值尺度稳定
|
||||
w = w * (self.horizon / (w.sum() + 1e-12))
|
||||
return w # [H]
|
||||
|
||||
def forward(self, pred: torch.Tensor, target: torch.Tensor):
|
||||
# pred,target: [B,H,F]
|
||||
if pred.shape != target.shape:
|
||||
raise ValueError("pred/target 形状不一致")
|
||||
B, H, F = pred.shape
|
||||
if H != self.horizon:
|
||||
raise ValueError("horizon 不匹配")
|
||||
err2 = (pred - target) ** 2 # [B,H,F]
|
||||
w = self.w.view(1, H, 1) # [1,H,1]
|
||||
w_err = err2 * w
|
||||
mse = w_err.sum() / (w.sum() * B * F) # 加权 MSE
|
||||
rmse = torch.sqrt(mse + 1e-12) # RMSE
|
||||
return rmse
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def evaluate_losses(model, loader, device, weighted_rmse: WeightedRMSELoss):
|
||||
model.eval()
|
||||
total_wrmse, total_mse = 0.0, 0.0
|
||||
count = 0
|
||||
mse_fn = nn.MSELoss()
|
||||
for xb, yb in loader:
|
||||
xb = xb.to(device)
|
||||
yb = yb.to(device)
|
||||
pred = model(xb)
|
||||
total_wrmse += weighted_rmse(pred, yb).item() * xb.size(0)
|
||||
total_mse += mse_fn(pred, yb).item() * xb.size(0)
|
||||
count += xb.size(0)
|
||||
return total_wrmse / max(1, count), total_mse / max(1, count)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def mae_in_original_scale(model, loader, device, mean: np.ndarray, std: np.ndarray):
|
||||
model.eval()
|
||||
total_mae, count = 0.0, 0
|
||||
mean_t = torch.from_numpy(mean).to(device).view(1, 1, -1)
|
||||
std_t = torch.from_numpy(std).to(device).view(1, 1, -1)
|
||||
for xb, yb in loader:
|
||||
xb = xb.to(device); yb = yb.to(device)
|
||||
pred = model(xb)
|
||||
pred_denorm = pred * std_t + mean_t
|
||||
yb_denorm = yb * std_t + mean_t
|
||||
mae = torch.abs(pred_denorm - yb_denorm).mean()
|
||||
total_mae += mae.item() * xb.size(0)
|
||||
count += xb.size(0)
|
||||
return total_mae / max(1, count)
|
||||
|
||||
|
||||
def read_and_align_csvs(paths: List[str]) -> pd.DataFrame:
|
||||
"""
|
||||
读取多个 CSV,确保数值列集合与顺序一致;若存在 frame_index 列按其排序并移除。
|
||||
将各文件按行拼接,返回对齐后的整表(仅数值列)。
|
||||
同时返回每个段落的起止行以备切片(通过 offsets)。
|
||||
"""
|
||||
numeric_cols_ref = None
|
||||
dfs = []
|
||||
for p in paths:
|
||||
df = pd.read_csv(p)
|
||||
if "frame_index" in df.columns:
|
||||
df = df.sort_values("frame_index").drop(columns=["frame_index"])
|
||||
df_num = df.select_dtypes(include=[np.number]).copy()
|
||||
if df_num.shape[1] == 0:
|
||||
raise ValueError(f"{p} 中没有数值列。")
|
||||
cols = list(df_num.columns)
|
||||
if numeric_cols_ref is None:
|
||||
numeric_cols_ref = cols
|
||||
else:
|
||||
if cols != numeric_cols_ref:
|
||||
raise ValueError(
|
||||
f"列不一致:\n 参考列: {numeric_cols_ref}\n {p} 列: {cols}\n"
|
||||
"请确保所有 CSV 的数值列名与顺序一致。"
|
||||
)
|
||||
dfs.append(df_num.reset_index(drop=True))
|
||||
big = pd.concat(dfs, axis=0, ignore_index=True)
|
||||
return big
|
||||
|
||||
|
||||
def build_multi_file_starts(lengths: List[int], seq_len: int, horizon: int, stride: int) -> np.ndarray:
|
||||
"""
|
||||
对多个段(各自长度 T_i)分别生成窗口起点,再加上段偏移并合并。
|
||||
确保窗口不跨段。
|
||||
"""
|
||||
starts_all = []
|
||||
offset = 0
|
||||
for T in lengths:
|
||||
local = build_windows_indices(T, seq_len, horizon, stride)
|
||||
if len(local) > 0:
|
||||
starts_all.append(local + offset)
|
||||
offset += T
|
||||
if not starts_all:
|
||||
return np.array([], dtype=np.int64)
|
||||
return np.concatenate(starts_all, axis=0)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
# === 关键默认:输入 240、预测 30 ===
|
||||
ap.add_argument("--csv_paths", type=str, nargs="+", help="多个 CSV 路径(空格分隔)")
|
||||
ap.add_argument("--csv_path", type=str, default="", help="兼容单文件旧参数;若提供也会被加入")
|
||||
ap.add_argument("--seq_len", type=int, default=180, help="输入窗口长度 L")
|
||||
ap.add_argument("--pred_horizon", type=int, default=45, help="预测步数 H(输出 30 帧)")
|
||||
ap.add_argument("--stride", type=int, default=1, help="窗口步长")
|
||||
ap.add_argument("--val_ratio", type=float, default=0.2, help="验证集比例(交错采样)")
|
||||
ap.add_argument("--val_gap", type=int, default=1, help="从训练中剔除与验证窗口相邻的窗口数量")
|
||||
ap.add_argument("--batch_size", type=int, default=512)
|
||||
ap.add_argument("--epochs", type=int, default=70)
|
||||
ap.add_argument("--lr", type=float, default=3e-4)
|
||||
ap.add_argument("--d_model", type=int, default=256)
|
||||
ap.add_argument("--nhead", type=int, default=8)
|
||||
ap.add_argument("--num_layers", type=int, default=4)
|
||||
ap.add_argument("--dim_ff", type=int, default=1024)
|
||||
ap.add_argument("--dropout", type=float, default=0.1)
|
||||
ap.add_argument("--grad_clip", type=float, default=1.0)
|
||||
ap.add_argument("--patience", type=int, default=20)
|
||||
ap.add_argument("--seed", type=int, default=42)
|
||||
ap.add_argument("--save_dir", type=str, default="./outputs/chaos")
|
||||
|
||||
# 加权 RMSE 配置
|
||||
ap.add_argument("--horizon_weight_mode", type=str, default="exp", choices=["exp", "linear"],
|
||||
help="预测步权重模式:exp=指数衰减;linear=线性衰减")
|
||||
ap.add_argument("--horizon_gamma", type=float, default=0.99,
|
||||
help="指数衰减系数 gamma (0<gamma<=1),越小靠后权重越低")
|
||||
ap.add_argument("--horizon_alpha", type=float, default=0.02,
|
||||
help="线性衰减步长 alpha(仅当 mode=linear 生效)")
|
||||
ap.add_argument("--horizon_min_w", type=float, default=0.05,
|
||||
help="线性衰减的最小权重下限(避免为 0)")
|
||||
|
||||
args = ap.parse_args()
|
||||
os.makedirs(args.save_dir, exist_ok=True)
|
||||
set_seed(args.seed)
|
||||
device = torch.device("cuda")
|
||||
print(f"Device: {device}")
|
||||
|
||||
# ====== 读取多个 CSV 并对齐列 ======
|
||||
paths: List[str] = []
|
||||
if args.csv_paths:
|
||||
paths.extend(args.csv_paths)
|
||||
if args.csv_path:
|
||||
paths.append(args.csv_path)
|
||||
if not paths:
|
||||
raise ValueError("请通过 --csv_paths 指定至少一个 CSV。")
|
||||
|
||||
big_df = read_and_align_csvs(paths)
|
||||
values = big_df.values.astype(np.float32) # [T_total, F]
|
||||
T_total, F = values.shape
|
||||
print(f"Loaded {len(paths)} files. Total T={T_total}, F={F}. Columns={list(big_df.columns)}")
|
||||
|
||||
# 段长度(用于保证窗口不跨文件)
|
||||
lengths = []
|
||||
for p in paths:
|
||||
df = pd.read_csv(p)
|
||||
if "frame_index" in df.columns:
|
||||
df = df.sort_values("frame_index").drop(columns=["frame_index"])
|
||||
lengths.append(len(df))
|
||||
|
||||
# ====== 生成全局窗口起点(不跨段) ======
|
||||
starts_all = build_multi_file_starts(lengths, args.seq_len, args.pred_horizon, args.stride)
|
||||
print(f"Windows total: {len(starts_all)}")
|
||||
if len(starts_all) == 0:
|
||||
raise ValueError("无可用窗口,请检查 seq_len/pred_horizon/stride 与数据长度。")
|
||||
|
||||
# ====== 均匀交错划分训练/验证 ======
|
||||
train_starts, val_starts = interleaved_split(starts_all, args.val_ratio, val_gap=args.val_gap)
|
||||
print(f"Split -> train: {len(train_starts)}, val: {len(val_starts)}")
|
||||
|
||||
# ====== 用训练窗口拟合标准化 ======
|
||||
mean, std = fit_scaler_from_windows(values, train_starts, args.seq_len)
|
||||
values_norm = (values - mean) / std
|
||||
|
||||
# ====== DataLoader ======
|
||||
train_ds = WindowedTSDataset(values_norm, train_starts, args.seq_len, args.pred_horizon)
|
||||
val_ds = WindowedTSDataset(values_norm, val_starts, args.seq_len, args.pred_horizon)
|
||||
train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True)
|
||||
val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False)
|
||||
|
||||
# ====== 模型/优化器/损失 ======
|
||||
model = TimeSeriesTransformer(
|
||||
in_features=F, d_model=args.d_model, nhead=args.nhead,
|
||||
num_layers=args.num_layers, dim_feedforward=args.dim_ff,
|
||||
dropout=args.dropout, horizon=args.pred_horizon
|
||||
).to(device)
|
||||
|
||||
weighted_rmse = WeightedRMSELoss(
|
||||
horizon=args.pred_horizon,
|
||||
mode=args.horizon_weight_mode,
|
||||
gamma=args.horizon_gamma,
|
||||
alpha=args.horizon_alpha,
|
||||
min_w=args.horizon_min_w
|
||||
).to(device)
|
||||
|
||||
optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=1e-4)
|
||||
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=3)
|
||||
|
||||
# ====== 训练(以加权 RMSE 为主损失,早停监控验证加权 RMSE) ======
|
||||
best_val = float("inf")
|
||||
best_path = os.path.join(args.save_dir, "best_model.pt")
|
||||
no_improve = 0
|
||||
|
||||
for epoch in range(1, args.epochs + 1):
|
||||
# 训练
|
||||
model.train()
|
||||
total_train_loss = 0.0
|
||||
n_train = 0
|
||||
for xb, yb in train_loader:
|
||||
xb = xb.to(device); yb = yb.to(device)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
pred = model(xb) # [B,H,F]
|
||||
loss = weighted_rmse(pred, yb) # 加权 RMSE
|
||||
loss.backward()
|
||||
if args.grad_clip is not None:
|
||||
nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)
|
||||
optimizer.step()
|
||||
total_train_loss += loss.item() * xb.size(0)
|
||||
n_train += xb.size(0)
|
||||
|
||||
# 验证
|
||||
val_wrmse, val_mse = evaluate_losses(model, val_loader, device, weighted_rmse)
|
||||
train_wrmse = total_train_loss / max(1, n_train)
|
||||
scheduler.step(val_wrmse)
|
||||
|
||||
print(f"Epoch {epoch:03d}/{args.epochs} | train wRMSE={train_wrmse:.6f} | "
|
||||
f"val wRMSE={val_wrmse:.6f} | val MSE={val_mse:.6f}")
|
||||
|
||||
if val_wrmse < best_val - 1e-8:
|
||||
best_val = val_wrmse
|
||||
no_improve = 0
|
||||
torch.save(
|
||||
{
|
||||
"model_state": model.state_dict(),
|
||||
"config": {
|
||||
"in_features": F, "d_model": args.d_model, "nhead": args.nhead,
|
||||
"num_layers": args.num_layers, "dim_ff": args.dim_ff,
|
||||
"dropout": args.dropout, "horizon": args.pred_horizon
|
||||
},
|
||||
"loss_cfg": {
|
||||
"mode": args.horizon_weight_mode,
|
||||
"gamma": args.horizon_gamma,
|
||||
"alpha": args.horizon_alpha,
|
||||
"min_w": args.horizon_min_w
|
||||
}
|
||||
},
|
||||
best_path
|
||||
)
|
||||
else:
|
||||
no_improve += 1
|
||||
if no_improve >= args.patience:
|
||||
print(f"Early stopping at epoch {epoch}. Best val wRMSE={best_val:.6f}")
|
||||
break
|
||||
|
||||
# 保存标准化
|
||||
np.savez(os.path.join(args.save_dir, "scaler.npz"),
|
||||
mean=mean, std=std, columns=np.array(big_df.columns))
|
||||
|
||||
# 评估(最佳模型)与原尺度 MAE
|
||||
ckpt = torch.load(best_path, map_location=device)
|
||||
model.load_state_dict(ckpt["model_state"])
|
||||
val_wrmse, val_mse = evaluate_losses(model, val_loader, device, weighted_rmse)
|
||||
val_mae_orig = mae_in_original_scale(model, val_loader, device, mean, std)
|
||||
print(f"[Best] Val wRMSE={val_wrmse:.6f} | Val MSE={val_mse:.6f} | Val MAE(original)={val_mae_orig:.6f}")
|
||||
|
||||
# 示例推理:取最后一个窗口打印 30 步预测(反标准化)
|
||||
with torch.no_grad():
|
||||
last_start = int(starts_all[-1])
|
||||
x_last = values_norm[last_start: last_start + args.seq_len]
|
||||
y_true = values[last_start + args.seq_len: last_start + args.seq_len + args.pred_horizon]
|
||||
pred = model(torch.from_numpy(x_last).unsqueeze(0).to(device)).cpu().numpy()[0]
|
||||
pred_denorm = pred * std + mean
|
||||
print("\n=== 示例(最后一个窗口) ===")
|
||||
for h in range(args.pred_horizon):
|
||||
line = f"+{h+1:02d}: pred={np.round(pred_denorm[h], 6)}"
|
||||
if h < y_true.shape[0]:
|
||||
line += f" | true={np.round(y_true[h], 6)}"
|
||||
print(line)
|
||||
|
||||
print(f"\n保存:\n- 模型: {best_path}\n- 标准化: {os.path.join(args.save_dir, 'scaler.npz')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,132 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
serve_api.py
|
||||
- FastAPI 推理服务:加载 best_model.pt 与 scaler.npz
|
||||
- /predict:接收最近 L=180 帧(按训练列顺序),返回未来 H 帧预测(原尺度)
|
||||
- /health:健康检查;/meta:返回模型与列信息
|
||||
|
||||
运行示例:
|
||||
uv run -m uvicorn chaos_pdl.serve_api:app --host 0.0.0.0 --port 8000 --workers 1
|
||||
|
||||
请求示例:
|
||||
POST /predict
|
||||
{
|
||||
"frames": [[...F个数值...], ..., 共180行]
|
||||
}
|
||||
|
||||
可选:若你以对象形式传参(带列名),也支持:
|
||||
{
|
||||
"frame_objects": [
|
||||
{"delta_time": 0.02, "green_angle_sin": 0.1, ...},
|
||||
... 共180个对象 ...
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
from typing import List, Optional, Dict, Any
|
||||
import os
|
||||
import numpy as np
|
||||
import torch
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
try:
|
||||
from chaos_pdl.pendulum_transformer import TimeSeriesTransformer
|
||||
except ModuleNotFoundError:
|
||||
from pendulum_transformer import TimeSeriesTransformer
|
||||
|
||||
|
||||
class PredictRequest(BaseModel):
|
||||
frames: Optional[List[List[float]]] = Field(
|
||||
default=None, description="最近L帧,形状 [L,F],按 scaler.npz 的列顺序提供"
|
||||
)
|
||||
frame_objects: Optional[List[Dict[str, float]]] = Field(
|
||||
default=None, description="最近L帧,形状 [L],每项是 {列名: 数值} 对象"
|
||||
)
|
||||
|
||||
|
||||
class PredictResponse(BaseModel):
|
||||
horizon: int
|
||||
columns: List[str]
|
||||
predictions: List[List[float]] # [H,F] 原尺度
|
||||
|
||||
|
||||
def load_artifacts(model_path: str, scaler_path: str, device: torch.device):
|
||||
sc = np.load(scaler_path, allow_pickle=True)
|
||||
mean = sc["mean"].astype(np.float32)
|
||||
std = sc["std"].astype(np.float32)
|
||||
columns = list(sc["columns"].tolist())
|
||||
|
||||
ckpt = torch.load(model_path, map_location=device)
|
||||
cfg = ckpt["config"]
|
||||
H = int(cfg["horizon"])
|
||||
F = len(columns)
|
||||
|
||||
model = TimeSeriesTransformer(
|
||||
in_features=F,
|
||||
d_model=cfg["d_model"],
|
||||
nhead=cfg["nhead"],
|
||||
num_layers=cfg["num_layers"],
|
||||
dim_feedforward=cfg["dim_ff"],
|
||||
dropout=cfg["dropout"],
|
||||
horizon=H,
|
||||
).to(device)
|
||||
model.load_state_dict(ckpt["model_state"])
|
||||
model.eval()
|
||||
return model, mean, std, columns, H
|
||||
|
||||
|
||||
MODEL_PATH = os.environ.get("MODEL_PATH", "./outputs/chaos/best_model.pt")
|
||||
SCALER_PATH = os.environ.get("SCALER_PATH", "./outputs/chaos/scaler.npz")
|
||||
SEQ_LEN = int(os.environ.get("SEQ_LEN", 180))
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
model, mean, std, columns, H = load_artifacts(MODEL_PATH, SCALER_PATH, device)
|
||||
|
||||
app = FastAPI(title="Pendulum Transformer Inference API", version="0.1.0")
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok", "device": str(device), "horizon": H, "seq_len": SEQ_LEN}
|
||||
|
||||
|
||||
@app.get("/meta")
|
||||
def meta():
|
||||
return {"columns": columns, "horizon": H, "seq_len": SEQ_LEN}
|
||||
|
||||
|
||||
def frames_from_objects(objs: List[Dict[str, Any]], cols: List[str]) -> np.ndarray:
|
||||
try:
|
||||
arr = np.array([[float(o[c]) for c in cols] for o in objs], dtype=np.float32)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"frame_objects 解析失败: {e}")
|
||||
return arr
|
||||
|
||||
|
||||
@app.post("/predict", response_model=PredictResponse)
|
||||
def predict(req: PredictRequest):
|
||||
# 取输入
|
||||
if (req.frames is None) == (req.frame_objects is None):
|
||||
raise HTTPException(status_code=400, detail="需要提供 frames 或 frame_objects 二选一")
|
||||
|
||||
if req.frames is not None:
|
||||
x = np.asarray(req.frames, dtype=np.float32)
|
||||
if x.ndim != 2:
|
||||
raise HTTPException(status_code=400, detail="frames 必须是二维数组 [L,F]")
|
||||
if x.shape[1] != len(columns):
|
||||
raise HTTPException(status_code=400, detail=f"列数不匹配,期望F={len(columns)}")
|
||||
else:
|
||||
x = frames_from_objects(req.frame_objects, columns)
|
||||
|
||||
if x.shape[0] != SEQ_LEN:
|
||||
raise HTTPException(status_code=400, detail=f"帧数不匹配,期望L={SEQ_LEN}")
|
||||
|
||||
# 标准化
|
||||
x_norm = (x - mean) / std
|
||||
xb = torch.from_numpy(x_norm).unsqueeze(0).to(device) # [1,L,F]
|
||||
|
||||
with torch.no_grad():
|
||||
pred = model(xb).cpu().numpy()[0] # [H,F] 标准化空间
|
||||
pred_denorm = pred * std + mean
|
||||
return PredictResponse(horizon=H, columns=columns, predictions=pred_denorm.tolist())
|
||||
Reference in New Issue
Block a user