refactor(log): simplify ClickHouse log deletion and add unit tests

Replace the per-batch ClickHouse mutation loop with a single
ALTER TABLE ... DELETE, since ClickHouse DELETE is a heavy mutation
that rewrites data parts and per-batch mutations are pathologically
slow. Add deterministic unit tests covering ClickHouse DSN handling,
main-database rejection, TTL DDL generation, log ordering, request_id
backfill, and display id assignment.
This commit is contained in:
CaIon
2026-06-22 19:32:32 +08:00
parent a162163b48
commit f84b7d591f
2 changed files with 138 additions and 21 deletions
+12 -21
View File
@@ -683,33 +683,24 @@ func DeleteOldLogBatch(ctx context.Context, targetTimestamp int64, limit int) (i
}
if common.UsingLogDatabase(common.DatabaseTypeClickHouse) {
var batchCount int64
if err := LOG_DB.WithContext(ctx).Raw(`
SELECT count() FROM (
SELECT created_at, request_id
FROM logs
WHERE created_at < ?
ORDER BY created_at ASC, request_id ASC
LIMIT ?
)`, targetTimestamp, limit).Scan(&batchCount).Error; err != nil {
// ClickHouse DELETE is a heavy mutation that rewrites data parts, so
// per-batch mutations would be pathologically slow. Remove all matching
// rows in a single synchronous mutation regardless of limit; the reported
// count lets the caller's progress loop complete in one pass.
total, err := CountOldLog(ctx, targetTimestamp)
if err != nil {
return 0, err
}
if batchCount == 0 {
if total == 0 {
return 0, nil
}
if err := LOG_DB.WithContext(ctx).Exec(`
ALTER TABLE logs DELETE WHERE (created_at, request_id) IN (
SELECT created_at, request_id
FROM logs
WHERE created_at < ?
ORDER BY created_at ASC, request_id ASC
LIMIT ?
) SETTINGS mutations_sync = 1`, targetTimestamp, limit).Error; err != nil {
if err := LOG_DB.WithContext(ctx).Exec(
"ALTER TABLE logs DELETE WHERE created_at < ? SETTINGS mutations_sync = 1",
targetTimestamp,
).Error; err != nil {
return 0, err
}
return batchCount, nil
return total, nil
}
result := LOG_DB.WithContext(ctx).Where("created_at < ?", targetTimestamp).Limit(limit).Delete(&Log{})