add: log management module (compressor, manager, retention)

internal/log/compressor.go

@@ -0,0 +1,225 @@
+package log
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+var rawLogRe = regexp.MustCompile(`^homed_(\d{4}-\d{2}-\d{2})_\d{2}-\d{2}-\d{2}\.log$`)
+var weekRe = regexp.MustCompile(`^week_(\d{4})-W(\d{2})\.tar\.gz$`)
+var monthRe = regexp.MustCompile(`^month_(\d{4}-\d{2})\.tar\.gz$`)
+
+type fileGroup struct {
+	name string
+	files []string
+}
+
+func compressWeekly(logDir string) error {
+	entries, err := os.ReadDir(logDir)
+	if err != nil {
+		return err
+	}
+
+	groups := make(map[string][]string)
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		m := rawLogRe.FindStringSubmatch(e.Name())
+		if m == nil {
+			continue
+		}
+		t, err := time.Parse("2006-01-02", m[1])
+		if err != nil {
+			continue
+		}
+		y, w := t.ISOWeek()
+		key := fmt.Sprintf("%04d-W%02d", y, w)
+		groups[key] = append(groups[key], e.Name())
+	}
+
+	for key, files := range groups {
+		dst := filepath.Join(logDir, fmt.Sprintf("week_%s.tar.gz", key))
+		if fileExists(dst) {
+			continue
+		}
+		sort.Strings(files)
+		if err := tarGzFiles(logDir, dst, files); err != nil {
+			return fmt.Errorf("compress week %s: %w", key, err)
+		}
+		for _, f := range files {
+			os.Remove(filepath.Join(logDir, f))
+		}
+	}
+	return nil
+}
+
+func compressMonthly(logDir string) error {
+	entries, err := os.ReadDir(logDir)
+	if err != nil {
+		return err
+	}
+
+	groups := make(map[string][]string)
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		m := weekRe.FindStringSubmatch(e.Name())
+		if m == nil {
+			continue
+		}
+		key := fmt.Sprintf("%s-%s", m[1], m[2][:2])
+		monthKey := key[:7]
+		groups[monthKey] = append(groups[monthKey], e.Name())
+	}
+
+	for monthKey, files := range groups {
+		dst := filepath.Join(logDir, fmt.Sprintf("month_%s.tar.gz", monthKey))
+		if fileExists(dst) {
+			continue
+		}
+		sort.Strings(files)
+		if err := tarGzFiles(logDir, dst, files); err != nil {
+			return fmt.Errorf("compress month %s: %w", monthKey, err)
+		}
+		for _, f := range files {
+			os.Remove(filepath.Join(logDir, f))
+		}
+	}
+	return nil
+}
+
+func compressYearly(logDir string) error {
+	entries, err := os.ReadDir(logDir)
+	if err != nil {
+		return err
+	}
+
+	groups := make(map[string][]string)
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		m := monthRe.FindStringSubmatch(e.Name())
+		if m == nil {
+			continue
+		}
+		yearKey := m[1][:4]
+		groups[yearKey] = append(groups[yearKey], e.Name())
+	}
+
+	for yearKey, files := range groups {
+		dst := filepath.Join(logDir, fmt.Sprintf("year_%s.tar.gz", yearKey))
+		if fileExists(dst) {
+			continue
+		}
+		sort.Strings(files)
+		if err := tarGzFiles(logDir, dst, files); err != nil {
+			return fmt.Errorf("compress year %s: %w", yearKey, err)
+		}
+		for _, f := range files {
+			os.Remove(filepath.Join(logDir, f))
+		}
+	}
+	return nil
+}
+
+func tarGzFiles(baseDir, dst string, filenames []string) error {
+	f, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	gw := gzip.NewWriter(f)
+	defer gw.Close()
+
+	tw := tar.NewWriter(gw)
+	defer tw.Close()
+
+	for _, fn := range filenames {
+		path := filepath.Join(baseDir, fn)
+		info, err := os.Stat(path)
+		if err != nil {
+			continue
+		}
+		header, err := tar.FileInfoHeader(info, "")
+		if err != nil {
+			continue
+		}
+		header.Name = fn
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+		r, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(tw, r); err != nil {
+			r.Close()
+			return err
+		}
+		r.Close()
+	}
+	return nil
+}
+
+func fileExists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+func lastWeekKey(t time.Time) string {
+	y, w := t.ISOWeek()
+	return fmt.Sprintf("%04d-W%02d", y, w)
+}
+
+func lastMonthKey(t time.Time) string {
+	return t.Format("2006-01")
+}
+
+func lastYearKey(t time.Time) string {
+	return t.Format("2006")
+}
+
+func parseWeekKey(key string) (time.Time, error) {
+	var y, w int
+	if _, err := fmt.Sscanf(key, "%04d-W%02d", &y, &w); err != nil {
+		return time.Time{}, err
+	}
+	return firstDayOfISOWeek(y, w), nil
+}
+
+func firstDayOfISOWeek(year, week int) time.Time {
+	jan1 := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC)
+	_, jan1Week := jan1.ISOWeek()
+	daysOffset := (week - jan1Week) * 7
+	t := jan1.AddDate(0, 0, daysOffset)
+	for t.Weekday() != time.Monday {
+		t = t.AddDate(0, 0, -1)
+	}
+	return t
+}
+
+func listArchives(logDir, prefix string) []string {
+	entries, err := os.ReadDir(logDir)
+	if err != nil {
+		return nil
+	}
+	var result []string
+	for _, e := range entries {
+		if !e.IsDir() && strings.HasPrefix(e.Name(), prefix) {
+			result = append(result, e.Name())
+		}
+	}
+	return result
+}

internal/log/manager.go

@@ -0,0 +1,114 @@
+package log
+
+import (
+	"context"
+	"log"
+	"time"
+)
+
+// ConfigProvider 是 log 包读取配置的最小接口，由 internalConfig.ConfigRegistry 实现。
+type ConfigProvider interface {
+	GetString(key, defaultVal string) string
+	GetInt(key string, defaultVal int) int
+}
+
+type Manager struct {
+	logDir string
+	cfg    ConfigProvider
+	done   chan struct{}
+}
+
+func NewManager(logDir string, cfg ConfigProvider) *Manager {
+	return &Manager{logDir: logDir, cfg: cfg, done: make(chan struct{})}
+}
+
+func (m *Manager) Start(ctx context.Context) {
+	go m.runLoop(ctx, "weekly", 24*time.Hour, m.weeklyTick, m.compressWeekly)
+	go m.runLoop(ctx, "monthly", 24*time.Hour, m.monthlyTick, m.compressMonthly)
+	go m.runLoop(ctx, "yearly", 24*time.Hour, m.yearlyTick, m.compressYearly)
+}
+
+func (m *Manager) Stop() {
+	close(m.done)
+}
+
+func (m *Manager) runLoop(ctx context.Context, name string, tickInterval time.Duration, nextFn func() time.Duration, work func(string) error) {
+	first := nextFn()
+	timer := time.NewTimer(first)
+	defer timer.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			log.Printf("[log manager] running %s compression", name)
+			if err := work(m.logDir); err != nil {
+				log.Printf("[log manager] %s compression error: %v", name, err)
+			}
+			m.applyRetention()
+			timer.Reset(nextFn())
+		}
+	}
+}
+
+func (m *Manager) applyRetention() {
+	policy := RetentionPolicy(m.cfg.GetString("core.log.retention", "forever"))
+	months := m.cfg.GetInt("core.log.retention_months", 3)
+	applyRetention(m.logDir, policy, months)
+}
+
+func (m *Manager) weeklyTick() time.Duration {
+	return nextWeekdayTime(time.Monday, 1, 0)
+}
+
+func (m *Manager) monthlyTick() time.Duration {
+	return nextMonthDayTime(1, 2, 0)
+}
+
+func (m *Manager) yearlyTick() time.Duration {
+	return nextYearDayTime(1, 3, 0)
+}
+
+func (m *Manager) compressWeekly(logDir string) error {
+	return compressWeekly(logDir)
+}
+
+func (m *Manager) compressMonthly(logDir string) error {
+	return compressMonthly(logDir)
+}
+
+func (m *Manager) compressYearly(logDir string) error {
+	return compressYearly(logDir)
+}
+
+func nextWeekdayTime(wd time.Weekday, hour, min int) time.Duration {
+	now := time.Now()
+	next := time.Date(now.Year(), now.Month(), now.Day(), hour, min, 0, 0, now.Location())
+	for next.Weekday() != wd || !next.After(now) {
+		next = next.AddDate(0, 0, 1)
+	}
+	return next.Sub(now)
+}
+
+func nextMonthDayTime(day, hour, min int) time.Duration {
+	now := time.Now()
+	next := time.Date(now.Year(), now.Month(), day, hour, min, 0, 0, now.Location())
+	if !next.After(now) {
+		if now.Month() == 12 {
+			next = time.Date(now.Year()+1, 1, day, hour, min, 0, 0, now.Location())
+		} else {
+			next = time.Date(now.Year(), now.Month()+1, day, hour, min, 0, 0, now.Location())
+		}
+	}
+	return next.Sub(now)
+}
+
+func nextYearDayTime(day, hour, min int) time.Duration {
+	now := time.Now()
+	next := time.Date(now.Year(), 1, day, hour, min, 0, 0, now.Location())
+	if !next.After(now) {
+		next = time.Date(now.Year()+1, 1, day, hour, min, 0, 0, now.Location())
+	}
+	return next.Sub(now)
+}

internal/log/retention.go

@@ -0,0 +1,95 @@
+package log
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type RetentionPolicy string
+
+const (
+	RetentionThisWeek  RetentionPolicy = "this_week"
+	RetentionThisMonth RetentionPolicy = "this_month"
+	RetentionNMonths   RetentionPolicy = "n_months"
+	RetentionForever   RetentionPolicy = "forever"
+)
+
+func applyRetention(logDir string, policy RetentionPolicy, months int) {
+	switch policy {
+	case RetentionForever:
+		return
+	case RetentionThisWeek:
+		cleanupExceptCurrentWeek(logDir)
+	case RetentionThisMonth:
+		cleanupExceptCurrentMonth(logDir)
+	case RetentionNMonths:
+		cleanupBeforeMonths(logDir, months)
+	}
+}
+
+func cleanupExceptCurrentWeek(logDir string) {
+	now := time.Now()
+	keep := lastWeekKey(now)
+	for _, f := range listArchives(logDir, "week_") {
+		if !strings.HasPrefix(f, "week_"+keep) {
+			os.Remove(filepath.Join(logDir, f))
+		}
+	}
+	cleanupAll(logDir, "month_")
+	cleanupAll(logDir, "year_")
+}
+
+func cleanupExceptCurrentMonth(logDir string) {
+	now := time.Now()
+	keep := lastMonthKey(now)
+	for _, f := range listArchives(logDir, "month_") {
+		if !strings.HasPrefix(f, "month_"+keep) {
+			os.Remove(filepath.Join(logDir, f))
+		}
+	}
+	cleanupAll(logDir, "year_")
+}
+
+func cleanupBeforeMonths(logDir string, months int) {
+	cutoff := time.Now().AddDate(0, -months, 0)
+	for _, f := range listArchives(logDir, "month_") {
+		// month_2006-01.tar.gz
+		if len(f) < 15 {
+			continue
+		}
+		dateStr := f[6:13]
+		t, err := time.Parse("2006-01", dateStr)
+		if err != nil {
+			continue
+		}
+		if t.Before(cutoff) {
+			os.Remove(filepath.Join(logDir, f))
+		}
+	}
+	for _, f := range listArchives(logDir, "year_") {
+		if len(f) < 12 {
+			continue
+		}
+		yearStr := f[5:9]
+		y, err := time.Parse("2006", yearStr)
+		if err != nil {
+			continue
+		}
+		if y.Before(cutoff) {
+			os.Remove(filepath.Join(logDir, f))
+		}
+	}
+}
+
+func cleanupAll(logDir, prefix string) {
+	for _, f := range listArchives(logDir, prefix) {
+		if err := os.Remove(filepath.Join(logDir, f)); err != nil {
+			log.Printf("[log retention] remove %s: %v", f, err)
+		}
+	}
+}
+
+