mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-20 17:08:09 +00:00
add: log management module (compressor, manager, retention)
This commit is contained in:
root
225
internal/log/compressor.go
Normal file
225
internal/log/compressor.go
Normal file
@ -0,0 +1,225 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var rawLogRe = regexp.MustCompile(`^homed_(\d{4}-\d{2}-\d{2})_\d{2}-\d{2}-\d{2}\.log$`)
|
||||
var weekRe = regexp.MustCompile(`^week_(\d{4})-W(\d{2})\.tar\.gz$`)
|
||||
var monthRe = regexp.MustCompile(`^month_(\d{4}-\d{2})\.tar\.gz$`)
|
||||
|
||||
type fileGroup struct {
|
||||
name string
|
||||
files []string
|
||||
}
|
||||
|
||||
func compressWeekly(logDir string) error {
|
||||
entries, err := os.ReadDir(logDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
groups := make(map[string][]string)
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
m := rawLogRe.FindStringSubmatch(e.Name())
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
t, err := time.Parse("2006-01-02", m[1])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
y, w := t.ISOWeek()
|
||||
key := fmt.Sprintf("%04d-W%02d", y, w)
|
||||
groups[key] = append(groups[key], e.Name())
|
||||
}
|
||||
|
||||
for key, files := range groups {
|
||||
dst := filepath.Join(logDir, fmt.Sprintf("week_%s.tar.gz", key))
|
||||
if fileExists(dst) {
|
||||
continue
|
||||
}
|
||||
sort.Strings(files)
|
||||
if err := tarGzFiles(logDir, dst, files); err != nil {
|
||||
return fmt.Errorf("compress week %s: %w", key, err)
|
||||
}
|
||||
for _, f := range files {
|
||||
os.Remove(filepath.Join(logDir, f))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func compressMonthly(logDir string) error {
|
||||
entries, err := os.ReadDir(logDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
groups := make(map[string][]string)
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
m := weekRe.FindStringSubmatch(e.Name())
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
key := fmt.Sprintf("%s-%s", m[1], m[2][:2])
|
||||
monthKey := key[:7]
|
||||
groups[monthKey] = append(groups[monthKey], e.Name())
|
||||
}
|
||||
|
||||
for monthKey, files := range groups {
|
||||
dst := filepath.Join(logDir, fmt.Sprintf("month_%s.tar.gz", monthKey))
|
||||
if fileExists(dst) {
|
||||
continue
|
||||
}
|
||||
sort.Strings(files)
|
||||
if err := tarGzFiles(logDir, dst, files); err != nil {
|
||||
return fmt.Errorf("compress month %s: %w", monthKey, err)
|
||||
}
|
||||
for _, f := range files {
|
||||
os.Remove(filepath.Join(logDir, f))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func compressYearly(logDir string) error {
|
||||
entries, err := os.ReadDir(logDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
groups := make(map[string][]string)
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
m := monthRe.FindStringSubmatch(e.Name())
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
yearKey := m[1][:4]
|
||||
groups[yearKey] = append(groups[yearKey], e.Name())
|
||||
}
|
||||
|
||||
for yearKey, files := range groups {
|
||||
dst := filepath.Join(logDir, fmt.Sprintf("year_%s.tar.gz", yearKey))
|
||||
if fileExists(dst) {
|
||||
continue
|
||||
}
|
||||
sort.Strings(files)
|
||||
if err := tarGzFiles(logDir, dst, files); err != nil {
|
||||
return fmt.Errorf("compress year %s: %w", yearKey, err)
|
||||
}
|
||||
for _, f := range files {
|
||||
os.Remove(filepath.Join(logDir, f))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func tarGzFiles(baseDir, dst string, filenames []string) error {
|
||||
f, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gw := gzip.NewWriter(f)
|
||||
defer gw.Close()
|
||||
|
||||
tw := tar.NewWriter(gw)
|
||||
defer tw.Close()
|
||||
|
||||
for _, fn := range filenames {
|
||||
path := filepath.Join(baseDir, fn)
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
header, err := tar.FileInfoHeader(info, "")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
header.Name = fn
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
r, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(tw, r); err != nil {
|
||||
r.Close()
|
||||
return err
|
||||
}
|
||||
r.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func lastWeekKey(t time.Time) string {
|
||||
y, w := t.ISOWeek()
|
||||
return fmt.Sprintf("%04d-W%02d", y, w)
|
||||
}
|
||||
|
||||
func lastMonthKey(t time.Time) string {
|
||||
return t.Format("2006-01")
|
||||
}
|
||||
|
||||
func lastYearKey(t time.Time) string {
|
||||
return t.Format("2006")
|
||||
}
|
||||
|
||||
func parseWeekKey(key string) (time.Time, error) {
|
||||
var y, w int
|
||||
if _, err := fmt.Sscanf(key, "%04d-W%02d", &y, &w); err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return firstDayOfISOWeek(y, w), nil
|
||||
}
|
||||
|
||||
func firstDayOfISOWeek(year, week int) time.Time {
|
||||
jan1 := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
_, jan1Week := jan1.ISOWeek()
|
||||
daysOffset := (week - jan1Week) * 7
|
||||
t := jan1.AddDate(0, 0, daysOffset)
|
||||
for t.Weekday() != time.Monday {
|
||||
t = t.AddDate(0, 0, -1)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func listArchives(logDir, prefix string) []string {
|
||||
entries, err := os.ReadDir(logDir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var result []string
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.HasPrefix(e.Name(), prefix) {
|
||||
result = append(result, e.Name())
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
114
internal/log/manager.go
Normal file
114
internal/log/manager.go
Normal file
@ -0,0 +1,114 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ConfigProvider 是 log 包读取配置的最小接口，由 internalConfig.ConfigRegistry 实现。
|
||||
type ConfigProvider interface {
|
||||
GetString(key, defaultVal string) string
|
||||
GetInt(key string, defaultVal int) int
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
logDir string
|
||||
cfg ConfigProvider
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
func NewManager(logDir string, cfg ConfigProvider) *Manager {
|
||||
return &Manager{logDir: logDir, cfg: cfg, done: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (m *Manager) Start(ctx context.Context) {
|
||||
go m.runLoop(ctx, "weekly", 24*time.Hour, m.weeklyTick, m.compressWeekly)
|
||||
go m.runLoop(ctx, "monthly", 24*time.Hour, m.monthlyTick, m.compressMonthly)
|
||||
go m.runLoop(ctx, "yearly", 24*time.Hour, m.yearlyTick, m.compressYearly)
|
||||
}
|
||||
|
||||
func (m *Manager) Stop() {
|
||||
close(m.done)
|
||||
}
|
||||
|
||||
func (m *Manager) runLoop(ctx context.Context, name string, tickInterval time.Duration, nextFn func() time.Duration, work func(string) error) {
|
||||
first := nextFn()
|
||||
timer := time.NewTimer(first)
|
||||
defer timer.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-timer.C:
|
||||
log.Printf("[log manager] running %s compression", name)
|
||||
if err := work(m.logDir); err != nil {
|
||||
log.Printf("[log manager] %s compression error: %v", name, err)
|
||||
}
|
||||
m.applyRetention()
|
||||
timer.Reset(nextFn())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) applyRetention() {
|
||||
policy := RetentionPolicy(m.cfg.GetString("core.log.retention", "forever"))
|
||||
months := m.cfg.GetInt("core.log.retention_months", 3)
|
||||
applyRetention(m.logDir, policy, months)
|
||||
}
|
||||
|
||||
func (m *Manager) weeklyTick() time.Duration {
|
||||
return nextWeekdayTime(time.Monday, 1, 0)
|
||||
}
|
||||
|
||||
func (m *Manager) monthlyTick() time.Duration {
|
||||
return nextMonthDayTime(1, 2, 0)
|
||||
}
|
||||
|
||||
func (m *Manager) yearlyTick() time.Duration {
|
||||
return nextYearDayTime(1, 3, 0)
|
||||
}
|
||||
|
||||
func (m *Manager) compressWeekly(logDir string) error {
|
||||
return compressWeekly(logDir)
|
||||
}
|
||||
|
||||
func (m *Manager) compressMonthly(logDir string) error {
|
||||
return compressMonthly(logDir)
|
||||
}
|
||||
|
||||
func (m *Manager) compressYearly(logDir string) error {
|
||||
return compressYearly(logDir)
|
||||
}
|
||||
|
||||
func nextWeekdayTime(wd time.Weekday, hour, min int) time.Duration {
|
||||
now := time.Now()
|
||||
next := time.Date(now.Year(), now.Month(), now.Day(), hour, min, 0, 0, now.Location())
|
||||
for next.Weekday() != wd || !next.After(now) {
|
||||
next = next.AddDate(0, 0, 1)
|
||||
}
|
||||
return next.Sub(now)
|
||||
}
|
||||
|
||||
func nextMonthDayTime(day, hour, min int) time.Duration {
|
||||
now := time.Now()
|
||||
next := time.Date(now.Year(), now.Month(), day, hour, min, 0, 0, now.Location())
|
||||
if !next.After(now) {
|
||||
if now.Month() == 12 {
|
||||
next = time.Date(now.Year()+1, 1, day, hour, min, 0, 0, now.Location())
|
||||
} else {
|
||||
next = time.Date(now.Year(), now.Month()+1, day, hour, min, 0, 0, now.Location())
|
||||
}
|
||||
}
|
||||
return next.Sub(now)
|
||||
}
|
||||
|
||||
func nextYearDayTime(day, hour, min int) time.Duration {
|
||||
now := time.Now()
|
||||
next := time.Date(now.Year(), 1, day, hour, min, 0, 0, now.Location())
|
||||
if !next.After(now) {
|
||||
next = time.Date(now.Year()+1, 1, day, hour, min, 0, 0, now.Location())
|
||||
}
|
||||
return next.Sub(now)
|
||||
}
|
||||
95
internal/log/retention.go
Normal file
95
internal/log/retention.go
Normal file
@ -0,0 +1,95 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RetentionPolicy string
|
||||
|
||||
const (
|
||||
RetentionThisWeek RetentionPolicy = "this_week"
|
||||
RetentionThisMonth RetentionPolicy = "this_month"
|
||||
RetentionNMonths RetentionPolicy = "n_months"
|
||||
RetentionForever RetentionPolicy = "forever"
|
||||
)
|
||||
|
||||
func applyRetention(logDir string, policy RetentionPolicy, months int) {
|
||||
switch policy {
|
||||
case RetentionForever:
|
||||
return
|
||||
case RetentionThisWeek:
|
||||
cleanupExceptCurrentWeek(logDir)
|
||||
case RetentionThisMonth:
|
||||
cleanupExceptCurrentMonth(logDir)
|
||||
case RetentionNMonths:
|
||||
cleanupBeforeMonths(logDir, months)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupExceptCurrentWeek(logDir string) {
|
||||
now := time.Now()
|
||||
keep := lastWeekKey(now)
|
||||
for _, f := range listArchives(logDir, "week_") {
|
||||
if !strings.HasPrefix(f, "week_"+keep) {
|
||||
os.Remove(filepath.Join(logDir, f))
|
||||
}
|
||||
}
|
||||
cleanupAll(logDir, "month_")
|
||||
cleanupAll(logDir, "year_")
|
||||
}
|
||||
|
||||
func cleanupExceptCurrentMonth(logDir string) {
|
||||
now := time.Now()
|
||||
keep := lastMonthKey(now)
|
||||
for _, f := range listArchives(logDir, "month_") {
|
||||
if !strings.HasPrefix(f, "month_"+keep) {
|
||||
os.Remove(filepath.Join(logDir, f))
|
||||
}
|
||||
}
|
||||
cleanupAll(logDir, "year_")
|
||||
}
|
||||
|
||||
func cleanupBeforeMonths(logDir string, months int) {
|
||||
cutoff := time.Now().AddDate(0, -months, 0)
|
||||
for _, f := range listArchives(logDir, "month_") {
|
||||
// month_2006-01.tar.gz
|
||||
if len(f) < 15 {
|
||||
continue
|
||||
}
|
||||
dateStr := f[6:13]
|
||||
t, err := time.Parse("2006-01", dateStr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if t.Before(cutoff) {
|
||||
os.Remove(filepath.Join(logDir, f))
|
||||
}
|
||||
}
|
||||
for _, f := range listArchives(logDir, "year_") {
|
||||
if len(f) < 12 {
|
||||
continue
|
||||
}
|
||||
yearStr := f[5:9]
|
||||
y, err := time.Parse("2006", yearStr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if y.Before(cutoff) {
|
||||
os.Remove(filepath.Join(logDir, f))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupAll(logDir, prefix string) {
|
||||
for _, f := range listArchives(logDir, prefix) {
|
||||
if err := os.Remove(filepath.Join(logDir, f)); err != nil {
|
||||
log.Printf("[log retention] remove %s: %v", f, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user