Added log rotation

README.md

@@ -1,6 +1,7 @@
 ## Process Compose
 
-[![made-with-Go](https://img.shields.io/badge/Made%20with-Go-1f425f.svg)](https://go.dev/) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/F1bonacc1/process-compose/graphs/commit-activity) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) ![Go Report](https://goreportcard.com/badge/github.com/F1bonacc1/process-compose) [![Releases](https://img.shields.io/github/downloads/F1bonacc1/process-compose/total.svg)]()
+[![made-with-Go](https://img.shields.io/badge/Made%20with-Go-1f425f.svg)](https://go.dev/) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/F1bonacc1/process-compose/graphs/commit-activity) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) ![Go Report](https://goreportcard.com/badge/github.com/F1bonacc1/process-compose) [![Releases](https://img.shields.io/github/downloads/F1bonacc1/process-compose/total.svg)]()![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/ProcessCompose?style=flat-square&logo=twitter&logoColor=white)
+
 
 Process Compose is a simple and flexible scheduler and orchestrator to manage non-containerized applications.
 
@@ -376,7 +377,7 @@ process2:
 
 ##### Capture StdErr output
 
-##### Merge into a single file
+##### Merge into a single file (Unified Logging)
 
 ```yaml
 processes:
@@ -398,6 +399,32 @@ processes:
 
 This setting controls the `process-compose` log level. The processes log level should be defined inside the process. It is recommended to support this definition with an environment variable in `process-compose.yaml`
 
+##### Log Rotation
+
+```yaml
+# unified log
+version: "0.5"
+log_level: info 
+log_location: /tmp/pc.log
+log_rotation:
+  max_size_mb: 1  # the max size in MB of the logfile before it's rolled
+  max_age_days: 3 # the max age in days to keep a logfile
+  max_backups: 3  # the max number of rolled files to keep
+  compress: true  # determines if the rotated log files should be compressed using gzip. The default is false
+
+#process level logging (same syntax)
+processes:
+  someProc:
+    command: "some command"
+    log_rotation:
+      max_size_mb: 1  # the max size in MB of the logfile before it's rolled
+      max_age_days: 3 # the max age in days to keep a logfile
+      max_backups: 3  # the max number of rolled files to keep
+      compress: true  # determines if the rotated log files should be compressed using gzip. The default is false
+```
+
+##### Process Compose Internal Log
+
 Default log location: `/tmp/process-compose-$USER.log`
 
 **Tip:** It is recommended to add the following process configuration to your `process-compose.yaml`:

default.nix

@@ -18,7 +18,7 @@ buildGoModule rec {
 
   nativeBuildInputs = [ installShellFiles ];
 
-  vendorSha256 = "Z5vCxzdpd2OmlZ/woHhlLN2QMgqa9mm873QGuqDToiM=";
+  vendorSha256 = "lU21nRfIi4/eobnHhX/fCWnWtoiQBiWvTUOjBL0I4X4=";
   #vendorSha256 = lib.fakeSha256;
 
   postInstall = ''

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/rivo/tview v0.0.0-20230530133550-8bd761dda819
 	github.com/spf13/cobra v1.7.0
 	github.com/swaggo/swag v1.16.1
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v2 v2.4.0
 )
 

go.sum

@@ -4,6 +4,8 @@ github.com/InVisionApp/go-logger v1.0.1 h1:WFL19PViM1mHUmUWfsv5zMo379KSWj2MRmBlz
 github.com/InVisionApp/go-logger v1.0.1/go.mod h1:+cGTDSn+P8105aZkeOfIhdd7vFO5X1afUHcjvanY0L8=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
 github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
@@ -249,6 +251,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

process-compose.override.yaml

@@ -36,6 +36,11 @@ processes:
   _process2:
     command: "./test_loop.bash process2"
     log_location: ./pc.proc2.{PC_REPLICA_NUM}.log
+    log_rotation:
+      max_size_mb: 1
+      max_age_days: 3
+      max_backups: 3
+      compress: true
     availability:
       restart: "on_failure"
     # depends_on:

process-compose.yaml

@@ -1,6 +1,11 @@
 version: "0.5"
 log_level: info
 log_length: 3000
+log_rotation:
+  max_size_mb: 1
+  max_age_days: 3
+  max_backups: 3
+  compress: true
 environment:
   - 'ABC=222'
 log_location: ./pc.log

src/app/process.go

@@ -294,7 +294,7 @@ func (p *Process) prepareForShutDown() {
 
 func (p *Process) onProcessStart() {
 	if isStringDefined(p.procConf.LogLocation) {
-		p.logger.Open(p.getLogPath())
+		p.logger.Open(p.getLogPath(), p.procConf.LogRotation)
 	}
 }
 

src/app/project_runner.go

@@ -57,7 +57,7 @@ func (p *ProjectRunner) Run() int {
 	p.logger = pclog.NewNilLogger()
 	if isStringDefined(p.project.LogLocation) {
 		p.logger = pclog.NewLogger()
-		p.logger.Open(p.project.LogLocation)
+		p.logger.Open(p.project.LogLocation, p.project.LogRotation)
 		defer p.logger.Close()
 	}
 	//zerolog.SetGlobalLevel(zerolog.PanicLevel)

src/pclog/logger_facade.go

@@ -2,7 +2,10 @@ package pclog
 
 import (
 	"bufio"
+	"github.com/f1bonacc1/process-compose/src/types"
 	"github.com/rs/zerolog/log"
+	"gopkg.in/natefinch/lumberjack.v2"
+	"io"
 	"os"
 	"path"
 	"sync"
@@ -14,7 +17,7 @@ import (
 type PCLog struct {
 	logger       zerolog.Logger
 	writer       *bufio.Writer
-	file         *os.File
+	file         io.WriteCloser
 	logEventChan chan logEvent
 	wg           sync.WaitGroup
 	closer       sync.Once
@@ -36,7 +39,7 @@ func NewLogger() *PCLog {
 	return l
 }
 
-func (l *PCLog) Open(filePath string) {
+func (l *PCLog) Open(filePath string, rotation *types.LogRotationConfig) {
 	if l.file != nil {
 		log.Error().Msgf("log file for %s is already open", filePath)
 		return
@@ -45,22 +48,54 @@ func (l *PCLog) Open(filePath string) {
 		log.Error().Msg("empty file path")
 		return
 	}
+
+	f, err := l.getWriter(filePath, rotation)
+	if err != nil {
+		l.isClosed.Store(true)
+		log.Err(err).Msgf("failed to create file %s", filePath)
+	}
+	l.writer = bufio.NewWriter(f)
+	l.file = f
+	l.logger = zerolog.New(l.writer)
+	l.wg.Add(1)
+	go l.runCollector()
+}
+
+func (l *PCLog) getWriter(filePath string, rotation *types.LogRotationConfig) (io.WriteCloser, error) {
 	dirName := path.Dir(filePath)
 	if err := os.MkdirAll(dirName, 0700); err != nil && !os.IsExist(err) {
 		l.isClosed.Store(true)
 		log.Err(err).Msgf("failed to create log file directory %s", dirName)
+		return nil, err
 	}
+	if rotation == nil {
+		log.Debug().Str("filePath", filePath).Msg("no rotation config")
+		return l.getFileWriter(filePath)
+	} else {
+		log.Debug().Str("filePath", filePath).Msg("rotation config")
+		return l.getRollingWriter(filePath, rotation)
+	}
+}
+
+func (l *PCLog) getFileWriter(filePath string) (io.WriteCloser, error) {
 	f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0600)
 	if err != nil {
 		l.isClosed.Store(true)
 		log.Err(err).Msgf("failed to open log file %s", filePath)
+		return nil, err
 	}
-	writer := bufio.NewWriter(f)
-	l.writer = writer
-	l.file = f
-	l.logger = zerolog.New(writer)
-	l.wg.Add(1)
-	go l.runCollector()
+	return f, nil
+}
+
+func (l *PCLog) getRollingWriter(filePath string, rotation *types.LogRotationConfig) (io.WriteCloser, error) {
+	return &lumberjack.Logger{
+		Filename:   filePath,
+		MaxBackups: rotation.MaxBackups, // files
+		MaxSize:    rotation.MaxSize,    // megabytes
+		MaxAge:     rotation.MaxAge,     // days
+		LocalTime:  true,
+		Compress:   rotation.Compress,
+	}, nil
 }
 
 func (l *PCLog) Info(message string, process string, replica int) {

src/pclog/logger_interface.go

@@ -1,7 +1,9 @@
 package pclog
 
+import "github.com/f1bonacc1/process-compose/src/types"
+
 type PcLogger interface {
-	Open(filePath string)
+	Open(filePath string, rotation *types.LogRotationConfig)
 	Info(message string, process string, replica int)
 	Error(message string, process string, replica int)
 	Close()

src/pclog/nil_logger.go

@@ -1,5 +1,7 @@
 package pclog
 
+import "github.com/f1bonacc1/process-compose/src/types"
+
 type PcNilLog struct {
 }
 
@@ -8,7 +10,7 @@ func NewNilLogger() *PcNilLog {
 	return &PcNilLog{}
 }
 
-func (l *PcNilLog) Open(_ string) {
+func (l *PcNilLog) Open(filePath string, rotation *types.LogRotationConfig) {
 }
 
 func (l *PcNilLog) Sync() {

src/types/logger.go

@@ -0,0 +1,18 @@
+package types
+
+// LogRotationConfig is the configuration for logging
+type LogRotationConfig struct {
+	// Directory to log to when filelogging is enabled
+	Directory string `yaml:"directory"`
+	// Filename is the name of the logfile which will be placed inside the directory
+	Filename string `yaml:"filename"`
+	// MaxSize the max size in MB of the logfile before it's rolled
+	MaxSize int `yaml:"max_size_mb"`
+	// MaxBackups the max number of rolled files to keep
+	MaxBackups int `yaml:"max_backups"`
+	// MaxAge the max age in days to keep a logfile
+	MaxAge int `yaml:"max_age_days"`
+	// Compress determines if the rotated log files should be compressed
+	// using gzip. The default is not to perform compression.
+	Compress bool `json:"compress" yaml:"compress"`
+}

src/types/process.go

@@ -17,6 +17,7 @@ type ProcessConfig struct {
 	IsDaemon          bool                   `yaml:"is_daemon,omitempty"`
 	Command           string                 `yaml:"command"`
 	LogLocation       string                 `yaml:"log_location,omitempty"`
+	LogRotation       *LogRotationConfig     `yaml:"log_rotation,omitempty"`
 	Environment       Environment            `yaml:"environment,omitempty"`
 	RestartPolicy     RestartPolicyConfig    `yaml:"availability,omitempty"`
 	DependsOn         DependsOnConfig        `yaml:"depends_on,omitempty"`

src/types/project.go

@@ -11,6 +11,7 @@ type Project struct {
 	LogLocation string               `yaml:"log_location,omitempty"`
 	LogLevel    string               `yaml:"log_level,omitempty"`
 	LogLength   int                  `yaml:"log_length,omitempty"`
+	LogRotation *LogRotationConfig   `yaml:"log_rotation,omitempty"`
 	Processes   Processes            `yaml:"processes"`
 	Environment Environment          `yaml:"environment,omitempty"`
 	ShellConfig *command.ShellConfig `yaml:"shell,omitempty"`