Skip to content

Commit

Permalink
resolve cihub#110 : explode archived logs.
Browse files Browse the repository at this point in the history 
**Why ?**
Popular command line utilities like `zcat` or `zgrep` are useful tools for processing log files. For instance, `zgrep` allow to search compressed files for a regular expression.
Those tools require that the analyzed archive contains only one file. Yet, when compressing archived logs, seelog put all the log files in the same archive.

So, The purpose of this pull request is to add the possibility for archived logs to be zipped in dedicated zip files.

**How ?**
As new compression formats might be introduced in future release of `seelog`, we want to clearly separate the choice of the compression algorithm to the choice of the gathering mode (grouped vs exploded).
Therefore, we propose a new attribute, `archiveexploded` (a boolean), specifying if the logs should be exploded or grouped inside the same archive file.
  • Loading branch information
Yohan Legat committed Apr 7, 2016
1 parent be5abe0 commit c434ee4
Show file tree
Hide file tree
Showing 5 changed files with 192 additions and 86 deletions.
36 changes: 27 additions & 9 deletions cfg_parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ const (
rollingFileDataPatternAttr = "datepattern"
rollingFileArchiveAttr = "archivetype"
rollingFileArchivePathAttr = "archivepath"
rollingFileArchiveExplodedAttr = "archiveexploded"
bufferedWriterID = "buffered"
bufferedSizeAttr = "size"
bufferedFlushPeriodAttr = "flushperiod"
Expand Down Expand Up @@ -1008,6 +1009,7 @@ func createRollingFileWriter(node *xmlNode, formatFromParent *formatter, formats

var rArchiveType rollingArchiveType
var rArchivePath string
var rArchiveExploded bool = false
if !archiveAttrExists {
rArchiveType = rollingArchiveNone
rArchivePath = ""
Expand All @@ -1020,12 +1022,28 @@ func createRollingFileWriter(node *xmlNode, formatFromParent *formatter, formats
if rArchiveType == rollingArchiveNone {
rArchivePath = ""
} else {
if rArchiveExplodedAttr, ok := node.attributes[rollingFileArchiveExplodedAttr]; ok {
if rArchiveExploded, err = strconv.ParseBool(rArchiveExplodedAttr); err != nil {
return nil, fmt.Errorf("archive exploded should be true or false, but was %v",
rArchiveExploded)
}
}

rArchivePath, ok = node.attributes[rollingFileArchivePathAttr]
if !ok {
rArchivePath, ok = rollingArchiveTypesDefaultNames[rArchiveType]
if !ok {
return nil, fmt.Errorf("cannot get default filename for archive type = %v",
rArchiveType)
if ok {
if rArchivePath == "" {
return nil, fmt.Errorf("empty archive path is not supported")
}
} else {
if rArchiveExploded {
rArchivePath = rollingArchiveDefaultExplodedName

} else {
rArchivePath, ok = rollingArchiveTypesDefaultNames[rArchiveType]
if !ok {
return nil, fmt.Errorf("cannot get default filename for archive type = %v",
rArchiveType)
}
}
}
}
Expand All @@ -1045,7 +1063,7 @@ func createRollingFileWriter(node *xmlNode, formatFromParent *formatter, formats
if rollingType == rollingTypeSize {
err := checkUnexpectedAttribute(node, outputFormatID, rollingFileTypeAttr, rollingFilePathAttr,
rollingFileMaxSizeAttr, rollingFileMaxRollsAttr, rollingFileArchiveAttr,
rollingFileArchivePathAttr, rollingFileNameModeAttr)
rollingFileArchivePathAttr, rollingFileArchiveExplodedAttr, rollingFileNameModeAttr)
if err != nil {
return nil, err
}
Expand All @@ -1069,7 +1087,7 @@ func createRollingFileWriter(node *xmlNode, formatFromParent *formatter, formats
}
}

rollingWriter, err := NewRollingFileWriterSize(path, rArchiveType, rArchivePath, maxSize, maxRolls, nameMode)
rollingWriter, err := NewRollingFileWriterSize(path, rArchiveType, rArchivePath, maxSize, maxRolls, nameMode, rArchiveExploded)
if err != nil {
return nil, err
}
Expand All @@ -1079,7 +1097,7 @@ func createRollingFileWriter(node *xmlNode, formatFromParent *formatter, formats
} else if rollingType == rollingTypeTime {
err := checkUnexpectedAttribute(node, outputFormatID, rollingFileTypeAttr, rollingFilePathAttr,
rollingFileDataPatternAttr, rollingFileArchiveAttr, rollingFileMaxRollsAttr,
rollingFileArchivePathAttr, rollingFileNameModeAttr)
rollingFileArchivePathAttr, rollingFileArchiveExplodedAttr, rollingFileNameModeAttr)
if err != nil {
return nil, err
}
Expand All @@ -1098,7 +1116,7 @@ func createRollingFileWriter(node *xmlNode, formatFromParent *formatter, formats
return nil, newMissingArgumentError(node.name, rollingFileDataPatternAttr)
}

rollingWriter, err := NewRollingFileWriterTime(path, rArchiveType, rArchivePath, maxRolls, dataPattern, rollingIntervalAny, nameMode)
rollingWriter, err := NewRollingFileWriterTime(path, rArchiveType, rArchivePath, maxRolls, dataPattern, rollingIntervalAny, nameMode, rArchiveExploded)
if err != nil {
return nil, err
}
Expand Down
58 changes: 51 additions & 7 deletions cfg_parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ func getParserTests() []parserTest {
testExpected = new(configForParsing)
testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl)
testExpected.Exceptions = nil
testrollingFileWriter, _ := NewRollingFileWriterSize(testLogFileName, rollingArchiveNone, "", 100, 5, rollingNameModePostfix)
testrollingFileWriter, _ := NewRollingFileWriterSize(testLogFileName, rollingArchiveNone, "", 100, 5, rollingNameModePostfix, false)
testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriter})
testExpected.LogType = syncloggerTypeFromString
testExpected.RootDispatcher = testHeadSplitter
Expand All @@ -286,7 +286,7 @@ func getParserTests() []parserTest {
testExpected = new(configForParsing)
testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl)
testExpected.Exceptions = nil
testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveZip, "log.zip", 100, 5, rollingNameModePostfix)
testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveZip, "log.zip", 100, 5, rollingNameModePostfix, false)
testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriter})
testExpected.LogType = syncloggerTypeFromString
testExpected.RootDispatcher = testHeadSplitter
Expand All @@ -303,7 +303,41 @@ func getParserTests() []parserTest {
testExpected = new(configForParsing)
testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl)
testExpected.Exceptions = nil
testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveZip, "test.zip", 100, 5, rollingNameModePrefix)
testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveZip, "test.zip", 100, 5, rollingNameModePrefix, false)
testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriter})
testExpected.LogType = syncloggerTypeFromString
testExpected.RootDispatcher = testHeadSplitter
parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil})

testName = "Rolling file writer archive zip exploded"
testLogFileName = getTestFileName(testName, "")
testConfig = `
<seelog type="sync">
<outputs>
<rollingfile type="size" filename="` + testLogFileName + `" maxsize="100" maxrolls="5" archivetype="zip" archiveexploded="true"/>
</outputs>
</seelog>`
testExpected = new(configForParsing)
testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl)
testExpected.Exceptions = nil
testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveZip, "old", 100, 5, rollingNameModePostfix , true)
testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriter})
testExpected.LogType = syncloggerTypeFromString
testExpected.RootDispatcher = testHeadSplitter
parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil})

testName = "Rolling file writer archive zip exploded with specified path"
testLogFileName = getTestFileName(testName, "")
testConfig = `
<seelog type="sync">
<outputs>
<rollingfile namemode="prefix" type="size" filename="` + testLogFileName + `" maxsize="100" maxrolls="5" archivetype="zip" archiveexploded="true" archivepath="test_old_logs"/>
</outputs>
</seelog>`
testExpected = new(configForParsing)
testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl)
testExpected.Exceptions = nil
testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveZip, "test_old_logs", 100, 5, rollingNameModePrefix, true)
testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriter})
testExpected.LogType = syncloggerTypeFromString
testExpected.RootDispatcher = testHeadSplitter
Expand All @@ -320,7 +354,7 @@ func getParserTests() []parserTest {
testExpected = new(configForParsing)
testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl)
testExpected.Exceptions = nil
testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveNone, "", 100, 5, rollingNameModePostfix)
testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveNone, "", 100, 5, rollingNameModePostfix, false)
testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriter})
testExpected.LogType = syncloggerTypeFromString
testExpected.RootDispatcher = testHeadSplitter
Expand All @@ -337,7 +371,7 @@ func getParserTests() []parserTest {
testExpected = new(configForParsing)
testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl)
testExpected.Exceptions = nil
testrollingFileWriterTime, _ := NewRollingFileWriterTime(testLogFileName, rollingArchiveNone, "", 0, "2006-01-02T15:04:05Z07:00", rollingIntervalAny, rollingNameModePostfix)
testrollingFileWriterTime, _ := NewRollingFileWriterTime(testLogFileName, rollingArchiveNone, "", 0, "2006-01-02T15:04:05Z07:00", rollingIntervalAny, rollingNameModePostfix, false)
testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriterTime})
testExpected.LogType = syncloggerTypeFromString
testExpected.RootDispatcher = testHeadSplitter
Expand All @@ -356,7 +390,7 @@ func getParserTests() []parserTest {
testExpected = new(configForParsing)
testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl)
testExpected.Exceptions = nil
testrollingFileWriterTime, _ = NewRollingFileWriterTime(testLogFileName, rollingArchiveNone, "", 0, "2006-01-02T15:04:05Z07:00", rollingIntervalDaily, rollingNameModePostfix)
testrollingFileWriterTime, _ = NewRollingFileWriterTime(testLogFileName, rollingArchiveNone, "", 0, "2006-01-02T15:04:05Z07:00", rollingIntervalDaily, rollingNameModePostfix, false)
testbufferedWriter, _ := NewBufferedWriter(testrollingFileWriterTime, 100500, 100)
testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testbufferedWriter})
testExpected.LogType = syncloggerTypeFromString
Expand Down Expand Up @@ -987,6 +1021,16 @@ func getParserTests() []parserTest {
</seelog>`
parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil})

testName = "Errors #27"
testLogFileName = getTestFileName(testName, "")
testConfig = `
<seelog type="sync">
<outputs>
<rollingfile type="size" filename="` + testLogFileName + `" maxsize="100" maxrolls="5" archivetype="zip" archivepath="" />
</outputs>
</seelog>`
parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil})

testName = "Buffered writer same formatid override"
testLogFileName = getTestFileName(testName, "")
testConfig = `
Expand All @@ -1003,7 +1047,7 @@ func getParserTests() []parserTest {
testExpected = new(configForParsing)
testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl)
testExpected.Exceptions = nil
testrollingFileWriterTime, _ = NewRollingFileWriterTime(testLogFileName, rollingArchiveNone, "", 0, "2006-01-02T15:04:05Z07:00", rollingIntervalDaily, rollingNameModePrefix)
testrollingFileWriterTime, _ = NewRollingFileWriterTime(testLogFileName, rollingArchiveNone, "", 0, "2006-01-02T15:04:05Z07:00", rollingIntervalDaily, rollingNameModePrefix, false)
testbufferedWriter, _ = NewBufferedWriter(testrollingFileWriterTime, 100500, 100)
testFormat, _ = NewFormatter("%Level %Msg %File 123")
formattedWriter, _ = NewFormattedWriter(testbufferedWriter, testFormat)
Expand Down
25 changes: 14 additions & 11 deletions writers_filewriter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,19 +52,22 @@ func simplefileWriterGetter(testCase *fileWriterTestCase) (io.WriteCloser, error

//===============================================================
type fileWriterTestCase struct {
files []string
fileName string
rollingType rollingType
fileSize int64
maxRolls int
datePattern string
writeCount int
resFiles []string
nameMode rollingNameMode
files []string
fileName string
rollingType rollingType
fileSize int64
maxRolls int
datePattern string
writeCount int
resFiles []string
nameMode rollingNameMode
archiveType rollingArchiveType
archiveExploded bool
archivePath string
}

func createSimplefileWriterTestCase(fileName string, writeCount int) *fileWriterTestCase {
return &fileWriterTestCase{[]string{}, fileName, rollingTypeSize, 0, 0, "", writeCount, []string{fileName}, 0}
return &fileWriterTestCase{[]string{}, fileName, rollingTypeSize, 0, 0, "", writeCount, []string{fileName}, 0, rollingArchiveNone, false, ""}
}

var simplefileWriterTests = []*fileWriterTestCase{
Expand All @@ -90,7 +93,7 @@ func NewFileWriterTester(
}

func isWriterTestFile(fn string) bool {
return strings.Contains(fn, ".testlog")
return strings.Contains(fn, ".testlog") || strings.Contains(fn, ".zip")
}

func cleanupWriterTest(t *testing.T) {
Expand Down
Loading

0 comments on commit c434ee4

Please sign in to comment.