Don't preformat loglines, we fully use structured logging now

This reduces the logfile size significantly nearly cutting it into
halves.

Monal/Classes/HelperTools.h

@@ -27,7 +27,6 @@ NS_ASSUME_NONNULL_BEGIN
 @class xmpp;
 @class XMPPStanza;
 @class UNNotificationRequest;
-@class DDLogFormatter;
 @class DDLogMessage;
 @class MLFileLogger;
 @class UIView;
@@ -40,7 +39,7 @@ void swizzle(Class c, SEL orig, SEL new);
 
 @property (class, nonatomic, strong) MLFileLogger* fileLogger;
 
-+(NSData* _Nullable) convertLogmessageToJsonData:(DDLogMessage*) logMessage usingFormatter:(id<DDLogFormatter> _Nullable) formatter counter:(uint64_t*) counter andError:(NSError** _Nullable) error;
++(NSData* _Nullable) convertLogmessageToJsonData:(DDLogMessage*) logMessage counter:(uint64_t*) counter andError:(NSError** _Nullable) error;
 +(void) initSystem;
 +(void) installExceptionHandler;
 +(int) pendingCrashreportCount;

Monal/Classes/HelperTools.m

@@ -15,6 +15,7 @@
 #include <objc/runtime.h> 
 #include <objc/message.h>
 #include <objc/objc-exception.h>
+#import <sys/qos.h>
 #import <CommonCrypto/CommonDigest.h>
 #import <CommonCrypto/CommonHMAC.h>
 #import <MapKit/MapKit.h>
@@ -1374,11 +1375,10 @@ +(void) activityLog
     if(log_activity)
     {
         dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_BACKGROUND, 0), ^{
-            BOOL appex = [HelperTools isAppExtension];
             unsigned long counter = 1;
             while(counter++)
             {
-                DDLogInfo(@"activity(%@): %lu, memory used / available: %.3fMiB / %.3fMiB", appex ? @"APPEX" : @"MAINAPP", counter, [self report_memory], (CGFloat)os_proc_available_memory() / 1048576);
+                DDLogInfo(@"activity: %lu, memory used / available: %.3fMiB / %.3fMiB", counter, [self report_memory], (CGFloat)os_proc_available_memory() / 1048576);
                 [NSThread sleepForTimeInterval:1];
             }
         });
@@ -1405,26 +1405,42 @@ +(void) setFileLogger:(DDFileLogger*) fileLogger
     _fileLogger = fileLogger;
 }
 
-+(NSData* _Nullable) convertLogmessageToJsonData:(DDLogMessage*) logMessage usingFormatter:(id<DDLogFormatter> _Nullable) formatter counter:(uint64_t*) counter andError:(NSError** _Nullable) error
++(NSData* _Nullable) convertLogmessageToJsonData:(DDLogMessage*) logMessage counter:(uint64_t*) counter andError:(NSError** _Nullable) error
 {
-    //format message using given formatter
-    NSString* logMsg = logMessage.message;
-    NSString* timestamp = [[NSISO8601DateFormatter new] stringFromDate:logMessage.timestamp];
-    if(formatter)
-    {
-        logMsg = [NSString stringWithFormat:@"%@", [formatter formatLogMessage:logMessage]];
-        timestamp = [(MLLogFormatter*)formatter stringFromDate:logMessage.timestamp];
-    }
+    static NSDateFormatter* dateFormatter = nil;
+    static NSString* (^qos2name)(NSUInteger) = nil;
+    static dispatch_once_t onceToken;
+    dispatch_once(&onceToken, ^{
+        dateFormatter = [[NSDateFormatter alloc] init];
+        [dateFormatter setFormatterBehavior:NSDateFormatterBehavior10_4];
+        [dateFormatter setDateFormat:@"yyyy-MM-dd HH:mm:ss:SSS"];
+        [dateFormatter setLocale:[NSLocale localeWithLocaleIdentifier:@"en_US_POSIX"]];
+        [dateFormatter setCalendar:[[NSCalendar alloc] initWithCalendarIdentifier:NSCalendarIdentifierGregorian]];
+
+        qos2name = ^(NSUInteger qos) {
+            switch ((qos_class_t) qos) {
+                case QOS_CLASS_USER_INTERACTIVE: return @"QOS_CLASS_USER_INTERACTIVE";
+                case QOS_CLASS_USER_INITIATED:   return @"QOS_CLASS_USER_INITIATED";
+                case QOS_CLASS_DEFAULT:          return @"QOS_CLASS_DEFAULT";
+                case QOS_CLASS_UTILITY:          return @"QOS_CLASS_UTILITY";
+                case QOS_CLASS_BACKGROUND:       return @"QOS_CLASS_BACKGROUND";
+                default:                         return [NSString stringWithFormat:@"QOS_UNKNOWN(%lu)", (unsigned long)qos];
+            }
+        };
+    });
 
     //construct json dictionary
     (*counter)++;
     NSDictionary* representedObject = @{
         @"queueThreadLabel": [self getQueueThreadLabelFor:logMessage],
         @"processType": [self isAppExtension] ? @"appex" : @"mainapp",
-        @"representedObject": logMessage.representedObject ? logMessage.representedObject : [NSNull null]
+        @"processName": [[[NSBundle mainBundle] executablePath] lastPathComponent],
+        @"counter": [NSNumber numberWithUnsignedLongLong:*counter],
+        @"processID": _processID,
+        @"qosName": qos2name(logMessage.qos),
+        @"representedObject": logMessage.representedObject ? logMessage.representedObject : [NSNull null],
     };
     NSDictionary* msgDict = @{
-        @"formattedMessage": logMsg,
         @"messageFormat": logMessage.messageFormat,
         @"message": logMessage.message,
         @"level": [NSNumber numberWithInteger:logMessage.level],
@@ -1436,13 +1452,11 @@ +(NSData* _Nullable) convertLogmessageToJsonData:(DDLogMessage*) logMessage usin
         @"line": [NSNumber numberWithInteger:logMessage.line],
         @"tag": representedObject,
         @"options": [NSNumber numberWithInteger:logMessage.options],
-        @"timestamp": timestamp,
+        @"timestamp": [dateFormatter stringFromDate:logMessage.timestamp],
         @"threadID": logMessage.threadID,
         @"threadName": logMessage.threadName,
         @"queueLabel": logMessage.queueLabel,
         @"qos": [NSNumber numberWithInteger:logMessage.qos],
-        @"_counter": [NSNumber numberWithUnsignedLongLong:*counter],
-        @"_processID": _processID,
     };
 
     //encode json into NSData
@@ -1467,22 +1481,16 @@ +(void) flushLogsWithTimeout:(double) timeout
 
 +(void) configureLogging
 {
-    //create log formatter
-    MLLogFormatter* formatter = [MLLogFormatter new];
-
     //don't log to the console (aka stderr) to not create loops with our redirected stderr
 //     //start console logger first (this one will *not* log own additional (and duplicated) informations like DDOSLogger would)
 // #if TARGET_OS_SIMULATOR
-//     [[DDTTYLogger sharedInstance] setLogFormatter:formatter];
 //     [DDLog addLogger:[DDTTYLogger sharedInstance]];
 // #else
-//     [[DDOSLogger sharedInstance] setLogFormatter:formatter];
 //     [DDLog addLogger:[DDOSLogger sharedInstance]];
 // #endif
 
     //network logger (start as early as possible)
     MLUDPLogger* udpLogger = [MLUDPLogger new];
-    [udpLogger setLogFormatter:formatter];
     [DDLog addLogger:udpLogger];
 
     //redirect stderr containing NSLog() messages
@@ -1503,7 +1511,6 @@ +(void) configureLogging
     self.fileLogger.doNotReuseLogFiles = NO;
     self.fileLogger.rollingFrequency = 60 * 60 * 48;    // 48 hour rolling
     self.fileLogger.maximumFileSize = 128 * 1024 * 1024;
-    self.fileLogger.logFormatter = formatter;
     self.fileLogger.archiveAllowed = YES;               //everything is configured now, engage logfile archiving
     [DDLog addLogger:self.fileLogger];
 
@@ -1524,6 +1531,12 @@ +(void) configureLogging
     DDLogInfo(@"Starting: Version %@ (%@ %@ UTC, %@)", version, buildDate, buildTime, [[NSBundle mainBundle] objectForInfoDictionaryKey:@"CFBundleIdentifier"]);
     [DDLog flushLog];
 
+    DDLogVerbose(@"QOS level: %@ = %d", @"QOS_CLASS_USER_INTERACTIVE", QOS_CLASS_USER_INTERACTIVE);
+    DDLogVerbose(@"QOS level: %@ = %d", @"QOS_CLASS_USER_INITIATED", QOS_CLASS_USER_INITIATED);
+    DDLogVerbose(@"QOS level: %@ = %d", @"QOS_CLASS_DEFAULT", QOS_CLASS_DEFAULT);
+    DDLogVerbose(@"QOS level: %@ = %d", @"QOS_CLASS_UTILITY", QOS_CLASS_UTILITY);
+    DDLogVerbose(@"QOS level: %@ = %d", @"QOS_CLASS_BACKGROUND", QOS_CLASS_BACKGROUND);
+
     //remove old ascii based logfiles
     for(NSString* file in [[[NSFileManager defaultManager] contentsOfDirectoryAtPath:containerUrl error:nil] filteredArrayUsingPredicate:[NSPredicate predicateWithFormat:@"self LIKE %@", @"Monal *.log"]])
     {
@@ -1610,6 +1623,7 @@ +(void) installCrashHandler
         NSString* buildTime = [NSString stringWithUTF8String:__TIME__];
         handler.userInfo = @{
             @"isAppex": @([self isAppExtension]),
+            @"processName": [[[NSBundle mainBundle] executablePath] lastPathComponent],
             @"bundleName": nilWrapper([[[NSBundle mainBundle] infoDictionary] objectForKey:@"CFBundleName"]),
             @"appVersion": [NSString stringWithFormat:NSLocalizedString(@"Version %@ (%@ %@ UTC)", @""), version, buildDate, buildTime],
         };

Monal/Classes/MLConstants.h

@@ -20,7 +20,6 @@
 #define DDLogStdout(frmt, ...)  LOG_MAYBE(NO,  ddLogLevel, LOG_FLAG_STDOUT,  0, nil, __PRETTY_FUNCTION__, frmt, ##__VA_ARGS__)
 static const DDLogLevel ddLogLevel = LOG_LEVEL_STDOUT;
 #import "MLLogFileManager.h"
-#import "MLLogFormatter.h"
 #import "MLFileLogger.h"
 
 

Monal/Classes/MLFileLogger.m

@@ -115,7 +115,7 @@ -(NSData*) lt_dataForMessage:(DDLogMessage*) logMessage
 
     //encode log message
     NSError* error;
-    NSData* rawData = [HelperTools convertLogmessageToJsonData:logMessage usingFormatter:_logFormatter counter:&counter andError:&error];
+    NSData* rawData = [HelperTools convertLogmessageToJsonData:logMessage counter:&counter andError:&error];
     if(error != nil || rawData == nil)
     {
         NSLog(@"Error jsonifying log message: %@, logMessage: %@", error, logMessage);

Monal/Classes/MLUDPLogger.m

@@ -233,16 +233,13 @@ -(void) logMessage:(DDLogMessage*) logMessage
         return;
 
     NSError* error = nil; 
-    NSData* rawData = [HelperTools convertLogmessageToJsonData:logMessage usingFormatter:self->_logFormatter counter:&counter andError:&error];
+    NSData* rawData = [HelperTools convertLogmessageToJsonData:logMessage counter:&counter andError:&error];
     if(error != nil || rawData == nil)
     {
         [[self class] logError:@"json encode error: %@", error];
         return;
     }
 
-    //you have to uncomment the following line to send only the formatted logline
-    //rawData = [logMsg dataUsingEncoding:NSUTF8StringEncoding];
-
     //compress data to account for udp size limits
     rawData = [self gzipDeflate:rawData];
 

Monal/Monal.xcodeproj/project.pbxproj

@@ -100,8 +100,6 @@
 		540E13A024CDCDB30038FDA0 /* MLProcessLock.m in Sources */ = {isa = PBXBuildFile; fileRef = 540E139F24CDCDB30038FDA0 /* MLProcessLock.m */; };
 		540E13A224CDCE3B0038FDA0 /* MLProcessLock.h in Headers */ = {isa = PBXBuildFile; fileRef = 540E13A124CDCE3B0038FDA0 /* MLProcessLock.h */; };
 		540E13A524CF6A8C0038FDA0 /* MLNotificationManager.m in Sources */ = {isa = PBXBuildFile; fileRef = 26EE8B1E179B67FA006781F3 /* MLNotificationManager.m */; };
-		540E13A724CF78900038FDA0 /* MLLogFormatter.m in Sources */ = {isa = PBXBuildFile; fileRef = 540E13A624CF78900038FDA0 /* MLLogFormatter.m */; };
-		540E13A924CF78D00038FDA0 /* MLLogFormatter.h in Headers */ = {isa = PBXBuildFile; fileRef = 540E13A824CF78D00038FDA0 /* MLLogFormatter.h */; };
 		540F625F24BA951E0008A6D8 /* HelperTools.m in Sources */ = {isa = PBXBuildFile; fileRef = C1AAC3E324B5EF4100BB15D6 /* HelperTools.m */; };
 		54179CC0251CBAFA008F398E /* XMPPStanza.m in Sources */ = {isa = PBXBuildFile; fileRef = 54179CBF251CBAF9008F398E /* XMPPStanza.m */; };
 		54179CC3251CBB2B008F398E /* XMPPStanza.h in Headers */ = {isa = PBXBuildFile; fileRef = 54179CC2251CBB2B008F398E /* XMPPStanza.h */; };
@@ -515,8 +513,6 @@
 		540BD0D324D8D1FF0087A743 /* IPC.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = IPC.m; sourceTree = "<group>"; };
 		540E139F24CDCDB30038FDA0 /* MLProcessLock.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = MLProcessLock.m; sourceTree = "<group>"; };
 		540E13A124CDCE3B0038FDA0 /* MLProcessLock.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MLProcessLock.h; sourceTree = "<group>"; };
-		540E13A624CF78900038FDA0 /* MLLogFormatter.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = MLLogFormatter.m; sourceTree = "<group>"; };
-		540E13A824CF78D00038FDA0 /* MLLogFormatter.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MLLogFormatter.h; sourceTree = "<group>"; };
 		54179CBF251CBAF9008F398E /* XMPPStanza.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = XMPPStanza.m; sourceTree = "<group>"; };
 		54179CC2251CBB2B008F398E /* XMPPStanza.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = XMPPStanza.h; sourceTree = "<group>"; };
 		541B6AB2262BC9040038B936 /* MLStream.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MLStream.m; sourceTree = "<group>"; };
@@ -1227,8 +1223,6 @@
 				C1AAC3E324B5EF4100BB15D6 /* HelperTools.m */,
 				540E139F24CDCDB30038FDA0 /* MLProcessLock.m */,
 				540E13A124CDCE3B0038FDA0 /* MLProcessLock.h */,
-				540E13A624CF78900038FDA0 /* MLLogFormatter.m */,
-				540E13A824CF78D00038FDA0 /* MLLogFormatter.h */,
 				541E4CBD254AA0B600FD7B28 /* MLHandler.h */,
 				541E4CBF254AA0E700FD7B28 /* MLHandler.m */,
 				C10490482612ED2F0054AC9E /* MLEmoji.swift */,
@@ -1457,7 +1451,6 @@
 				542CF4002763314F002C3710 /* hsluv.h in Headers */,
 				54E594BD2523C34B00E4172B /* MLPubSub.h in Headers */,
 				540BD0D224D8D1F40087A743 /* IPC.h in Headers */,
-				540E13A924CF78D00038FDA0 /* MLLogFormatter.h in Headers */,
 				C1C839DD24F15DF800BBCF17 /* MLOMEMO.h in Headers */,
 				841EE42B2A3F471100D3AF14 /* MLFileLogger.h in Headers */,
 				26D4389123A5EB6C00242AAA /* MLConstants.h in Headers */,
@@ -2128,7 +2121,6 @@
 				26CC57A223A086AA00ABB92A /* xmpp.m in Sources */,
 				26CC57B423A086CC00ABB92A /* XMPPPresence.m in Sources */,
 				541E4CC0254AA0E700FD7B28 /* MLHandler.m in Sources */,
-				540E13A724CF78900038FDA0 /* MLLogFormatter.m in Sources */,
 				841EE4292A3F46F700D3AF14 /* MLFileLogger.m in Sources */,
 				C158D41425A0AC630005AA40 /* MLMucProcessor.m in Sources */,
 				C16D18362792A4AF00F869A0 /* DataLayerMigrations.m in Sources */,