From ae256fd530d8233752418c00b7f77714a0dfe786 Mon Sep 17 00:00:00 2001 From: "Ling Wang@g731gw" Date: Tue, 1 Sep 2020 23:26:08 +0200 Subject: [PATCH 1/5] fix for meson > 0.52; actually use log level --- mcw/cmake.py | 103 +++++++++++++++++++++++++++------------------- mcw/logging.py | 10 +++++ mcw/meson.py | 33 ++++++++------- mcw/ninja.py | 2 +- mcw/server.py | 31 +++++++------- test/cmake-client | 16 +++---- 6 files changed, 111 insertions(+), 84 deletions(-) diff --git a/mcw/cmake.py b/mcw/cmake.py index c5c93f8..04696ae 100644 --- a/mcw/cmake.py +++ b/mcw/cmake.py @@ -1,3 +1,4 @@ +from _datetime import datetime import os import sys import pickle @@ -6,22 +7,31 @@ import json import xml.etree.ElementTree as ETree -from .logging import ServerLogHandler +from .logging import ServerLogHandler, logger_fmt, logger_fmt_with_func from .util import find_executables from .commandtool import CommandToolWrapper from .server import (UnixSocketServer, NamedPipeServer) from .meson import Meson +from typing import NamedTuple, List, Iterable, Union +class cmake_target_t(NamedTuple): + name: str + id: str + type: str + filename: Union[None, str] + class CMakeWrapper: """ Class that emulates CMake commands and translates them to the equivalent in Meson. """ + debug: int + logger: logging.Logger def __init__(self): self.version = [3, 10, 0] self.path = sys.argv[0] - self.debug = False + self.debug = 1 self.command = 'generate' self.generator = None self.build_type = None @@ -37,13 +47,13 @@ def __init__(self): else: self.server = UnixSocketServer(self) self.tool = CommandToolWrapper(self) - self.logger = None + self.init_logging() def run(self, args): self.init_logging() - self.log('(args) "%s"' % args) - self.log('(cwd) "%s"' % os.getcwd()) + self.logger.info('(args) "%s"' % args) + self.logger.info('(cwd) "%s"' % os.getcwd()) # debug_connect() @@ -59,10 +69,10 @@ def run(self, args): getattr(self, self.command + '_cmd')() except RuntimeError as e: print(e.args[0]) - self.log(e) + self.logger.error(e.args[0], exc_info=e) exit(1) except Exception as e: - self.log(e) + self.logger.error('Uncaught error.', exc_info=e) raise e self.save_cache_entries() @@ -200,55 +210,60 @@ def tool_cmd(self): self.tool.run(self.command_args) def init_logging(self, dir=None): + + if 'MCW_LOG_LEVEL' in os.environ: + self.debug = int(os.environ['MCW_LOG_LEVEL']) + else: + self.debug = 1 # Setup loggers loggers = [] self.logger = logging.getLogger('CMake Wrapper') loggers.append(self.logger) - self.meson.logger = logging.getLogger('Meson') - loggers.append(self.meson.logger) - self.server.logger = logging.getLogger('Server') - loggers.append(self.server.logger) + loggers.append(self.meson.get_logger()) + loggers.append(self.server.get_logger()) # Cleanup if reinitialized for logger in loggers: logger.handlers = [] - if not dir and self.debug: + if not dir and self.debug > 0: dir = os.getcwd() # Setup handlers and formatters handlers = [] if dir: - handler = logging.FileHandler(os.path.join(dir, 'meson-cmake-wrapper.log')) - handler.setLevel(logging.INFO) - formatter = logging.Formatter('%(asctime)s - %(name)s: %(message)s') - handler.setFormatter(formatter) - handlers.append(handler) - - if self.debug: - handler = logging.StreamHandler(sys.stderr) + datetime_str = datetime.now().strftime('%Y%m%d-%H%M%S.%f') + handler = logging.FileHandler(os.path.join(dir, 'meson-cmake-wrapper-{}.log'.format(datetime_str))) handler.setLevel(logging.INFO) - formatter = logging.Formatter('%(name)s: %(message)s') - handler.setFormatter(formatter) + handler.setFormatter(logger_fmt_with_func) handlers.append(handler) - - handler = ServerLogHandler(self.server) - handler.setLevel(logging.INFO) - formatter = logging.Formatter('%(name)s: %(message)s') - handler.setFormatter(formatter) - handlers.append(handler) - + stderr_handler = logging.StreamHandler(sys.stderr) + stderr_handler.setLevel(logging.WARN) + stderr_handler.setFormatter(logger_fmt) + handlers.append(stderr_handler) + server_handler = ServerLogHandler(self.server) + server_handler.setLevel(logging.WARN) + server_handler.setFormatter(logger_fmt) + handlers.append(server_handler) + logger_level = None + if self.debug == 0: + logger_level = logging.WARN + if self.debug == 2: + logger_level = logging.INFO + if self.debug >= 3: + logger_level = logging.DEBUG + + for handler in handlers: + if logger_level is not None: + handler.setLevel(logger_level) + if self.debug > 1: + handler.setFormatter(logger_fmt_with_func) for logger in loggers: - logger.setLevel(logging.INFO) + if logger_level: + logger.setLevel(logger_level) for handler in handlers: logger.addHandler(handler) - def log(self, msg): - if isinstance(msg, Exception): - self.logger.info(msg, exc_info=msg) - else: - self.logger.info(msg) - def set_generator(self, generator): if generator == 'Ninja': self.meson.set_backend('ninja') @@ -264,7 +279,7 @@ def set_generator(self, generator): raise Exception('Generator not supported: ' + generator) self.generator = generator - self.log('(generator) "%s"' % self.generator) + self.logger.info('(generator) "%s"' % self.generator) def set_source_dir(self, source_dir): if not os.path.exists(source_dir): @@ -275,7 +290,7 @@ def set_source_dir(self, source_dir): self.source_dir = os.path.abspath(source_dir) self.meson.source_dir = self.source_dir - self.log('(source_dir) "%s"' % self.source_dir) + self.logger.info('(source_dir) "%s"' % self.source_dir) def set_build_dir(self, build_dir): if not os.path.exists(build_dir): @@ -296,7 +311,7 @@ def set_build_dir(self, build_dir): with open(meson_file, 'w') as file: file.write('project(\'empty\')') - self.log('(build_dir) "%s"' % self.build_dir) + self.logger.info('(build_dir) "%s"' % self.build_dir) def set_build_type(self, build_type): self.build_type = build_type @@ -313,7 +328,7 @@ def set_build_type(self, build_type): self.cache_entries['CMAKE_BUILD_TYPE'] = (build_type, 'STRING') - self.log('(build_type) "%s"' % self.build_type) + self.logger.info('(build_type) "%s"' % self.build_type) def get_entry(self, entry): if entry in self.cache_entries: @@ -416,6 +431,7 @@ def gen_cmake_cache(self): file.write('%s:%s=%s\n' % (entry[0], entry[1][1], entry[1][0])) def gen_cmake_project(self): + self.logger.warning('call gen_cmake_project()') with open(os.path.join(self.source_dir, 'CMakeLists.txt'), 'w') as file: file.write('cmake_minimum_required(VERSION %s)\n' % '.'.join(map(str, self.version))) file.write('project(%s)\n\n' % self.meson.get_project_name()) @@ -458,7 +474,7 @@ def gen_codeblocks_project(self): 'name': 'all', 'id': 'all', 'type': 'custom', - 'filename': '' + 'filename': [''] } for target in [all_target] + self.meson.get_targets(): @@ -473,13 +489,14 @@ def gen_codeblocks_project(self): 'static library': '2', 'shared library': '3', 'custom': '4', - 'run': '4' + 'run': '4', + 'shared module': '3' # I do not use Code::Blocks, does it has shared module type as cmake and meson? }[target['type']] ETree.SubElement(build_target, 'Option', {'type': ty}) compiler = self.meson.get_compiler(target) if compiler: - ETree.SubElement(build_target, 'Option', {'compiler': 'gcc'}) + ETree.SubElement(build_target, 'Option', {'compiler': compiler}) compiler = ETree.SubElement(build_target, 'Compiler') for define in self.meson.get_defines(target): diff --git a/mcw/logging.py b/mcw/logging.py index 916f88e..938533e 100644 --- a/mcw/logging.py +++ b/mcw/logging.py @@ -1,5 +1,15 @@ import logging +logger_fmt_with_func = logging.Formatter( + '%(asctime)s - %(name)s' + '-%(funcName)s()' + ': %(message)s' +) + +logger_fmt = logging.Formatter( + '%(asctime)s - %(name)s' + ': %(message)s' +) class ServerLogHandler(logging.Handler): def __init__(self, server): diff --git a/mcw/meson.py b/mcw/meson.py index f10448a..f80ec81 100644 --- a/mcw/meson.py +++ b/mcw/meson.py @@ -1,6 +1,7 @@ import os import json import subprocess +import logging from .ninja import NinjaBackend @@ -28,14 +29,12 @@ def __init__(self, path='meson'): self.c_compile_commands = None self.c_compile_commands_target = {} self.c_default_inc_dirs = {} + self.logger = logging.getLogger('Meson') - def log(self, msg): - if isinstance(msg, Exception): - self.logger.info(msg, exc_info=msg) - else: - self.logger.info(msg) + def get_logger(self): + return self.logger - def call(self, args, show=False): + def call(self, args, show=False) -> str: child = subprocess.Popen([self.path] + args, stdout=subprocess.PIPE) fulloutput = b'' while True: @@ -89,8 +88,10 @@ def get_project_name(self): def get_targets(self): if not self.c_targets: output = self.call(['introspect', '--targets', self.build_dir]) - self.log('(targets) "%s"' % output) + self.logger.debug('(targets) "%s"' % output) self.c_targets = json.loads(output) + target_names = [ i ['name'] for i in self.c_targets] + self.logger.info('(targets) {}'.format(target_names)) return self.c_targets def get_target_files(self, target): @@ -108,9 +109,9 @@ def get_target_files(self, target): return self.c_target_files[id] # Handle meson versions before 0.50.0 - self.log('(target) "%s"' % id) + self.logger.debug('(target) "%s"' % id) output = self.call(['introspect', '--target-files', id, self.build_dir]) - self.log('(target files) "%s"' % output) + self.logger.debug('(target files) "%s"' % output) # Workaround https://github.com/mesonbuild/meson/issues/2783 if output == '': @@ -122,14 +123,14 @@ def get_target_files(self, target): def get_buildsystem_files(self): if not self.c_buildsystem_files: output = self.call(['introspect', '--buildsystem-files', self.build_dir]) - self.log('(buildsystem files) "%s"' % output) + self.logger.debug('(buildsystem files) "%s"' % output) self.c_buildsystem_files = json.loads(output) return self.c_buildsystem_files def get_project_info(self): if not self.c_project_info: output = self.call(['introspect', '--projectinfo', self.build_dir]) - self.log('(project info) "%s"' % output) + self.logger.info('(project info) "%s"' % output) self.c_project_info = json.loads(output) return self.c_project_info @@ -227,10 +228,14 @@ def get_output(self, target): return os.path.join(self.build_dir, self.get_target_filename(target)) def get_target_filename(self, target): - if self.get_version()[1] >= 50: - return target['filename'][0] + filename = target['filename'] + + if isinstance(filename, list): + if len(filename) > 1: + self.logger.warning('Target {} has more than 1 filename {}'.format(target['name'], filename)) + return filename[0] else: - return target['filename'] + return filename def get_options(self): meson_options = [] diff --git a/mcw/ninja.py b/mcw/ninja.py index 8fbb840..777d1cd 100644 --- a/mcw/ninja.py +++ b/mcw/ninja.py @@ -44,6 +44,6 @@ def build(self, target): def get_target(self, target_name): target = next((t for t in self.meson.get_targets() if t['name'] == target_name), None) if target: - return target['filename'] + return target['name'] return target_name diff --git a/mcw/server.py b/mcw/server.py index 4fbb8bb..90e9233 100644 --- a/mcw/server.py +++ b/mcw/server.py @@ -1,6 +1,7 @@ import os import json import socket +import logging SERVER_HEADER = b'\n[== "CMake Server" ==[\n' SERVER_FOOTER = b'\n]== "CMake Server" ==]\n' @@ -20,12 +21,11 @@ def __init__(self, cmake): self.requests = [] self.protocol_version = (1, 1) self.cookies = {} + self.logger = logging.getLogger('Server') + + def get_logger(self): + return self.logger - def log(self, msg): - if isinstance(msg, Exception): - self.logger.info(msg, exc_info=msg) - else: - self.logger.info(msg) def run(self, args): try: @@ -33,7 +33,7 @@ def run(self, args): self.connected = True self.handle_hello() self.handle_handshake() - self.log('running on "%s"' % self.pipe) + self.logger.info('running on "%s"' % self.pipe) while 1: request = self.recv() if not request: @@ -41,14 +41,14 @@ def run(self, args): break if not hasattr(self, 'handle_' + request['type'].lower()): - self.log('unhandled request: "%s"' % request) + self.logger.warning('unhandled request: "%s"' % request) break getattr(self, 'handle_' + request['type'].lower())(request) except BrokenPipeError: self.connected = False - self.log('lost connection to client') + self.logger.error('lost connection to client') finally: - self.log('closing connection') + self.logger.info('closing connection') self.cleanup() def connect(self, args): @@ -81,9 +81,9 @@ def send(self, response, log=True): if response['inReplyTo'] in self.cookies: response['cookie'] = self.cookies[response['inReplyTo']] if log: - self.log('%s (%s) "%s"' % (response['inReplyTo'], response['type'], response)) + self.logger.debug('%s (%s) "%s"' % (response['inReplyTo'], response['type'], response)) elif log: - self.log(response) + self.logger.debug(response) response = SERVER_HEADER + json.dumps(response).encode('utf-8') + SERVER_FOOTER self.write(response) @@ -97,7 +97,7 @@ def parse_recv(self, data): request = json.loads(request) if 'cookie' in request: self.cookies[request['type']] = request['cookie'] - self.log('received (%s) "%s"' % (request['type'], request)) + self.logger.debug('received (%s) "%s"' % (request['type'], request)) self.requests.append(request) def send_reply(self, reply_to, log=True): @@ -262,7 +262,7 @@ def get_include_paths(self, target): def get_file_groups(self, target): sources = [] for target_file in self.meson.get_target_files(target): - sources.append(os.path.relpath(target_file, os.path.dirname(target['filename']))) + sources.append(os.path.relpath(target_file, os.path.dirname(self.meson.get_target_filename(target)))) file_group = { 'isGenerated': False, 'sources': sources, @@ -302,8 +302,9 @@ def get_project(self): target['artifacts'] = [ os.path.join(self.meson.build_dir, mtarget['filename']) ] - target['buildDirectory'] = os.path.join(self.cmake.build_dir, os.path.dirname(mtarget['filename'])) - target['sourceDirectory'] = os.path.join(self.cmake.source_dir, os.path.dirname(mtarget['filename'])) + file_parent = os.path.dirname(self.meson.get_target_filename(mtarget)) + target['buildDirectory'] = os.path.join(self.cmake.build_dir, file_parent) + target['sourceDirectory'] = os.path.join(self.cmake.source_dir, file_parent) target['type'] = type_mapper[mtarget['type']] target['fileGroups'] = self.get_file_groups(mtarget) project['targets'].append(target) diff --git a/test/cmake-client b/test/cmake-client index 706358a..645c331 100755 --- a/test/cmake-client +++ b/test/cmake-client @@ -36,12 +36,6 @@ class CMakeClient: handler.setFormatter(formatter) self.logger.addHandler(handler) - def log(self, msg): - if isinstance(msg, Exception): - self.logger.info(msg, exc_info=msg) - else: - self.logger.info(msg) - def run(self): try: self.connect() @@ -70,7 +64,7 @@ class CMakeClient: except ConnectionRefusedError: print('Could not connect to: ' + self.pipe) except Exception as e: - self.log(e) + self.logger.error('Uncaught error', exc_info=e) raise e finally: self.cleanup() @@ -106,7 +100,7 @@ class CMakeClient: request['cookie'] = str(random()) self.cookies[request['type']] = request['cookie'] - self.log('Sent (%s) "%s"' % (request['type'], json.dumps(request, indent=4, sort_keys=True))) + self.logger.debug('Sent (%s) "%s"' % (request['type'], json.dumps(request, indent=4, sort_keys=True))) request = SERVER_HEADER + json.dumps(request).encode('utf-8') + SERVER_FOOTER self.write(request) @@ -118,9 +112,9 @@ class CMakeClient: for response in responses: response = json.loads(response) if 'inReplyTo' in response: - self.log('Received %s: (%s)\n%s' % (response['type'], response['inReplyTo'], json.dumps(response, indent=4, sort_keys=True))) + self.logger.debug('Received %s: (%s)\n%s' % (response['type'], response['inReplyTo'], json.dumps(response, indent=4, sort_keys=True))) else: - self.log('Received %s:\n%s' % (response['type'], json.dumps(response, indent=4, sort_keys=True))) + self.logger.debug('Received %s:\n%s' % (response['type'], json.dumps(response, indent=4, sort_keys=True))) self.responses.append(response) def check_cookie(self, response): @@ -211,7 +205,7 @@ class UnixSocketClient(CMakeClient): def connect(self): self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.sock.connect(self.pipe) - self.log('Connected to socket: ' + self.pipe) + self.logger.info('Connected to socket: ' + self.pipe) def cleanup(self): self.sock.close() From 15b74765d9795be5dc5c4dbcc76b734202339f40 Mon Sep 17 00:00:00 2001 From: "Ling Wang@g731gw" Date: Wed, 2 Sep 2020 00:12:35 +0200 Subject: [PATCH 2/5] only create one log file at each run --- mcw/cmake.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/mcw/cmake.py b/mcw/cmake.py index 04696ae..f66e12e 100644 --- a/mcw/cmake.py +++ b/mcw/cmake.py @@ -47,6 +47,7 @@ def __init__(self): else: self.server = UnixSocketServer(self) self.tool = CommandToolWrapper(self) + self._datetime_str = datetime.now().strftime('%Y%m%d-%H%M%S.%f') self.init_logging() def run(self, args): @@ -231,12 +232,13 @@ def init_logging(self, dir=None): # Setup handlers and formatters handlers = [] + file_handler = None if dir: - datetime_str = datetime.now().strftime('%Y%m%d-%H%M%S.%f') - handler = logging.FileHandler(os.path.join(dir, 'meson-cmake-wrapper-{}.log'.format(datetime_str))) - handler.setLevel(logging.INFO) - handler.setFormatter(logger_fmt_with_func) - handlers.append(handler) + + file_handler = logging.FileHandler(os.path.join(dir, 'meson-cmake-wrapper-{}.log'.format(self._datetime_str))) + file_handler.setLevel(logging.INFO) + file_handler.setFormatter(logger_fmt_with_func) + handlers.append(file_handler) stderr_handler = logging.StreamHandler(sys.stderr) stderr_handler.setLevel(logging.WARN) stderr_handler.setFormatter(logger_fmt) @@ -245,24 +247,27 @@ def init_logging(self, dir=None): server_handler.setLevel(logging.WARN) server_handler.setFormatter(logger_fmt) handlers.append(server_handler) - logger_level = None + logger_level = logging.INFO if self.debug == 0: + for handler in handlers: + handler.setLevel(logging.WARN) logger_level = logging.WARN - if self.debug == 2: + if self.debug >= 2: + server_handler.setLevel(logging.INFO) logger_level = logging.INFO if self.debug >= 3: + if file_handler: + file_handler.setLevel(logging.DEBUG) logger_level = logging.DEBUG - for handler in handlers: - if logger_level is not None: - handler.setLevel(logger_level) + for file_handler in handlers: if self.debug > 1: - handler.setFormatter(logger_fmt_with_func) + file_handler.setFormatter(logger_fmt_with_func) for logger in loggers: if logger_level: logger.setLevel(logger_level) - for handler in handlers: - logger.addHandler(handler) + for file_handler in handlers: + logger.addHandler(file_handler) def set_generator(self, generator): if generator == 'Ninja': @@ -490,7 +495,7 @@ def gen_codeblocks_project(self): 'shared library': '3', 'custom': '4', 'run': '4', - 'shared module': '3' # I do not use Code::Blocks, does it has shared module type as cmake and meson? + 'shared module': '3' # Code::Blocks does not have shared module type as cmake and meson }[target['type']] ETree.SubElement(build_target, 'Option', {'type': ty}) From e4e3408aa398431dc6a3cd44c7bb1edbb29f478a Mon Sep 17 00:00:00 2001 From: "Ling Wang@g731gw" Date: Wed, 2 Sep 2020 00:20:02 +0200 Subject: [PATCH 3/5] control log to different files while env `MCW_LOG_TO_DIFFERENT_FILES` --- mcw/cmake.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/mcw/cmake.py b/mcw/cmake.py index f66e12e..4faf890 100644 --- a/mcw/cmake.py +++ b/mcw/cmake.py @@ -48,6 +48,14 @@ def __init__(self): self.server = UnixSocketServer(self) self.tool = CommandToolWrapper(self) self._datetime_str = datetime.now().strftime('%Y%m%d-%H%M%S.%f') + if 'MCW_LOG_LEVEL' in os.environ: + self.debug = int(os.environ['MCW_LOG_LEVEL']) + else: + self.debug = 1 + if 'MCW_LOG_TO_DIFFERENT_FILES' in os.environ: + self._log_to_different_files = bool(os.environ['MCW_LOG_TO_DIFFERENT_FILES']) + else: + self._log_to_different_files = False self.init_logging() def run(self, args): @@ -211,11 +219,6 @@ def tool_cmd(self): self.tool.run(self.command_args) def init_logging(self, dir=None): - - if 'MCW_LOG_LEVEL' in os.environ: - self.debug = int(os.environ['MCW_LOG_LEVEL']) - else: - self.debug = 1 # Setup loggers loggers = [] self.logger = logging.getLogger('CMake Wrapper') @@ -234,8 +237,13 @@ def init_logging(self, dir=None): handlers = [] file_handler = None if dir: + if self._log_to_different_files: + log_file_name = 'meson-cmake-wrapper-{}.log'.format(self._datetime_str) + else: + log_file_name = 'meson-cmake-wrapper.log' + log_path = os.path.join(dir, log_file_name) - file_handler = logging.FileHandler(os.path.join(dir, 'meson-cmake-wrapper-{}.log'.format(self._datetime_str))) + file_handler = logging.FileHandler(log_path) file_handler.setLevel(logging.INFO) file_handler.setFormatter(logger_fmt_with_func) handlers.append(file_handler) From 824f4d710f3a553c0709e70e2c198423ffde0f1c Mon Sep 17 00:00:00 2001 From: "Ling Wang@g731gw" Date: Wed, 2 Sep 2020 00:20:11 +0200 Subject: [PATCH 4/5] reduce log --- mcw/meson.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mcw/meson.py b/mcw/meson.py index f80ec81..2e4c518 100644 --- a/mcw/meson.py +++ b/mcw/meson.py @@ -232,7 +232,7 @@ def get_target_filename(self, target): if isinstance(filename, list): if len(filename) > 1: - self.logger.warning('Target {} has more than 1 filename {}'.format(target['name'], filename)) + self.logger.debug('Target {} has more than 1 filename {}'.format(target['name'], filename)) return filename[0] else: return filename From 6357c3a315601ec581410be9897d29e70bd31de5 Mon Sep 17 00:00:00 2001 From: "Ling Wang@g731gw" Date: Wed, 2 Sep 2020 00:24:00 +0200 Subject: [PATCH 5/5] ignore .idea --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 65f53b7..2636d45 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ cmake-client.log dist *.egg-info __pycache__ +.idea \ No newline at end of file