From 60b64d0e2be5197b7bd7184b7a6f2abfb5e8b0fa Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Mon, 12 Sep 2022 19:00:14 +0200 Subject: [PATCH 01/17] P4: Add support for json output Simple json output support for p4 commands. A few handful functions: - reconcile current workspace in default cl - submit current default cl - get a file current revision in the workspace - print file content by revision --- nimp/utils/p4.py | 80 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 0d2ba4ee..3db1363e 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -23,6 +23,7 @@ ''' Perforce utilities ''' import argparse +import json import logging import os import os.path @@ -228,6 +229,23 @@ def reconcile(self, cl_number, *files): return ret + def reconcile_workspace(self, cl_number=None, workspace_path_to_reconcile=None, dry_run=False): + ''' Reconciles given workspace ''' + p4_reconcile_args = ['reconcile', '-f', '-e', '-a', '-d'] + if dry_run: + p4_reconcile_args.append('-n') + if cl_number: + p4_reconcile_args.extend(['-c', cl_number]) + if workspace_path_to_reconcile: + if not workspace_path_to_reconcile.endswith('...'): + workspace_path_to_reconcile = os.path.join(workspace_path_to_reconcile, '...') + workspace_path_to_reconcile = nimp.system.sanitize_path(workspace_path_to_reconcile) + p4_reconcile_args.append(workspace_path_to_reconcile) + + if self._run(*p4_reconcile_args) is None: + return False + return True + def get_changelist_description(self, cl_number): ''' Returns description of given changelist ''' desc, = next(self._parse_command_output(["describe", cl_number], r"\.\.\. desc (.*)")) @@ -248,6 +266,26 @@ def get_last_synced_changelist(self): return cl_number + def get_file_workspace_current_revision(self, file): + ''' Returns the file revision currently synced in the workspace ''' + revision, = next(self._parse_command_output(['have', file], r'\.\.\. haveRev (\d+)')) + return revision + + def print_file_by_revision(self, file, revision_number): + ''' Prints the file revision for a given revision number ''' + output = self._run('print', f'{file}#{revision_number}', use_json_format=True) + return self.parse_json_output_for_data(output) + + @staticmethod + def parse_json_output_for_data(output): + data = '' + output = [json_element for json_element in output.split('\n') if json_element] + for output_chunk in output: + output_chunk = json.loads(output_chunk) + if 'data' in output_chunk.keys(): + data += output_chunk['data'] + return data + def get_or_create_changelist(self, description): ''' Creates or returns changelist number if it's not already created ''' pending_changelists = self.get_pending_changelists() @@ -335,6 +373,31 @@ def submit(self, cl_number): return True + def submit_default_changelist(self, description=None, dry_run=False): + ''' Submits given changelist ''' + assert description is not None + logging.info("Submitting default changelist...") + command = self._get_p4_command('submit', '-f', 'revertunchanged') + # descriptions could be too long for perforce limitations + command_length = len(' '.join(command)) + perforce_cmd_max_characters_limit = 4096 + if description: + d_flag = 4 # accounts for ' -d ' string + description_max_characters_limit = perforce_cmd_max_characters_limit - command_length - d_flag + description = description[:description_max_characters_limit] + command.extend(['-d', description]) + if dry_run: + logging.info(command) + return True + else: + _, _, error = nimp.sys.process.call(command, capture_output=True) + + if error is not None and error != "": + logging.error("%s", error) + return False + + return True + def sync(self, *files, cl_number = None): ''' Udpate given file ''' command = ["sync"] @@ -357,8 +420,7 @@ def get_modified_files(self, *cl_numbers, root = '//...'): for cl_number in cl_numbers: for filename, action in self._parse_command_output(["fstat", "-e", cl_number , root], r"^\.\.\. depotFile(.*)$", - r"^\.\.\. headAction(.*)", - hide_output=True): + r"^\.\.\. headAction(.*)"): filename = os.path.normpath(filename) if filename is not None else '' yield filename, action @@ -370,8 +432,10 @@ def _escape_filename(name): .replace('#', '%23') \ .replace('*', '%2A') - def _get_p4_command(self, *args): + def _get_p4_command(self, *args, use_json_format=False): command = ['p4', '-z', 'tag'] + if use_json_format: + command.append('-Mj') if self._port is not None: command += ['-p', self._port] if self._user is not None: @@ -384,11 +448,11 @@ def _get_p4_command(self, *args): command += list(args) return command - def _run(self, *args, stdin=None, hide_output=False): - command = self._get_p4_command(*args) + def _run(self, *args, stdin=None, use_json_format=False): + command = self._get_p4_command(*args, use_json_format=use_json_format) for _ in range(5): - result, output, error = nimp.sys.process.call(command, stdin=stdin, encoding='cp437', capture_output=True, hide_output=hide_output) + result, output, error = nimp.sys.process.call(command, stdin=stdin, encoding='cp437', capture_output=True) if 'Operation took too long ' in error: continue @@ -405,8 +469,8 @@ def _run(self, *args, stdin=None, hide_output=False): return output - def _parse_command_output(self, command, *patterns, stdin = None, hide_output = False): - output = self._run(*command, stdin = stdin, hide_output = hide_output) + def _parse_command_output(self, command, *patterns, stdin = None): + output = self._run(*command, stdin = stdin) if output is not None: match_list = [] From 85b6bd0568df4796264112ef587929b3f5a605a7 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Fri, 23 Sep 2022 16:45:57 +0200 Subject: [PATCH 02/17] P4: get_file_workspace_current_revision handles file not existing It returns None if the file is not synced. --- nimp/utils/p4.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 3db1363e..6212c819 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -268,7 +268,10 @@ def get_last_synced_changelist(self): def get_file_workspace_current_revision(self, file): ''' Returns the file revision currently synced in the workspace ''' - revision, = next(self._parse_command_output(['have', file], r'\.\.\. haveRev (\d+)')) + try: + revision, = next(self._parse_command_output(['have', file], r'\.\.\. haveRev (\d+)')) + except StopIteration as e: + revision = None return revision def print_file_by_revision(self, file, revision_number): From 30864d11a65e5ad1837039a64dbf17d78833480f Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Fri, 23 Sep 2022 16:47:42 +0200 Subject: [PATCH 03/17] P4: tranform p4_print_** into a p4 print wrapper function --- nimp/utils/p4.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 6212c819..7a8e3b97 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -274,9 +274,9 @@ def get_file_workspace_current_revision(self, file): revision = None return revision - def print_file_by_revision(self, file, revision_number): - ''' Prints the file revision for a given revision number ''' - output = self._run('print', f'{file}#{revision_number}', use_json_format=True) + def print(self, p4_print_command_args): + ''' wrapper for p4 print command ''' + output = self._run('print', f'{p4_print_command_args}', use_json_format=True) return self.parse_json_output_for_data(output) @staticmethod From 218b8293ee9d7e31a213d9931efe0f9c35b5fe34 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Fri, 23 Sep 2022 16:48:18 +0200 Subject: [PATCH 04/17] P4: fix use splitlines instead of split('\n') --- nimp/utils/p4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 7a8e3b97..c6c99bed 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -282,7 +282,7 @@ def print(self, p4_print_command_args): @staticmethod def parse_json_output_for_data(output): data = '' - output = [json_element for json_element in output.split('\n') if json_element] + output = [json_element for json_element in output.splitlines() if json_element] for output_chunk in output: output_chunk = json.loads(output_chunk) if 'data' in output_chunk.keys(): From 90e34dfb08d470cf84e63f9945726afb4a79de7e Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Fri, 23 Sep 2022 16:49:57 +0200 Subject: [PATCH 05/17] P4: rework submit_default_changes function revertunchanged is optionnal Remove useless assert Replace 4096 limit by 8091 cmd limit --- nimp/utils/p4.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index c6c99bed..545476bc 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -376,14 +376,15 @@ def submit(self, cl_number): return True - def submit_default_changelist(self, description=None, dry_run=False): + def submit_default_changelist(self, description=None, revert_unchanged=False, dry_run=False): ''' Submits given changelist ''' - assert description is not None logging.info("Submitting default changelist...") - command = self._get_p4_command('submit', '-f', 'revertunchanged') + command = self._get_p4_command('submit', '-f') + if revert_unchanged: + command.append('revertunchanged') # descriptions could be too long for perforce limitations command_length = len(' '.join(command)) - perforce_cmd_max_characters_limit = 4096 + perforce_cmd_max_characters_limit = 8191 if description: d_flag = 4 # accounts for ' -d ' string description_max_characters_limit = perforce_cmd_max_characters_limit - command_length - d_flag From 3dc4a95cb646f68293159733d2d82daa2c501ef3 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Fri, 23 Sep 2022 16:51:07 +0200 Subject: [PATCH 06/17] P4: enable optionnal hide_output flag back to p4 commands --- nimp/utils/p4.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 545476bc..1c3aea49 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -452,11 +452,12 @@ def _get_p4_command(self, *args, use_json_format=False): command += list(args) return command - def _run(self, *args, stdin=None, use_json_format=False): + def _run(self, *args, stdin=None, hide_output=False, use_json_format=False): command = self._get_p4_command(*args, use_json_format=use_json_format) for _ in range(5): - result, output, error = nimp.sys.process.call(command, stdin=stdin, encoding='cp437', capture_output=True) + result, output, error = nimp.sys.process.call( + command, stdin=stdin, encoding='cp437', capture_output=True, hide_output=hide_output) if 'Operation took too long ' in error: continue @@ -473,8 +474,8 @@ def _run(self, *args, stdin=None, use_json_format=False): return output - def _parse_command_output(self, command, *patterns, stdin = None): - output = self._run(*command, stdin = stdin) + def _parse_command_output(self, command, *patterns, stdin = None, hide_output = False): + output = self._run(*command, stdin = stdin, hide_output = hide_output) if output is not None: match_list = [] From 1566e409a36b4e788603776e6149d2d130674cc4 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Tue, 8 Nov 2022 11:54:52 +0100 Subject: [PATCH 07/17] P4: fix recursive search safety net when reconciling with a path --- nimp/utils/p4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 1c3aea49..3b4e44dd 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -237,7 +237,7 @@ def reconcile_workspace(self, cl_number=None, workspace_path_to_reconcile=None, if cl_number: p4_reconcile_args.extend(['-c', cl_number]) if workspace_path_to_reconcile: - if not workspace_path_to_reconcile.endswith('...'): + if os.path.isdir(workspace_path_to_reconcile) or workspace_path_to_reconcile.endswith(('/', '\\')): workspace_path_to_reconcile = os.path.join(workspace_path_to_reconcile, '...') workspace_path_to_reconcile = nimp.system.sanitize_path(workspace_path_to_reconcile) p4_reconcile_args.append(workspace_path_to_reconcile) From f3b029a1ac05cf7c0b81d56afd0182bc1a1f3494 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Tue, 22 Nov 2022 18:04:52 +0100 Subject: [PATCH 08/17] P4Utils: fix python styling --- nimp/utils/p4.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 3b4e44dd..fb3139df 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -242,9 +242,7 @@ def reconcile_workspace(self, cl_number=None, workspace_path_to_reconcile=None, workspace_path_to_reconcile = nimp.system.sanitize_path(workspace_path_to_reconcile) p4_reconcile_args.append(workspace_path_to_reconcile) - if self._run(*p4_reconcile_args) is None: - return False - return True + return self._run(*p4_reconcile_args) is None def get_changelist_description(self, cl_number): ''' Returns description of given changelist ''' @@ -268,15 +266,11 @@ def get_last_synced_changelist(self): def get_file_workspace_current_revision(self, file): ''' Returns the file revision currently synced in the workspace ''' - try: - revision, = next(self._parse_command_output(['have', file], r'\.\.\. haveRev (\d+)')) - except StopIteration as e: - revision = None - return revision + return next(self._parse_command_output(['have', file], r'\.\.\. haveRev (\d+)'), default=None) def print(self, p4_print_command_args): ''' wrapper for p4 print command ''' - output = self._run('print', f'{p4_print_command_args}', use_json_format=True) + output = self._run('print', p4_print_command_args, use_json_format=True) return self.parse_json_output_for_data(output) @staticmethod @@ -285,7 +279,7 @@ def parse_json_output_for_data(output): output = [json_element for json_element in output.splitlines() if json_element] for output_chunk in output: output_chunk = json.loads(output_chunk) - if 'data' in output_chunk.keys(): + if 'data' in output_chunk: data += output_chunk['data'] return data From e6e441e1f59fb7942247ec4f688986561b6fd0a7 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Tue, 22 Nov 2022 18:16:57 +0100 Subject: [PATCH 09/17] P4: avoid possible parsing errors when dispalying p4 command error --- nimp/utils/p4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index fb3139df..04ef72ec 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -391,7 +391,7 @@ def submit_default_changelist(self, description=None, revert_unchanged=False, dr _, _, error = nimp.sys.process.call(command, capture_output=True) if error is not None and error != "": - logging.error("%s", error) + logging.error("%s", error.strip()) return False return True From d9160736d320fd0945ee93d75ad586b153aed684 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Tue, 22 Nov 2022 18:19:22 +0100 Subject: [PATCH 10/17] P4: fix get_modified_files displaying command output --- nimp/utils/p4.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 04ef72ec..ed000ba1 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -418,7 +418,8 @@ def get_modified_files(self, *cl_numbers, root = '//...'): for cl_number in cl_numbers: for filename, action in self._parse_command_output(["fstat", "-e", cl_number , root], r"^\.\.\. depotFile(.*)$", - r"^\.\.\. headAction(.*)"): + r"^\.\.\. headAction(.*)", + hide_output=True): filename = os.path.normpath(filename) if filename is not None else '' yield filename, action From 25856d9f219a976134be8d6eb132d93b8d9df513 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Tue, 22 Nov 2022 18:24:10 +0100 Subject: [PATCH 11/17] P4: inline code to extract json data for printing in print function --- nimp/utils/p4.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index ed000ba1..5046cf8f 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -270,12 +270,8 @@ def get_file_workspace_current_revision(self, file): def print(self, p4_print_command_args): ''' wrapper for p4 print command ''' - output = self._run('print', p4_print_command_args, use_json_format=True) - return self.parse_json_output_for_data(output) - - @staticmethod - def parse_json_output_for_data(output): data = '' + output = self._run('print', p4_print_command_args, use_json_format=True) output = [json_element for json_element in output.splitlines() if json_element] for output_chunk in output: output_chunk = json.loads(output_chunk) From 4919739bd16fbcaa7b136c7ce7d55afddabb38aa Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Tue, 22 Nov 2022 19:06:24 +0100 Subject: [PATCH 12/17] P4: reconcile now takes a lists of paths --- nimp/utils/p4.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 5046cf8f..bf6835d1 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -229,18 +229,18 @@ def reconcile(self, cl_number, *files): return ret - def reconcile_workspace(self, cl_number=None, workspace_path_to_reconcile=None, dry_run=False): + def reconcile_workspace(self, *paths_to_reconcile, cl_number=None, dry_run=False): ''' Reconciles given workspace ''' p4_reconcile_args = ['reconcile', '-f', '-e', '-a', '-d'] if dry_run: p4_reconcile_args.append('-n') if cl_number: p4_reconcile_args.extend(['-c', cl_number]) - if workspace_path_to_reconcile: - if os.path.isdir(workspace_path_to_reconcile) or workspace_path_to_reconcile.endswith(('/', '\\')): - workspace_path_to_reconcile = os.path.join(workspace_path_to_reconcile, '...') - workspace_path_to_reconcile = nimp.system.sanitize_path(workspace_path_to_reconcile) - p4_reconcile_args.append(workspace_path_to_reconcile) + for path_to_reconcile in paths_to_reconcile: + if not path_to_reconcile.endswith('...'): + if os.path.isdir(path_to_reconcile) or path_to_reconcile.endswith(('/', '\\')): + path_to_reconcile = os.path.join(path_to_reconcile, '...') + p4_reconcile_args.append(path_to_reconcile) return self._run(*p4_reconcile_args) is None From 59298fb10dd9b7c400795a96ca7ec878145b1c03 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Thu, 24 Nov 2022 17:20:46 +0100 Subject: [PATCH 13/17] P4: use _run_with_arg_file for reconcile and submit funcs --- nimp/utils/p4.py | 51 ++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index bf6835d1..cc202360 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -28,6 +28,7 @@ import os import os.path import re +import tempfile import nimp.sys.process import nimp.system @@ -231,7 +232,8 @@ def reconcile(self, cl_number, *files): def reconcile_workspace(self, *paths_to_reconcile, cl_number=None, dry_run=False): ''' Reconciles given workspace ''' - p4_reconcile_args = ['reconcile', '-f', '-e', '-a', '-d'] + + p4_reconcile_args = ['-f', '-e', '-a', '-d'] if dry_run: p4_reconcile_args.append('-n') if cl_number: @@ -242,7 +244,7 @@ def reconcile_workspace(self, *paths_to_reconcile, cl_number=None, dry_run=False path_to_reconcile = os.path.join(path_to_reconcile, '...') p4_reconcile_args.append(path_to_reconcile) - return self._run(*p4_reconcile_args) is None + return self._run_using_arg_file('reconcile', *p4_reconcile_args) is not None def get_changelist_description(self, cl_number): ''' Returns description of given changelist ''' @@ -271,7 +273,7 @@ def get_file_workspace_current_revision(self, file): def print(self, p4_print_command_args): ''' wrapper for p4 print command ''' data = '' - output = self._run('print', p4_print_command_args, use_json_format=True) + output = self._run('print', p4_print_command_args, use_json_format=True, hide_output=True) output = [json_element for json_element in output.splitlines() if json_element] for output_chunk in output: output_chunk = json.loads(output_chunk) @@ -369,28 +371,22 @@ def submit(self, cl_number): def submit_default_changelist(self, description=None, revert_unchanged=False, dry_run=False): ''' Submits given changelist ''' logging.info("Submitting default changelist...") - command = self._get_p4_command('submit', '-f') + submit_args = [] if revert_unchanged: - command.append('revertunchanged') - # descriptions could be too long for perforce limitations - command_length = len(' '.join(command)) - perforce_cmd_max_characters_limit = 8191 + submit_args.extend(['-f', 'revertunchanged']) if description: - d_flag = 4 # accounts for ' -d ' string - description_max_characters_limit = perforce_cmd_max_characters_limit - command_length - d_flag - description = description[:description_max_characters_limit] - command.extend(['-d', description]) + # description has to fit on one line, even when using -x arg_file + # there is a perforce limitation to how long the desc can be however, arg_file or not, I tested this. + # p4 command failed: Identifiers too long. Must not be longer than 1024 bytes of UTF-8 + # ...happens when using 130000-ish bytes utf8 words + # anything under that seems to be fine, whatever happened to this 1024 bytes limit... + # couldn't find more info on this subject in the perforce documentation + description_limit = 120000 + submit_args.extend(['-d', description[:description_limit]]) if dry_run: - logging.info(command) - return True - else: - _, _, error = nimp.sys.process.call(command, capture_output=True) - - if error is not None and error != "": - logging.error("%s", error.strip()) - return False - + logging.info(f'{self._get_p4_command("submit")} {submit_args}') return True + return self._run_using_arg_file('submit', *submit_args) is not None def sync(self, *files, cl_number = None): ''' Udpate given file ''' @@ -465,6 +461,19 @@ def _run(self, *args, stdin=None, hide_output=False, use_json_format=False): return output + def _run_using_arg_file(self, command, *command_args): + ''' runs p4 -x arg_file_containing_command_args command ''' + # perforce seems unable to handle a tempfile.TemporaryFile(): + # p4 command failed: Perforce client error: open for read: + # : The process cannot access the file because it is being used by another process + # Use a temp dir instead so the temp file can be used by perforce... + # The dir and file is wiped anyway when exiting the context manager + with tempfile.TemporaryDirectory(prefix="p4_arg_file_") as tmp_dir: + arg_file_path = os.path.normpath(os.path.join(tmp_dir, 'p4_arg_file')) + with open(arg_file_path, 'w') as fp: + fp.write('\n'.join(command_args)) + return self._run(*['-x', arg_file_path, command]) + def _parse_command_output(self, command, *patterns, stdin = None, hide_output = False): output = self._run(*command, stdin = stdin, hide_output = hide_output) From 9aed010f420ca3d0f1348bca73ddb6ce383b3e49 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Fri, 25 Nov 2022 15:00:59 +0100 Subject: [PATCH 14/17] P4: print function takes one file at a time The p4 print function can output the contents of path/to/pattern* We want this to be a handy function to print a given file data. --- nimp/utils/p4.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index cc202360..6e0f5a4c 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -270,15 +270,15 @@ def get_file_workspace_current_revision(self, file): ''' Returns the file revision currently synced in the workspace ''' return next(self._parse_command_output(['have', file], r'\.\.\. haveRev (\d+)'), default=None) - def print(self, p4_print_command_args): + def print_file_data(self, file, revision=None): ''' wrapper for p4 print command ''' - data = '' - output = self._run('print', p4_print_command_args, use_json_format=True, hide_output=True) - output = [json_element for json_element in output.splitlines() if json_element] + revision = f"#{revision}" if revision is not None else '' + data = None + output = self._run('print', file+revision, use_json_format=True, hide_output=False) + output = [json.loads(json_element) for json_element in output.splitlines() if json_element] for output_chunk in output: - output_chunk = json.loads(output_chunk) if 'data' in output_chunk: - data += output_chunk['data'] + data = output_chunk['data'] return data def get_or_create_changelist(self, description): From 7098e243f9952c8487951526bfc10b416337ba31 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Mon, 28 Nov 2022 16:30:37 +0100 Subject: [PATCH 15/17] P4: modifiy submit function so it updates cl spec before submitting It lets us submit default cl from sractch as well as submitting knwon changelists (while still being able to update cl desc) --- nimp/utils/p4.py | 87 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 5 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 6e0f5a4c..91ad0ba9 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -353,7 +353,7 @@ def revert_unchanged(self, cl_number): output = self._run('revert', '-a', '-c', cl_number, '//...') return output is not None - def submit(self, cl_number): + def _submit(self, cl_number): ''' Submits given changelist ''' logging.info("Submiting changelist %s...", cl_number) command = self._get_p4_command('submit', '-f', 'revertunchanged', '-c', cl_number) @@ -388,6 +388,50 @@ def submit_default_changelist(self, description=None, revert_unchanged=False, dr return True return self._run_using_arg_file('submit', *submit_args) is not None + + def _get_cl_spec(self, cl_number=None): + command = ['change', '-o'] + if cl_number is not None: + command.append(str(cl_number)) + # We need the cl_spec with no -z tag, for later use as + return self._run(*command, hide_output=False, use_ztag=False) + + def _set_cl_spec(self, cl_spec): + output = self._run(*['change', '-i'], stdin=cl_spec) + pattern = r"Change (\d+) (created|updated)" + matches = re.findall(pattern, output, re.MULTILINE) + assert len(matches) == 1 + return matches[0] + + def _update_cl_spec_field(self, cl_spec, spec_field, field_content): + assert spec_field in ALL_SPEC_FIELDS + possible_following_fields = r":\r\n|".join(ALL_SPEC_FIELDS) + # using dotall flag rather than multiline, that stops at first encountered \r\n + pattern = re.compile(rf'{spec_field}:\r\n\t(.*)\r\n\r\n({possible_following_fields}:\r\n)', re.DOTALL) + matches = re.findall(pattern, cl_spec) + if not matches: + # It's possible that there is no following field + pattern = re.compile(rf'{spec_field}:\r\n\t(.*)(\r\n\r\n)', re.DOTALL) + matches = re.findall(pattern, cl_spec) + assert len(matches) == 1 + initial_desc, following_field = matches[0] + return re.sub(pattern, f'{spec_field}:\r\n\t{field_content}\r\n\r\n{following_field}', cl_spec) + + def submit(self, cl_number=None, description=None): + ''' submit from default if no cl_number is provided + or else submit given cl_number + description can be set or updated ''' + assert any(a is not None for a in [cl_number, description]) + + if description is not None: + cl_spec = self._get_cl_spec(cl_number=cl_number) + cl_spec = self._update_cl_spec_field(cl_spec, SpecField.description, + "\n\t".join(line for line in description.splitlines())) + cl_number, status = self._set_cl_spec(cl_spec) + + return self._submit(cl_number) + + def sync(self, *files, cl_number = None): ''' Udpate given file ''' command = ["sync"] @@ -423,8 +467,10 @@ def _escape_filename(name): .replace('#', '%23') \ .replace('*', '%2A') - def _get_p4_command(self, *args, use_json_format=False): - command = ['p4', '-z', 'tag'] + def _get_p4_command(self, *args, use_ztag=True, use_json_format=False): + command = ['p4'] + if use_ztag: + command += ['-z', 'tag'] if use_json_format: command.append('-Mj') if self._port is not None: @@ -439,8 +485,8 @@ def _get_p4_command(self, *args, use_json_format=False): command += list(args) return command - def _run(self, *args, stdin=None, hide_output=False, use_json_format=False): - command = self._get_p4_command(*args, use_json_format=use_json_format) + def _run(self, *args, stdin=None, hide_output=False, use_json_format=False, use_ztag=True): + command = self._get_p4_command(*args, use_json_format=use_json_format, use_ztag=use_ztag) for _ in range(5): result, output, error = nimp.sys.process.call( @@ -491,3 +537,34 @@ def _parse_command_output(self, command, *patterns, stdin = None, hide_output = for elem in zip(*match_list): yield elem + + +class SpecField(): + change = 'Change' + date = 'Date' + client = 'Client' + user = 'User' + status = 'Status' + type = 'Type' + description = 'Description' + imported_by = 'ImportedBy' + identity = 'Identity' + jobs = 'jobs' + stream = 'Stream' + files= 'Files' + + +ALL_SPEC_FIELDS = [ + SpecField.change, + SpecField.date, + SpecField.client, + SpecField.user, + SpecField.status, + SpecField.type, + SpecField.description, + SpecField.imported_by, + SpecField.identity, + SpecField.jobs, + SpecField.stream, + SpecField.files +] From b2ba53b48060b1cbc2d5530d897624829a97cbb2 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Mon, 28 Nov 2022 16:45:38 +0100 Subject: [PATCH 16/17] P4: use os.linesep when updating cl spec field For better compatibility across systems --- nimp/utils/p4.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 91ad0ba9..07d1b1e2 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -405,17 +405,18 @@ def _set_cl_spec(self, cl_spec): def _update_cl_spec_field(self, cl_spec, spec_field, field_content): assert spec_field in ALL_SPEC_FIELDS - possible_following_fields = r":\r\n|".join(ALL_SPEC_FIELDS) - # using dotall flag rather than multiline, that stops at first encountered \r\n - pattern = re.compile(rf'{spec_field}:\r\n\t(.*)\r\n\r\n({possible_following_fields}:\r\n)', re.DOTALL) + possible_following_fields = fr":{os.linesep}|".join(ALL_SPEC_FIELDS) + fr':{os.linesep}' + pattern = re.compile(rf'{spec_field}:{os.linesep}\t(.*){os.linesep}{os.linesep}({possible_following_fields})', + # using dotall flag rather than multiline, that stops at first encountered \r\n + re.DOTALL) matches = re.findall(pattern, cl_spec) if not matches: # It's possible that there is no following field - pattern = re.compile(rf'{spec_field}:\r\n\t(.*)(\r\n\r\n)', re.DOTALL) + pattern = re.compile(rf'{spec_field}:{os.linesep}\t(.*)({os.linesep}{os.linesep})', re.DOTALL) matches = re.findall(pattern, cl_spec) assert len(matches) == 1 initial_desc, following_field = matches[0] - return re.sub(pattern, f'{spec_field}:\r\n\t{field_content}\r\n\r\n{following_field}', cl_spec) + return re.sub(pattern, f'{spec_field}:{os.linesep}\t{field_content}{os.linesep}{os.linesep}{following_field}', cl_spec) def submit(self, cl_number=None, description=None): ''' submit from default if no cl_number is provided From 95a7c7383828b2de14c9db45117066b68d05c1a7 Mon Sep 17 00:00:00 2001 From: Laurent Cahour Date: Mon, 28 Nov 2022 20:32:48 +0100 Subject: [PATCH 17/17] P4: fix cl spec field updating process Any backslash would be interpreted as a regex. Use replacing method rather than re.sub to preserve initial desc. --- nimp/utils/p4.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nimp/utils/p4.py b/nimp/utils/p4.py index 07d1b1e2..b1b2d85c 100644 --- a/nimp/utils/p4.py +++ b/nimp/utils/p4.py @@ -416,7 +416,9 @@ def _update_cl_spec_field(self, cl_spec, spec_field, field_content): matches = re.findall(pattern, cl_spec) assert len(matches) == 1 initial_desc, following_field = matches[0] - return re.sub(pattern, f'{spec_field}:{os.linesep}\t{field_content}{os.linesep}{os.linesep}{following_field}', cl_spec) + return cl_spec.replace(initial_desc, field_content) + # Replace desc rather than re.sub to preserve utf8 chars like \x7A\x3A + # return re.sub(pattern, rf'{spec_field}:{os.linesep}\t{field_content}{os.linesep}{os.linesep}{following_field}', cl_spec) def submit(self, cl_number=None, description=None): ''' submit from default if no cl_number is provided @@ -426,8 +428,8 @@ def submit(self, cl_number=None, description=None): if description is not None: cl_spec = self._get_cl_spec(cl_number=cl_number) - cl_spec = self._update_cl_spec_field(cl_spec, SpecField.description, - "\n\t".join(line for line in description.splitlines())) + tabbed_description = "\n\t".join(line for line in description.splitlines()) + cl_spec = self._update_cl_spec_field(cl_spec, SpecField.description, tabbed_description) cl_number, status = self._set_cl_spec(cl_spec) return self._submit(cl_number)