From 28c1517590cc559640a550efb44cde6771ff8888 Mon Sep 17 00:00:00 2001 From: "Brett T. Warden" Date: Wed, 27 Mar 2019 16:08:14 -0700 Subject: [PATCH 1/4] Support using the auto-detected name Rework to absorb a lot of the work we were doing in makefiles so that we can avoid setting the package name manually in 'make autospecnew'. --- autospec/autospec.py | 7 +++++-- autospec/config.py | 11 +++++++---- autospec/git.py | 7 +++++-- autospec/pkg_scan.py | 5 +++-- autospec/tarball.py | 16 +++++++++++++++- 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/autospec/autospec.py b/autospec/autospec.py index 4b916e0c..8398725d 100644 --- a/autospec/autospec.py +++ b/autospec/autospec.py @@ -176,6 +176,9 @@ def main(): parser.add_argument("-t", "--target", dest="target", action="store", default=None, help="Target location to create or reuse") + parser.add_argument("-P", "--package-dir", dest="package_dir", action="store", + default=None, + help="Location under which to create/clone a package") parser.add_argument("-i", "--integrity", action="store_true", default=False, help="Search for package signature from source URL and " @@ -247,7 +250,7 @@ def package(args, url, name, archives, workingdir, infile_dict): # of static analysis on the content of the tarball. # filemanager = files.FileManager() - tarball.process(url, name, args.version, args.target, archives, filemanager) + tarball.process(url, name, args.version, args.target, args.package_dir, archives, filemanager) _dir = tarball.path if args.license_only: @@ -341,7 +344,7 @@ def package(args, url, name, archives, workingdir, infile_dict): examine_abi(build.download_path) if os.path.exists("/var/lib/rpm"): - pkg_scan.get_whatrequires(tarball.name) + pkg_scan.get_whatrequires(tarball.name, build.download_path) write_out(build.download_path + "/release", tarball.release + "\n") diff --git a/autospec/config.py b/autospec/config.py index f4a49204..4da5b0b7 100644 --- a/autospec/config.py +++ b/autospec/config.py @@ -64,7 +64,8 @@ license_fetch = None license_show = None -git_uri = None +git_pull_uri = None +git_push_uri = None os_packages = set() config_file = None old_version = None @@ -547,7 +548,8 @@ def parse_config_files(path, bump, filemanager, version): global parallel_build global license_fetch global license_show - global git_uri + global git_pull_uri + global git_push_uri global os_packages global urlban global config_file @@ -585,7 +587,8 @@ def parse_config_files(path, bump, filemanager, version): print("Missing autospec section..") sys.exit(1) - git_uri = config['autospec'].get('git', None) + git_push_uri = config['autospec'].get('git', None) + git_pull_uri = config['autospec'].get('git_pull', git_push_uri) license_fetch = config['autospec'].get('license_fetch', None) license_show = config['autospec'].get('license_show', None) packages_file = config['autospec'].get('packages_file', None) @@ -610,7 +613,7 @@ def parse_config_files(path, bump, filemanager, version): # Read values from options.conf (and deprecated files) and rewrite as necessary read_config_opts(path) - if not git_uri: + if not git_pull_uri: print("Warning: Set [autospec][git] upstream template for remote git URI configuration") if not license_fetch: print("Warning: Set [autospec][license_fetch] uri for license fetch support") diff --git a/autospec/git.py b/autospec/git.py index 911102e6..6d6154c6 100644 --- a/autospec/git.py +++ b/autospec/git.py @@ -34,12 +34,15 @@ def commit_to_git(path): call("git init", stdout=subprocess.DEVNULL, cwd=path) # This config is used for setting the remote URI, so it is optional. - if config.git_uri: + if config.git_pull_uri: try: call("git config --get remote.origin.url", cwd=path) except subprocess.CalledProcessError: - upstream_uri = config.git_uri % {'NAME': tarball.name} + upstream_uri = config.git_pull_uri % {'NAME': tarball.name} call("git remote add origin %s" % upstream_uri, cwd=path) + push_uri = config.git_push_uri % {'NAME': tarball.name} + if push_uri != upstream_uri: + call("git remote set-url origin --push %s" % upstream_uri, cwd=path) for config_file in config.config_files: call("git add %s" % config_file, cwd=path, check=False) diff --git a/autospec/pkg_scan.py b/autospec/pkg_scan.py index d232199e..4faecd35 100644 --- a/autospec/pkg_scan.py +++ b/autospec/pkg_scan.py @@ -15,13 +15,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +import os import subprocess import config import util -def get_whatrequires(pkg): +def get_whatrequires(pkg, download_path): """ Write list of packages. @@ -47,5 +48,5 @@ def get_whatrequires(pkg): util.print_warning("dnf repoquery whatrequires for {} failed with: {}".format(pkg, err)) return - util.write_out('whatrequires', '# This file contains recursive sources that ' + util.write_out(os.path.join(download_path, 'whatrequires'), '# This file contains recursive sources that ' 'require this package\n' + out) diff --git a/autospec/tarball.py b/autospec/tarball.py index a4854694..952012e7 100644 --- a/autospec/tarball.py +++ b/autospec/tarball.py @@ -510,6 +510,14 @@ def prepare_and_extract(extract_cmd): call(extract_cmd) +def write_makefile(name, url, archives): + makefile_path = os.path.join(build.download_path, "Makefile") + if not os.path.exists(makefile_path): + write_out(makefile_path, + "PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n" + % (name, url, ' '.join(archives))) + + def process_archives(archives): """Download and process archives.""" for archive, destination in zip(archives[::2], archives[1::2]): @@ -544,7 +552,7 @@ def process_archives(archives): write_upstream(sha1, os.path.basename(archive), mode="a") -def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager): +def process(url_arg, name_arg, ver_arg, target, package_dir, archives_arg, filemanager): """Download and process the tarball at url_arg.""" global url global name @@ -565,6 +573,11 @@ def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager): # name is created by adding ".gcov" to the package name (if a gcov file # exists) set_gcov() + # build the target path, if necessary + if not target and package_dir: + target = os.path.join(package_dir, name) + # create the path, by cloning existing repo or manually, as necessary + call("make clone_%s" % name, check=False, stderr=subprocess.DEVNULL) # download the tarball to tar_path tar_path = download_tarball(target) # write the sha of the upstream tarfile to the "upstream" file @@ -576,6 +589,7 @@ def process(url_arg, name_arg, ver_arg, target, archives_arg, filemanager): # Now that the metadata has been collected print the header print_header() # write out the Makefile with the name, url, and archives we found + write_makefile(name, url, archives) # prepare directory and extract tarball prepare_and_extract(extract_cmd) # locate or download archives and move them into the right spot From 1f769ae846507dfaf6fd5f7af8a0a224ffa15063 Mon Sep 17 00:00:00 2001 From: "Brett T. Warden" Date: Wed, 27 Mar 2019 16:46:43 -0700 Subject: [PATCH 2/4] Fix PEP-8 complaints --- autospec/tarball.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/autospec/tarball.py b/autospec/tarball.py index 952012e7..fbda2c77 100644 --- a/autospec/tarball.py +++ b/autospec/tarball.py @@ -511,11 +511,12 @@ def prepare_and_extract(extract_cmd): def write_makefile(name, url, archives): + """Write the package makefile.""" makefile_path = os.path.join(build.download_path, "Makefile") if not os.path.exists(makefile_path): write_out(makefile_path, - "PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n" - % (name, url, ' '.join(archives))) + "PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n" + % (name, url, ' '.join(archives))) def process_archives(archives): From 2472494afe5b011b2b18a3a469de01b9f6aedb74 Mon Sep 17 00:00:00 2001 From: "Brett T. Warden" Date: Tue, 2 Apr 2019 14:31:39 -0700 Subject: [PATCH 3/4] Fix wrong variable names from review --- autospec/config.py | 2 +- autospec/git.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/autospec/config.py b/autospec/config.py index 4da5b0b7..348c798d 100644 --- a/autospec/config.py +++ b/autospec/config.py @@ -613,7 +613,7 @@ def parse_config_files(path, bump, filemanager, version): # Read values from options.conf (and deprecated files) and rewrite as necessary read_config_opts(path) - if not git_pull_uri: + if not git_push_uri: print("Warning: Set [autospec][git] upstream template for remote git URI configuration") if not license_fetch: print("Warning: Set [autospec][license_fetch] uri for license fetch support") diff --git a/autospec/git.py b/autospec/git.py index 6d6154c6..c5bda336 100644 --- a/autospec/git.py +++ b/autospec/git.py @@ -42,7 +42,7 @@ def commit_to_git(path): call("git remote add origin %s" % upstream_uri, cwd=path) push_uri = config.git_push_uri % {'NAME': tarball.name} if push_uri != upstream_uri: - call("git remote set-url origin --push %s" % upstream_uri, cwd=path) + call("git remote set-url origin --push %s" % push_uri, cwd=path) for config_file in config.config_files: call("git add %s" % config_file, cwd=path, check=False) From 35665e22cd1212ccabc3d41051bee5247a0b426b Mon Sep 17 00:00:00 2001 From: "Brett T. Warden" Date: Fri, 19 Apr 2019 10:52:05 -0700 Subject: [PATCH 4/4] Catch github url special case When using github urls, the code tries to test the detected name against the previously provided name. If we didn't provide a name, though, this would throw an exception. --- autospec/tarball.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autospec/tarball.py b/autospec/tarball.py index fbda2c77..2a0c98e5 100644 --- a/autospec/tarball.py +++ b/autospec/tarball.py @@ -380,7 +380,7 @@ def name_and_version(name_arg, version_arg, filemanager): m = re.search(pattern, url) if m: repo = m.group(2).strip() - if repo not in name: + if not name or repo not in name: # Only take the repo name as the package name if it's more descriptive name = repo elif name != repo: