fix: use .conda

.ci_scripts/condarc

@@ -2,3 +2,6 @@ channels:
   - conda-forge
 show_channel_urls: True
 channel_priority: strict
+conda-build:
+  pkg_format: '2'
+  zstd_compression_level: 19

.gitignore

@@ -127,3 +127,4 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+*.ipynb

azure-pipelines.yml

@@ -3,26 +3,63 @@ jobs:
   pool:
     vmImage: 'ubuntu-latest'
   timeoutInMinutes: 360
+  condition: or(and(eq(variables['System.PullRequest.IsFork'], 'True'), eq(variables['Build.Reason'], 'PullRequest')), and(ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'True'), ne(variables['Build.Reason'], 'PullRequest')))
   strategy:
     matrix:
-      cos7_x86_64:
+      cos7_x86_64_1:
         DISTARCH: 'centos7-x86_64'
-      cos7_aarch64:
+        PTOP: '1:4'
+      cos7_x86_64_2:
+        DISTARCH: 'centos7-x86_64'
+        PTOP: '2:4'
+      cos7_x86_64_3:
+        DISTARCH: 'centos7-x86_64'
+        PTOP: '3:4'
+      cos7_x86_64_4:
+        DISTARCH: 'centos7-x86_64'
+        PTOP: '4:4'
+      cos7_aarch64_1:
+        DISTARCH: 'centos7-aarch64'
+        PTOP: '1:4'
+      cos7_aarch64_2:
+        DISTARCH: 'centos7-aarch64'
+        PTOP: '2:4'
+      cos7_aarch64_3:
         DISTARCH: 'centos7-aarch64'
-      cos7_ppc64le:
+        PTOP: '3:4'
+      cos7_aarch64_4:
+        DISTARCH: 'centos7-aarch64'
+        PTOP: '4:4'
+      cos7_ppc64le_1:
+        DISTARCH: 'centos7-ppc64le'
+        PTOP: '1:4'
+      cos7_ppc64le_2:
+        DISTARCH: 'centos7-ppc64le'
+        PTOP: '2:4'
+      cos7_ppc64le_3:
+        DISTARCH: 'centos7-ppc64le'
+        PTOP: '3:4'
+      cos7_ppc64le_4:
         DISTARCH: 'centos7-ppc64le'
+        PTOP: '4:4'
       alma8_x86_64:
         DISTARCH: 'alma8-x86_64'
+        PTOP: '1:1'
       alma8_aarch64:
         DISTARCH: 'alma8-aarch64'
+        PTOP: '1:1'
       alma8_ppc64le:
         DISTARCH: 'alma8-ppc64le'
+        PTOP: '1:1'
       alma9_x86_64:
         DISTARCH: 'alma9-x86_64'
+        PTOP: '1:1'
       alma9_aarch64:
         DISTARCH: 'alma9-aarch64'
+        PTOP: '1:1'
       alma9_ppc64le:
         DISTARCH: 'alma9-ppc64le'
+        PTOP: '1:1'
     maxParallel: 0
 
   steps:
@@ -65,6 +102,7 @@ jobs:
         sed -h
         echo `uname -s`
     displayName: removing system compilers
+    condition: 'false'
 
   - bash: .ci_scripts/install_miniconda.sh
     displayName: install miniconda
@@ -78,8 +116,8 @@ jobs:
 
         mkdir -p build_logs
 
-        echo "cmd args: ${DISTARCH}"
-        python build_cdt_recipes.py ${DISTARCH}
+        echo "cmd args: ${DISTARCH} ${PTOP}"
+        python build_cdt_recipes.py ${DISTARCH} --part-to-process=${PTOP}
     displayName: build and upload CDTs
     env:
       ANACONDA_TOKEN: $(anaconda.token)

build_cdt_recipes.py

@@ -1,3 +1,4 @@
+import hashlib
 import os
 import subprocess
 import glob
@@ -171,7 +172,7 @@ def _build_cdt(cdt_meta_node, no_temp=False):
         recipe = os.path.basename(cdt_meta_node["recipe_path"])
         pkg = folder_to_package(recipe)
         cdt_file = glob.glob(
-            os.path.expandvars("${HOME}/miniforge3/conda-bld/*/%s-*.tar.bz2" % pkg)
+            os.path.expandvars("${HOME}/miniforge3/conda-bld/*/%s-*.conda" % pkg)
         )
         assert len(cdt_file) == 1
         for _ in range(5):
@@ -191,25 +192,78 @@ def _build_cdt(cdt_meta_node, no_temp=False):
     return c, c_up
 
 
-def _build_all_cdts(cdt_path, custom_cdt_path, dist_arch_slug):
+def _build_cdt_groups(cdt_meta):
+    all_cdts = set(cdt_meta.keys())
+    groups = {}
+    cdt_to_group = {}
+    for name, info in cdt_meta.items():
+        cdt_reqs = set(info["all_requirements"]) & all_cdts
+        cdt_reqs.add(name)
+        curr_groups = set()
+        for group_name, group_members in groups.items():
+            if cdt_reqs & group_members:
+                curr_groups.add(group_name)
+
+        if not curr_groups:
+            groups[name] = cdt_reqs
+            for cdt in cdt_reqs:
+                cdt_to_group[cdt] = name
+        else:
+            new_group = set()
+            for group_name in curr_groups:
+                new_group |= groups.pop(group_name)
+            new_group |= cdt_reqs
+            groups[name] = new_group
+            for cdt in new_group:
+                cdt_to_group[cdt] = name
+
+    # this bit of code double checks that all reqs are in the
+    # same group
+    for name, info in cdt_meta.items():
+        cdt_reqs = set(info["all_requirements"]) & all_cdts
+        cdt_reqs.add(name)
+        curr_groups = set(
+            cdt_to_group[nm] for nm in cdt_reqs
+        )
+        assert len(curr_groups) == 1
+    assert len(set(cdt_to_group.values())) <= len(cdt_to_group)
+
+    return cdt_to_group
+
+
+def _cdt_name_to_part(name, num_parts):
+    sha = hashlib.sha1(name.encode("utf-8"))
+    part_zero = int(sha.hexdigest(), 16) % num_parts
+    return part_zero + 1
+
+
+def _build_all_cdts(cdt_path, custom_cdt_path, dist_arch_slug, part=1, num_parts=1):
     recipes = (
         glob.glob(cdt_path + "/*")
         + glob.glob(custom_cdt_path + "/*")
     )
     cdt_meta = _build_cdt_meta(recipes, dist_arch_slug)
+    cdt_to_group = _build_cdt_groups(cdt_meta)
+
+    # skip CDTs that we are not building in this job
+    for node in cdt_meta:
+        group_name = cdt_to_group[node]
+        if _cdt_name_to_part(group_name, num_parts) != part:
+            cdt_meta[node]["skip"] = True
 
     skipped = set(k for k, v in cdt_meta.items() if v["skip"])
     for node in sorted(skipped):
         print(
-            f"WARNING: skipping CDT {node} since it has already been built!",
+            f"WARNING: skipping CDT {node} since it has "
+            "already been built or is not part of this job!",
             flush=True,
         )
 
     print("\ncdts to build:", flush=True)
     for cdt in sorted(set(cdt_meta.keys()) - skipped):
         print(f"    {cdt}", flush=True)
 
-    num_workers = 4
+    num_workers = 2
     build_logs = ""
 
     with ThreadPoolExecutor(max_workers=num_workers) as exec:
@@ -287,15 +341,28 @@ def _build_all_cdts(cdt_path, custom_cdt_path, dist_arch_slug):
 
 @click.command()
 @click.argument("dist_arch_slug", required=True)
+@click.option(
+    "--part-to-process",
+    default="1:1",
+    type=str,
+    help=(
+        "the part of the list of CDTs to process, denoted as "
+        "'<part starting at 1>:<total_parts>' (e.g. '1:4', '2:4'"
+        ", etc. for four parts)"
+    ),
+)
 def _main(
     dist_arch_slug,
+    part_to_process,
 ):
     """
     Build all CDT recipes for a given DIST_ARCH_SLUG, i.e. `f"{distro}-{arch}"`,
     where we use the full distro name, and not the shortform
     """
-
-    _build_all_cdts(CDT_PATH, CUSTOM_CDT_PATH, dist_arch_slug)
+    part, num_parts = part_to_process.split(":")
+    part = int(part)
+    num_parts = int(num_parts)
+    _build_all_cdts(CDT_PATH, CUSTOM_CDT_PATH, dist_arch_slug, part=part, num_parts=num_parts)
 
 
 if __name__ == "__main__":

conda_build_config.yaml

@@ -1,5 +1,5 @@
 cdt_build_number:
-  - "107"
+  - "108"
 
 # this variant is here to help distinguish package hashes
 distro: