BUG make sure to test CDTs properly (#23)

* BUG make sure to test CDTs properly

* BUG skip non-links

* BUg turn off binary relocation

* ENH try again

* ENH try fewer threads and publish build artifacts

* BUG add slash

* ENH maybe it is all i/o?

* ENH try this; remove circular dep

* BUG make sure build artifact is unique; add one more process

azure-pipelines.yml

@@ -50,6 +50,8 @@ jobs:
         conda activate base
         cat .ci_scripts/condarc > $HOME/.condarc
 
+        mkdir -p build_logs
+
         echo "cmd args: ${DISTARCH} ${LEGACYCDT}"
         python build_cdt_recipes.py ${DISTARCH} ${LEGACYCDT}
     displayName: build CDTs
@@ -83,3 +85,6 @@ jobs:
     env:
       ANACONDA_TOKEN: $(anaconda.token)
       IS_FORK: $(System.PullRequest.IsFork)
+
+  - publish: build_logs/
+    artifact: build-logs-$(DISTARCH)$(LEGACYCDT)

build_cdt_recipes.py

@@ -235,7 +235,11 @@ def _build_all_cdts(cdt_path, custom_cdt_path, dist_arch_slug):
                         sys.stderr.flush()
                         raise RuntimeError("Could not build CDT %s!" % node)
 
-            print(build_logs, flush=True)
+    log_dir = "build_logs"
+    log_pth = os.path.join(log_dir, "build_logs_%s.txt" % dist_arch_slug)
+    os.makedirs(log_dir, exist_ok=True)
+    with open(log_pth, "w") as fp:
+        fp.write(build_logs)
 
 
 @click.command()

cdt_slugs.yaml

@@ -85,6 +85,14 @@ gtk2-devel:
 java-1.7.0-openjdk:
   custom: false
   license_file: licenses/openjdk-license
+  build_append:
+    cos6-x86_64: |
+      jvm_slug=$(compgen -G "${SYSROOT_DIR}/usr/lib/jvm/java-1.7.0-openjdk-*")
+      jvm_slug=$(basename ${jvm_slug})
+
+      pushd ${SYSROOT_DIR}/usr/lib/jvm/${jvm_slug}/jre/lib/security > /dev/null 2>&1
+      mkdir -p ../../../../../../../etc/pki/java/cacerts
+      popd > /dev/null 2>&1
 java-1.7.0-openjdk-headless:
   custom: false
   skipped_cdts:
@@ -131,10 +139,26 @@ libdrm:
 libdrm-devel:
   custom: false
   license_file: null
-# libglvnd:
-#   custom: false
-# libglvnd-glx:
-#   custom: false
+libglvnd:
+  custom: false
+  skipped_cdts:
+    - cos6-x86_64
+  license_file: licenses/libglvnd-license
+libglvnd-glx:
+  custom: false
+  skipped_cdts:
+    - cos6-x86_64
+  license_file: licenses/libglvnd-license
+  build_append:
+    all: |
+      pushd ${SYSROOT_DIR}/usr/lib64 > /dev/null 2>&1
+      libgl_file=$(find . -maxdepth 1 -name "libGL.so.*.*.*")
+      libgl_file=$(basename ${libgl_file})
+      ln -fs ${libgl_file} libGL.so
+      ls -lah .
+      popd > /dev/null 2>&1
+  dep_remove:
+    - mesa-libgl
 libibcm:
   custom: false
   skipped_cdts:
@@ -292,6 +316,9 @@ libXrender-devel:
 libXScrnSaver-devel:
   custom: false
   license_file: licenses/libxscrnsaver-license
+libxshmfence:
+  custom: false
+  license_file: licenses/libxshmfence-license
 libxshmfence-devel:
   custom: false
   license_file: licenses/libxshmfence-license
@@ -300,6 +327,8 @@ libXt:
 libXt-devel:
   custom: false
   license_file: licenses/libxt-license
+libXtst:
+  custom: false
 libXtst-devel:
   custom: false
   license_file: licenses/libxtst-license
@@ -356,33 +385,18 @@ mesa-libglapi:
   license_file: licenses/mesa-license
 nspr:
   custom: false
-  skipped_cdts:
-    - cos6-x86_64
-    - cos7-x86_64
   license_file: licenses/nspr-license
 nss:
   custom: false
-  skipped_cdts:
-    - cos6-x86_64
-    - cos7-x86_64
   license_file: licenses/nss-license
 nss-softokn:
   custom: false
-  skipped_cdts:
-    - cos6-x86_64
-    - cos7-x86_64
   license_file: licenses/nss-license
 nss-softokn-freebl:
   custom: false
-  skipped_cdts:
-    - cos6-x86_64
-    - cos7-x86_64
   license_file: licenses/nss-license
 nss-util:
   custom: false
-  skipped_cdts:
-    - cos6-x86_64
-    - cos7-x86_64
   license_file: licenses/nss-license
 numactl:
   custom: false

cdts/at-spi-cos6-x86_64/build.sh

@@ -17,4 +17,43 @@ if [[ -d usr/lib64 ]]; then
 fi
 pushd ${SRC_DIR}/binary > /dev/null 2>&1
 rsync -K -a . "${SYSROOT_DIR}"
-popd
+popd > /dev/null 2>&1
+
+# START OF INSERTED BUILD APPENDS
+
+# END OF INSERTED BUILD APPENDS
+
+# this code makes sure that any symlinks are relative and their targets exist
+# the CDT would fail at test time, but doing it here produces useful error
+# messages for fixing things
+error_code=0
+for blnk in $(find ./binary -type l); do
+  # loop is over symlinks in the RPM, so get the path in the sysroot
+  lnk=${SYSROOT_DIR}${blnk#"./binary"}
+
+  # if it is not a link in the sysroot, move on
+  if [[ ! -L ${lnk} ]]; then
+    continue
+  fi
+
+  # get the link dir and the destination of the link
+  lnk_dir=$(dirname ${lnk})
+  lnk_dst_nm=$(readlink ${lnk})
+  if [[ ${lnk_dst_nm:0:1} == "/" ]]; then
+    lnk_dst=${lnk_dst_nm}
+  else
+    lnk_dst="${lnk_dir}/${lnk_dst_nm}"
+  fi
+
+  # now test if it is absolute and relative to the system and not the PREFIX
+  # also test if the dest file exists
+  if [[ ${lnk_dst_nm:0:1} == "/" ]] && [[ ! ${lnk_dst_nm} == ${SYSROOT_DIR}* ]]; then
+    echo "***WARNING ABSOLUTE SYMLINK***: ${lnk} -> ${lnk_dst}"
+    error_code=1
+  elif [[ ! -e "${lnk_dst}" ]]; then
+     echo "***WARNING SYMLINK W/O DESTINATION: ${lnk} -> ${lnk_dst}"
+     error_code=1
+  fi
+done
+
+exit ${error_code}

cdts/at-spi-cos6-x86_64/meta.yaml

@@ -14,6 +14,8 @@ source:
 build:
   number: {{ cdt_build_number|int + 1000 }}
   noarch: generic
+  binary_relocation: False
+  detect_binary_files_with_prefix: False
   missing_dso_whitelist:
     - '*'
 
@@ -39,6 +41,10 @@ requirements:
     - libbonobo-cos6-x86_64 >=2.4.0 *_{{ cdt_build_number|int + 1000 }}
     - sysroot_linux-64 2.12.*
 
+test:
+  commands:
+    - echo "it installs!"
+
 about:
   home: http://developer.gnome.org/projects/gap/
   license: LGPL-2.0-or-later

cdts/at-spi-cos7-aarch64/build.sh

@@ -17,4 +17,43 @@ if [[ -d usr/lib64 ]]; then
 fi
 pushd ${SRC_DIR}/binary > /dev/null 2>&1
 rsync -K -a . "${SYSROOT_DIR}"
-popd
+popd > /dev/null 2>&1
+
+# START OF INSERTED BUILD APPENDS
+
+# END OF INSERTED BUILD APPENDS
+
+# this code makes sure that any symlinks are relative and their targets exist
+# the CDT would fail at test time, but doing it here produces useful error
+# messages for fixing things
+error_code=0
+for blnk in $(find ./binary -type l); do
+  # loop is over symlinks in the RPM, so get the path in the sysroot
+  lnk=${SYSROOT_DIR}${blnk#"./binary"}
+
+  # if it is not a link in the sysroot, move on
+  if [[ ! -L ${lnk} ]]; then
+    continue
+  fi
+
+  # get the link dir and the destination of the link
+  lnk_dir=$(dirname ${lnk})
+  lnk_dst_nm=$(readlink ${lnk})
+  if [[ ${lnk_dst_nm:0:1} == "/" ]]; then
+    lnk_dst=${lnk_dst_nm}
+  else
+    lnk_dst="${lnk_dir}/${lnk_dst_nm}"
+  fi
+
+  # now test if it is absolute and relative to the system and not the PREFIX
+  # also test if the dest file exists
+  if [[ ${lnk_dst_nm:0:1} == "/" ]] && [[ ! ${lnk_dst_nm} == ${SYSROOT_DIR}* ]]; then
+    echo "***WARNING ABSOLUTE SYMLINK***: ${lnk} -> ${lnk_dst}"
+    error_code=1
+  elif [[ ! -e "${lnk_dst}" ]]; then
+     echo "***WARNING SYMLINK W/O DESTINATION: ${lnk} -> ${lnk_dst}"
+     error_code=1
+  fi
+done
+
+exit ${error_code}

cdts/at-spi-cos7-aarch64/meta.yaml

@@ -14,6 +14,8 @@ source:
 build:
   number: {{ cdt_build_number|int + 1000 }}
   noarch: generic
+  binary_relocation: False
+  detect_binary_files_with_prefix: False
   missing_dso_whitelist:
     - '*'
 
@@ -39,6 +41,10 @@ requirements:
     - libbonobo-cos7-aarch64 >=2.4.0 *_{{ cdt_build_number|int + 1000 }}
     - sysroot_linux-aarch64 2.17.*
 
+test:
+  commands:
+    - echo "it installs!"
+
 about:
   home: http://developer.gnome.org/projects/gap/
   license: LGPL-2.0-or-later

cdts/at-spi-cos7-ppc64le/build.sh

@@ -17,4 +17,43 @@ if [[ -d usr/lib64 ]]; then
 fi
 pushd ${SRC_DIR}/binary > /dev/null 2>&1
 rsync -K -a . "${SYSROOT_DIR}"
-popd
+popd > /dev/null 2>&1
+
+# START OF INSERTED BUILD APPENDS
+
+# END OF INSERTED BUILD APPENDS
+
+# this code makes sure that any symlinks are relative and their targets exist
+# the CDT would fail at test time, but doing it here produces useful error
+# messages for fixing things
+error_code=0
+for blnk in $(find ./binary -type l); do
+  # loop is over symlinks in the RPM, so get the path in the sysroot
+  lnk=${SYSROOT_DIR}${blnk#"./binary"}
+
+  # if it is not a link in the sysroot, move on
+  if [[ ! -L ${lnk} ]]; then
+    continue
+  fi
+
+  # get the link dir and the destination of the link
+  lnk_dir=$(dirname ${lnk})
+  lnk_dst_nm=$(readlink ${lnk})
+  if [[ ${lnk_dst_nm:0:1} == "/" ]]; then
+    lnk_dst=${lnk_dst_nm}
+  else
+    lnk_dst="${lnk_dir}/${lnk_dst_nm}"
+  fi
+
+  # now test if it is absolute and relative to the system and not the PREFIX
+  # also test if the dest file exists
+  if [[ ${lnk_dst_nm:0:1} == "/" ]] && [[ ! ${lnk_dst_nm} == ${SYSROOT_DIR}* ]]; then
+    echo "***WARNING ABSOLUTE SYMLINK***: ${lnk} -> ${lnk_dst}"
+    error_code=1
+  elif [[ ! -e "${lnk_dst}" ]]; then
+     echo "***WARNING SYMLINK W/O DESTINATION: ${lnk} -> ${lnk_dst}"
+     error_code=1
+  fi
+done
+
+exit ${error_code}

cdts/at-spi-cos7-ppc64le/meta.yaml

@@ -14,6 +14,8 @@ source:
 build:
   number: {{ cdt_build_number|int + 1000 }}
   noarch: generic
+  binary_relocation: False
+  detect_binary_files_with_prefix: False
   missing_dso_whitelist:
     - '*'
 
@@ -39,6 +41,10 @@ requirements:
     - libbonobo-cos7-ppc64le >=2.4.0 *_{{ cdt_build_number|int + 1000 }}
     - sysroot_linux-ppc64le 2.17.*
 
+test:
+  commands:
+    - echo "it installs!"
+
 about:
   home: http://developer.gnome.org/projects/gap/
   license: LGPL-2.0-or-later

cdts/at-spi-cos7-x86_64/build.sh

@@ -17,4 +17,43 @@ if [[ -d usr/lib64 ]]; then
 fi
 pushd ${SRC_DIR}/binary > /dev/null 2>&1
 rsync -K -a . "${SYSROOT_DIR}"
-popd
+popd > /dev/null 2>&1
+
+# START OF INSERTED BUILD APPENDS
+
+# END OF INSERTED BUILD APPENDS
+
+# this code makes sure that any symlinks are relative and their targets exist
+# the CDT would fail at test time, but doing it here produces useful error
+# messages for fixing things
+error_code=0
+for blnk in $(find ./binary -type l); do
+  # loop is over symlinks in the RPM, so get the path in the sysroot
+  lnk=${SYSROOT_DIR}${blnk#"./binary"}
+
+  # if it is not a link in the sysroot, move on
+  if [[ ! -L ${lnk} ]]; then
+    continue
+  fi
+
+  # get the link dir and the destination of the link
+  lnk_dir=$(dirname ${lnk})
+  lnk_dst_nm=$(readlink ${lnk})
+  if [[ ${lnk_dst_nm:0:1} == "/" ]]; then
+    lnk_dst=${lnk_dst_nm}
+  else
+    lnk_dst="${lnk_dir}/${lnk_dst_nm}"
+  fi
+
+  # now test if it is absolute and relative to the system and not the PREFIX
+  # also test if the dest file exists
+  if [[ ${lnk_dst_nm:0:1} == "/" ]] && [[ ! ${lnk_dst_nm} == ${SYSROOT_DIR}* ]]; then
+    echo "***WARNING ABSOLUTE SYMLINK***: ${lnk} -> ${lnk_dst}"
+    error_code=1
+  elif [[ ! -e "${lnk_dst}" ]]; then
+     echo "***WARNING SYMLINK W/O DESTINATION: ${lnk} -> ${lnk_dst}"
+     error_code=1
+  fi
+done
+
+exit ${error_code}

cdts/at-spi-cos7-x86_64/meta.yaml

@@ -14,6 +14,8 @@ source:
 build:
   number: {{ cdt_build_number|int + 1000 }}
   noarch: generic
+  binary_relocation: False
+  detect_binary_files_with_prefix: False
   missing_dso_whitelist:
     - '*'
 
@@ -39,6 +41,10 @@ requirements:
     - libbonobo-cos7-x86_64 >=2.4.0 *_{{ cdt_build_number|int + 1000 }}
     - sysroot_linux-64 2.17.*
 
+test:
+  commands:
+    - echo "it installs!"
+
 about:
   home: http://developer.gnome.org/projects/gap/
   license: LGPL-2.0-or-later