From f7a77ed148e44e43cbf13635acc3c008074b754d Mon Sep 17 00:00:00 2001 From: Peter Eacmen Date: Wed, 24 Mar 2021 17:38:31 -0400 Subject: [PATCH 1/8] fail deps.sh if any install command fails --- .gitignore | 1 + deps.sh | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index af5c03a05..b2e167e7b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist .coverage *.extracted *.pyc +**/*.egg-info \ No newline at end of file diff --git a/deps.sh b/deps.sh index 959566080..5072c7060 100755 --- a/deps.sh +++ b/deps.sh @@ -8,7 +8,9 @@ else YES=0 fi +set -eu set -o nounset +set -x if ! which lsb_release > /dev/null then @@ -65,6 +67,7 @@ PIP_COMMANDS="pip3" # Check for root privileges if [ $UID -eq 0 ] then + echo "UID is 0, sudo not required" SUDO="" else SUDO="sudo" @@ -80,7 +83,7 @@ function install_yaffshiv function install_sasquatch { - git clone https://github.com/devttys0/sasquatch + git clone --quiet --depth 1 --branch "patch-1" https://github.com/eacmen/sasquatch (cd sasquatch && $SUDO ./build.sh) $SUDO rm -rf sasquatch } From 7fc3de141fa0a12e81838aa2704f14d0b061b534 Mon Sep 17 00:00:00 2001 From: Peter Eacmen Date: Wed, 24 Mar 2021 18:19:50 -0400 Subject: [PATCH 2/8] we no longer need to bifurcate python3 and python2 deps --- .travis.yml | 1 + deps.sh | 16 ++-------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90f3cfa66..4efe049e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ os: linux dist: bionic language: python arch: amd64 +sudo: true addons: apt: update: true diff --git a/deps.sh b/deps.sh index 5072c7060..03df74bb7 100755 --- a/deps.sh +++ b/deps.sh @@ -62,7 +62,7 @@ fi PYTHON3_APT_CANDIDATES="" PYTHON3_YUM_CANDIDATES="" YUM_CANDIDATES="git gcc gcc-c++ make openssl-devel qtwebkit-devel qt-devel gzip bzip2 tar arj p7zip p7zip-plugins cabextract squashfs-tools zlib zlib-devel lzo lzo-devel xz xz-compat-libs xz-libs xz-devel xz-lzma-compat python-backports-lzma lzip pyliblzma perl-Compress-Raw-Lzma lzop srecord" -PIP_COMMANDS="pip3" +PIP_COMMANDS="pip" # Check for root privileges if [ $UID -eq 0 ] @@ -134,11 +134,7 @@ function install_ubireader function install_pip_package { PACKAGE="$1" - - for PIP_COMMAND in $PIP_COMMANDS - do - $SUDO $PIP_COMMAND install $PACKAGE - done + $SUDO pip install $PACKAGE } function find_path @@ -243,14 +239,6 @@ then exit 1 fi -# Check to see if we should install modules for python3 as well -find_path python3 -if [ $? -eq 0 ] -then - PKG_CANDIDATES="$PKG_CANDIDATES $PKG_PYTHON3_CANDIDATES" - PIP_COMMANDS="pip3 $PIP_COMMANDS" -fi - # Do the install(s) cd /tmp $SUDO $PKGCMD $PKGCMD_OPTS $PKG_CANDIDATES From 8f62e92f1a32f073b66e099e2a800766b14cafff Mon Sep 17 00:00:00 2001 From: Peter Eacmen Date: Wed, 24 Mar 2021 18:40:43 -0400 Subject: [PATCH 3/8] using pip3 is causing issues, use whatever pip the user has in their environment --- .travis.yml | 1 - deps.sh | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4efe049e7..90f3cfa66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ os: linux dist: bionic language: python arch: amd64 -sudo: true addons: apt: update: true diff --git a/deps.sh b/deps.sh index 03df74bb7..230b8ccd5 100755 --- a/deps.sh +++ b/deps.sh @@ -62,7 +62,6 @@ fi PYTHON3_APT_CANDIDATES="" PYTHON3_YUM_CANDIDATES="" YUM_CANDIDATES="git gcc gcc-c++ make openssl-devel qtwebkit-devel qt-devel gzip bzip2 tar arj p7zip p7zip-plugins cabextract squashfs-tools zlib zlib-devel lzo lzo-devel xz xz-compat-libs xz-libs xz-devel xz-lzma-compat python-backports-lzma lzip pyliblzma perl-Compress-Raw-Lzma lzop srecord" -PIP_COMMANDS="pip" # Check for root privileges if [ $UID -eq 0 ] @@ -91,7 +90,7 @@ function install_sasquatch function install_jefferson { git clone https://github.com/sviehb/jefferson - (cd jefferson && $SUDO pip3 install -r requirements.txt && $SUDO python3 setup.py install) + (cd jefferson && $SUDO pip install -r requirements.txt && $SUDO python3 setup.py install) $SUDO rm -rf jefferson } From 4696cab05e37270be98117a02745557963ed8265 Mon Sep 17 00:00:00 2001 From: Peter Eacmen Date: Wed, 24 Mar 2021 18:51:57 -0400 Subject: [PATCH 4/8] Travis-CI does not preserve the specified python when invoking sudo --- deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.sh b/deps.sh index 230b8ccd5..8a267b102 100755 --- a/deps.sh +++ b/deps.sh @@ -69,7 +69,7 @@ then echo "UID is 0, sudo not required" SUDO="" else - SUDO="sudo" + SUDO="sudo -E" REQUIRED_UTILS="sudo $REQUIRED_UTILS" fi From 4e380ec2b2529f35d81fd061a225bc1cc043cb56 Mon Sep 17 00:00:00 2001 From: Peter Eacmen Date: Wed, 24 Mar 2021 18:55:26 -0400 Subject: [PATCH 5/8] there is no rhyme or reason to travis --- deps.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deps.sh b/deps.sh index 8a267b102..339d66a10 100755 --- a/deps.sh +++ b/deps.sh @@ -90,7 +90,7 @@ function install_sasquatch function install_jefferson { git clone https://github.com/sviehb/jefferson - (cd jefferson && $SUDO pip install -r requirements.txt && $SUDO python3 setup.py install) + (cd jefferson && $SUDO pip3 install -r requirements.txt && $SUDO python3 setup.py install) $SUDO rm -rf jefferson } @@ -133,7 +133,7 @@ function install_ubireader function install_pip_package { PACKAGE="$1" - $SUDO pip install $PACKAGE + $SUDO pip3 install $PACKAGE } function find_path From 3e0fe9beafebc5218a67ddfb523850be4f135463 Mon Sep 17 00:00:00 2001 From: Peter Eacmen Date: Wed, 24 Mar 2021 19:05:02 -0400 Subject: [PATCH 6/8] try to preserve the python being used across sudo --- deps.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deps.sh b/deps.sh index 339d66a10..f44d4106e 100755 --- a/deps.sh +++ b/deps.sh @@ -62,6 +62,7 @@ fi PYTHON3_APT_CANDIDATES="" PYTHON3_YUM_CANDIDATES="" YUM_CANDIDATES="git gcc gcc-c++ make openssl-devel qtwebkit-devel qt-devel gzip bzip2 tar arj p7zip p7zip-plugins cabextract squashfs-tools zlib zlib-devel lzo lzo-devel xz xz-compat-libs xz-libs xz-devel xz-lzma-compat python-backports-lzma lzip pyliblzma perl-Compress-Raw-Lzma lzop srecord" +PYTHON="$(which python3)" # Check for root privileges if [ $UID -eq 0 ] @@ -76,7 +77,7 @@ fi function install_yaffshiv { git clone https://github.com/devttys0/yaffshiv - (cd yaffshiv && $SUDO python3 setup.py install) + (cd yaffshiv && $SUDO $PYTHON setup.py install) $SUDO rm -rf yaffshiv } @@ -90,7 +91,7 @@ function install_sasquatch function install_jefferson { git clone https://github.com/sviehb/jefferson - (cd jefferson && $SUDO pip3 install -r requirements.txt && $SUDO python3 setup.py install) + (cd jefferson && $SUDO $PYTHON -mpip install -r requirements.txt && $SUDO $PYTHON setup.py install) $SUDO rm -rf jefferson } @@ -126,14 +127,14 @@ function install_cramfstools function install_ubireader { git clone https://github.com/jrspruitt/ubi_reader - (cd ubi_reader && $SUDO python3 setup.py install) + (cd ubi_reader && $SUDO $PYTHON setup.py install) $SUDO rm -rf ubi_reader } function install_pip_package { PACKAGE="$1" - $SUDO pip3 install $PACKAGE + $SUDO $PYTHON -mpip install $PACKAGE } function find_path From 366de69a3d16949eb29aff2592607260fad5395f Mon Sep 17 00:00:00 2001 From: Peter Eacmen Date: Wed, 24 Mar 2021 19:10:41 -0400 Subject: [PATCH 7/8] nop --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90f3cfa66..8b7410ab3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,13 @@ addons: apt: update: true install: + - export PYTHON=$(which python3) - ./deps.sh --yes - - pip install coverage - - ./setup.py install -script: ./setup.py test -cache: pip + - sudo -E $PYTHON -mpip install coverage + - sudo -E $PYTHON ./setup.py install +script: + - export PYTHON=$(which python3) + - sudo -E $PYTHON ./setup.py test jobs: include: - stage: test From c84bec17d6304f96e0e7b7ffa5c2d9f2581a4f81 Mon Sep 17 00:00:00 2001 From: Peter Eacmen Date: Wed, 24 Mar 2021 21:28:37 -0400 Subject: [PATCH 8/8] sasquatch patch was merged --- deps.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deps.sh b/deps.sh index f44d4106e..cbd821cd9 100755 --- a/deps.sh +++ b/deps.sh @@ -76,21 +76,21 @@ fi function install_yaffshiv { - git clone https://github.com/devttys0/yaffshiv + git clone --quiet --depth 1 --branch "master" https://github.com/devttys0/yaffshiv (cd yaffshiv && $SUDO $PYTHON setup.py install) $SUDO rm -rf yaffshiv } function install_sasquatch { - git clone --quiet --depth 1 --branch "patch-1" https://github.com/eacmen/sasquatch + git clone --quiet --depth 1 --branch "master" https://github.com/devttys0/sasquatch (cd sasquatch && $SUDO ./build.sh) $SUDO rm -rf sasquatch } function install_jefferson { - git clone https://github.com/sviehb/jefferson + git clone --quiet --depth 1 --branch "master" https://github.com/sviehb/jefferson (cd jefferson && $SUDO $PYTHON -mpip install -r requirements.txt && $SUDO $PYTHON setup.py install) $SUDO rm -rf jefferson } @@ -113,7 +113,7 @@ function install_cramfstools INSTALL_LOCATION=/usr/local/bin # https://github.com/torvalds/linux/blob/master/fs/cramfs/README#L106 - git clone https://github.com/npitre/cramfs-tools + git clone --quiet --depth 1 --branch "master" https://github.com/npitre/cramfs-tools # There is no "make install" (cd cramfs-tools \ && make \ @@ -126,7 +126,7 @@ function install_cramfstools function install_ubireader { - git clone https://github.com/jrspruitt/ubi_reader + git clone --quiet --depth 1 --branch "master" https://github.com/jrspruitt/ubi_reader (cd ubi_reader && $SUDO $PYTHON setup.py install) $SUDO rm -rf ubi_reader }