.gitlab-ci.yml

stages:
  - build
  - test
  - collect_artifacts

cache:
  key: ${CI_COMMIT_REF_SLUG}
  paths:
    - .docker_ccache

build_wheel_debug_linux_python27:
  tags:
    - docker
  image: docker:19.03.8
  services:
    - docker:19.03.8-dind
  stage: build
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  script:
    - apk add bash nodejs
    - mkdir -p /etc/ssl/certs/
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4 --debug --docker-python2.7
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_wheel_linux_python27:
  tags:
    - docker
  services:
    - docker:19.03.8-dind
  image: docker:19.03.8
  stage: build
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  script:
    - apk add bash nodejs
    - mkdir -p /etc/ssl/certs/
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4 --docker-python2.7
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_wheel_linux_python35:
  tags:
    - docker
  services:
    - docker:19.03.8-dind
  image: docker:19.03.8
  stage: build
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  script:
    - apk add bash nodejs
    - mkdir -p /etc/ssl/certs/
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4 --docker-python3.5
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_wheel_linux_python36:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: build
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  script:
    - apk add bash nodejs
    - mkdir -p /etc/ssl/certs/
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4 --docker-python3.6
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_wheel_linux_python37:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: build
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  script:
    - apk add bash
    - mkdir -p /etc/ssl/certs/
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4 --docker-python3.7
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_wheel_linux_python38:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: build
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  script:
    - apk add bash
    - mkdir -p /etc/ssl/certs/
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4 --docker-python3.8
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_wheel_mac_10_15_python27:
  tags:
    - macos10.15
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: build
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
    - source scripts/use_ccache.sh
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_wheel_mac_10_15_python35:
  tags:
    - macos10.15
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: build
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.5/bin/virtualenv"
    - source scripts/use_ccache.sh
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_wheel_mac_10_15_python36:
  tags:
    - macos10.15
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: build
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.6/bin/virtualenv"
    - source scripts/use_ccache.sh
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_wheel_mac_10_15_python37:
  tags:
    - macos10.15
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: build
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.7/bin/virtualenv"
    - source scripts/use_ccache.sh
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_wheel_mac_10_15_python38:
  tags:
    - macos10.15
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: build
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.8/bin/virtualenv"
    - source scripts/use_ccache.sh
    - bash -e scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number=$CI_PIPELINE_ID --num_procs=4
  artifacts:
    expire_in: 1 day
    paths:
      - target/

build_dylib_macos_10_15:
  tags:
    - macos10.15
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: build
  script:
    - ./configure --no-visualization --no-python --no-remotefs
      --release-opt-for-size --with-capi --builder=xcode --no-pre-commit
    - cd release
    - xcodebuild -j 8 -project Turi.xcodeproj/ -target Recommender -quiet
    - xcodebuild -j 8 -project Turi.xcodeproj/ -target AudioPreprocessing -quiet
  artifacts:
    expire_in: 1 day
    paths:
      - release/src/deployment/Release/libRecommender.dylib
      - release/src/deployment/Release/libAudioPreprocessing.dylib

build_dylib_ios_12:
  tags:
    - macos10.15
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: build
  script:
    - ./configure --no-visualization --no-python --no-remotefs
      --release-opt-for-size --target=iphoneos --arch=arm64
      --with-capi --builder=xcode --no-pre-commit
    - cd release
    - xcodebuild -j 8 -project Turi.xcodeproj/ -target Recommender -arch arm64 only_active_arch=no -quiet
    - xcodebuild -j 8 -project Turi.xcodeproj/ -target AudioPreprocessing -arch arm64 only_active_arch=no -quiet
  artifacts:
    expire_in: 1 day
    paths:
      - release/src/deployment/Release-iphoneos/libRecommender.dylib
      - release/src/deployment/Release-iphoneos/libAudioPreprocessing.dylib

build_documentation:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python37
  script:
    - apk add bash
    - bash scripts/make_documentation.sh --docker
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - target/docs.zip

test_cpp_linux:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python27
  script:
    - apk add bash
    - apk add python2
    - python2.7 scripts/run_cpp_tests.py -j 2 --docker

test_cpp_linux_no_remotefs:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python27
  script:
    - apk add bash
    - apk add python2
    - python2.7 scripts/run_cpp_tests.py -j 2 --docker --configure-args='--no-remotefs --no-visualization'


test_cpp_mac_10_13:
  tags:
    - macos10.13
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python27
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
    - (test -d debug && test -d release) || ./configure --no-python --no-pre-commit
    - cd debug/test
    - bash -c -e -o pipefail "make -k -j8 2>&1 | grep -v -E '^ld[^a-z]*warning[^a-z]*direct access'"
    - python ../../scripts/run_cpp_tests.py -j 2

test_cpp_mac_10_14:
  tags:
    - macos10.14
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python27
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
    - (test -d debug && test -d release) || ./configure --no-python --no-pre-commit
    - cd debug/test
    - bash -c -e -o pipefail "make -k -j8 2>&1 | grep -v -E '^ld[^a-z]*warning[^a-z]*direct access'"
    - python ../../scripts/run_cpp_tests.py -j 2

test_cpp_mac_10_15:
  tags:
    - macos10.15
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python27
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
    - (test -d debug && test -d release) || ./configure --no-python --no-pre-commit
    - cd debug/test
    - bash -c -e -o pipefail "make -k -j8 2>&1 | grep -v -E '^ld[^a-z]*warning[^a-z]*direct access'"
    - python ../../scripts/run_cpp_tests.py -j 2

test_cpp_mac_10_15_no_remotefs:
  tags:
    - macos10.15
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python27
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
    - ./configure --no-remotefs --no-python --no-visualization --no-pre-commit
    - cd debug/test
    - bash -c -e -o pipefail "make -k -j8 2>&1 | grep -v -E '^ld[^a-z]*warning[^a-z]*direct access'"
    - python ../../scripts/run_cpp_tests.py -j 2

test_python_linux_python27:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python27
  script:
    - apk add bash
    - bash -e scripts/test_wheel.sh --docker-python2.7
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_linux_python27_minimal:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python27
  script:
    - apk add bash
    - bash -e scripts/test_wheel.sh --docker-python2.7 --minimal
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_linux_python35:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python35
  script:
    - apk add bash
    - bash -e scripts/test_wheel.sh --docker-python3.5
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_linux_python36:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python36
  script:
    - apk add bash
    # TODO get python 3.6 working on a test image
    - bash -e scripts/test_wheel.sh --docker-python3.6
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_linux_python37:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python37
  script:
    - apk add bash
    - bash -e scripts/test_wheel.sh --docker-python3.7
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_linux_python38:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python38
  script:
    - apk add bash
    - bash -e scripts/test_wheel.sh --docker-python3.8
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_mac_python27_10_13:
  tags:
    - macos10.13
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python27
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
    - bash scripts/test_wheel.sh
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_mac_python27_10_14:
  tags:
    - macos10.14
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python27
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
    - bash scripts/test_wheel.sh
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_mac_python35_10_14:
  tags:
    - macos10.14
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python35
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.5/bin/virtualenv"
    - bash -e scripts/test_wheel.sh
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_mac_python36_10_14:
  tags:
    - macos10.14
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python36
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.6/bin/virtualenv"
    - bash -e scripts/test_wheel.sh
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_mac_python37_10_14:
  tags:
    - macos10.14
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python37
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.7/bin/virtualenv"
    - bash -e scripts/test_wheel.sh
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

test_python_mac_python37_10_14_minimal:
  tags:
    - macos10.14
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python37
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.7/bin/virtualenv"
    - bash -e scripts/test_wheel.sh --minimal
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

# Core ML Tool currently has only beta support for Python 3.8.
# We can't currently rely on it as part of our CI.
# See GitHub issue:
#    https://github.com/apple/coremltools/issues/372
#test_python_mac_python38_10_15:
#  tags:
#    - macos10.15
#  only:
#    variables:
#      - $TC_HAS_MACOS_RUNNERS == "1"
#  stage: test
#  dependencies:
#    - build_wheel_mac_10_15_python38
#  script:
#    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.8/bin/virtualenv"
#    - bash -e scripts/test_wheel.sh
#  artifacts:
#    when: always
#    expire_in: 2 weeks
#    paths:
#      - pytest.*

test_python_mac_python38_10_14_minimal:
  tags:
    - macos10.14
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python38
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/3.8/bin/virtualenv"
    - bash -e scripts/test_wheel.sh --minimal
  artifacts:
    when: always
    expire_in: 2 weeks
    paths:
      - pytest.*

scenario_tests_linux_python27:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python27
  script:
    - apk add bash
    - cd scenario-tests
    - ./run_scenario_tests.sh --docker-python2.7 \
        "$(ls -1 ../target/turicreate-*.whl | grep -v minimal)"

scenario_tests_linux_python35:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python35
  script:
    - apk add bash
    - cd scenario-tests
    - ./run_scenario_tests.sh --docker-python3.5 \
        "$(ls -1 ../target/turicreate-*.whl | grep -v minimal)"

scenario_tests_linux_python36:
  tags:
    - docker
  services:
    - docker:19.03-dind
  image: docker:19.03.8
  stage: test
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_DRIVER: overlay2
    DOCKER_TLS_CERTDIR: ""
  dependencies:
    - build_wheel_linux_python36
  script:
    - apk add bash
    - cd scenario-tests
    - ./run_scenario_tests.sh --docker-python3.6 \
        "$(ls -1 ../target/turicreate-*.whl | grep -v minimal)"

scenario_tests_mac_python27_10_13:
  tags:
    - macos10.13
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python27
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
    - cd scenario-tests
    - ./run_scenario_tests.sh "$(ls -1 ../target/turicreate-*.whl | grep -v minimal)"

scenario_tests_mac_python27_10_14:
  tags:
    - macos10.14
  only:
    variables:
      - $TC_HAS_MACOS_RUNNERS == "1"
  stage: test
  dependencies:
    - build_wheel_mac_10_15_python27
  script:
    - export VIRTUALENV="/Library/Frameworks/Python.framework/Versions/2.7/bin/virtualenv"
    - cd scenario-tests
    - ./run_scenario_tests.sh "$(ls -1 ../target/turicreate-*.whl | grep -v minimal)"

collect_artifacts:
  tags:
    - docker
  image: alpine:latest
  stage: collect_artifacts
  dependencies:
    - build_dylib_ios_12
    - build_dylib_macos_10_15
    - build_wheel_linux_python27
    - build_wheel_linux_python35
    - build_wheel_linux_python36
    - build_wheel_linux_python37
    - build_wheel_linux_python38
    - build_wheel_mac_10_15_python27
    - build_wheel_mac_10_15_python35
    - build_wheel_mac_10_15_python36
    - build_wheel_mac_10_15_python37
    - build_wheel_mac_10_15_python38
    - build_documentation
  script:
    - find release/src/deployment/ -name '*.dylib' -exec mv {} target/ \;
    - rmdir -p release || find release
  artifacts:
    expire_in: 2 weeks
    paths:
      - target/