Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update Enso_File download to Cloud API changes #11596

Open
wants to merge 7 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions .github/workflows/engine-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ jobs:
- run: ./run backend test jvm
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Engine Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -327,7 +327,7 @@ jobs:
- run: ./run backend test jvm
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Engine Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -382,7 +382,7 @@ jobs:
- run: ./run backend test jvm
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Engine Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -438,7 +438,7 @@ jobs:
- run: ./run backend test jvm
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Engine Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -494,7 +494,7 @@ jobs:
- run: ./run backend test jvm
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Engine Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -553,7 +553,7 @@ jobs:
ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Standard Library Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -613,7 +613,7 @@ jobs:
ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Standard Library Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -671,7 +671,7 @@ jobs:
ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Standard Library Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -730,7 +730,7 @@ jobs:
ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Standard Library Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -789,7 +789,7 @@ jobs:
ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Standard Library Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/extra-nightly-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ jobs:
ENSO_SNOWFLAKE_USER: ${{ secrets.ENSO_SNOWFLAKE_USER }}
ENSO_SNOWFLAKE_WAREHOUSE: ${{ secrets.ENSO_SNOWFLAKE_WAREHOUSE }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Extra Library Test Reporter
uses: dorny/test-reporter@v1
with:
Expand All @@ -77,6 +77,7 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
env:
GRAAL_EDITION: GraalVM CE
REPORT_ALL_TESTS: "1"
permissions:
checks: write
enso-build-ci-gen-job-standard-library-tests-graal-vm-ce-linux-amd64:
Expand Down Expand Up @@ -118,7 +119,7 @@ jobs:
ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Standard Library Test Reporter
uses: dorny/test-reporter@v1
with:
Expand All @@ -140,6 +141,7 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
env:
GRAAL_EDITION: GraalVM CE
REPORT_ALL_TESTS: "1"
permissions:
checks: write
env:
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/scala-new.yml
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ jobs:
- run: ./run backend test jvm
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Engine Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -248,7 +248,7 @@ jobs:
- run: ./run backend test jvm
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Engine Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -304,7 +304,7 @@ jobs:
- run: ./run backend test jvm
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Engine Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -363,7 +363,7 @@ jobs:
ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Standard Library Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -421,7 +421,7 @@ jobs:
ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Standard Library Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down Expand Up @@ -480,7 +480,7 @@ jobs:
ENSO_LIB_S3_AWS_REGION: ${{ secrets.ENSO_LIB_S3_AWS_REGION }}
ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY: ${{ secrets.ENSO_LIB_S3_AWS_SECRET_ACCESS_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
- if: (success() || failure()) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
name: Standard Library Test Reporter
uses: dorny/test-reporter@v1
with:
Expand Down
6 changes: 4 additions & 2 deletions build/build/src/ci_gen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -161,11 +161,13 @@ pub fn not_default_branch() -> String {
format!("github.ref != 'refs/heads/{DEFAULT_BRANCH_NAME}'")
}

/// Expression piece that evaluates to `true` if we are **not** building a fork.
/// Expression piece that evaluates to `true` if we are **not** building a PR from a fork.
///
/// As fork builds are run with different permissions, sometimes we need to skip some steps.
/// If we are not on a PR build, the first condition makes this expression evaluate to `true`.
/// If it is a PR run, we check if the PR is in the same repository as the base branch.
pub fn not_a_fork() -> String {
"github.event.pull_request.head.repo.full_name == github.repository".into()
"(github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)".into()
}

pub fn release_concurrency() -> Concurrency {
Expand Down
9 changes: 9 additions & 0 deletions build/build/src/ci_gen/job.rs
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,14 @@ impl JobArchetype for StandardLibraryTests {
graalvm::Edition::Enterprise =>
job.env(env::GRAAL_EDITION, graalvm::Edition::Enterprise),
}

// If running extra cloud tests, enable reporting all tests. These tests run on a nightly
// schedule, and so the normal test reporter is not available to them. Thus we want to see
// the full log in the CI to be able to tell which tests have been run.
if should_enable_cloud_tests {
job.env(crate::libraries_tests::env::REPORT_ALL_TESTS, "1");
}

job
}

Expand Down Expand Up @@ -370,6 +378,7 @@ impl JobArchetype for SnowflakeTests {
.build_job(job_name, RunnerLabel::LinuxLatest)
.with_permission(Permission::Checks, Access::Write);
job.env(env::GRAAL_EDITION, GRAAL_EDITION_FOR_EXTRA_TESTS);
job.env(crate::libraries_tests::env::REPORT_ALL_TESTS, "1");
job
}

Expand Down
8 changes: 8 additions & 0 deletions build/build/src/libraries_tests.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
pub mod env {
ide_ci::define_env_var! {
/// Whether to display all test results in the CI output.
/// (By default successful tests are hidden.)
REPORT_ALL_TESTS, String;
}
}

pub mod s3 {
/// Environment variables used inside of the S3 tests.
pub mod env {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,7 @@ type Enso_File
asset = Existing_Enso_Asset.get_asset_reference_for self
response = case asset.asset_type of
Enso_Asset_Type.File ->
presigned_url = asset.get_file_description |> get_required_field "url" expected_type=Text
presigned_url = asset.get_file_description want_presinged_url=True |> get_required_field "url" expected_type=Text
# We are skipping the cache, because pre-signed URLs are unlikely to repeat, so caching them would be pointless.
# Caching of cloud files tracked by https://github.com/enso-org/enso/issues/11439
HTTP.fetch presigned_url HTTP_Method.Get cache_policy=..No_Cache
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,17 @@ type Existing_Enso_Asset

## PRIVATE
Fetches the basic information about a file from the Cloud endpoint.
get_file_description self -> JS_Object =

Arguments:
- want_presigned_url: Whether to request presigned URL in the response.
The cloud does not include the presigned URL in the response by default.
It should only be asked for when needed.
get_file_description self want_presinged_url:Boolean=False -> JS_Object =
if self.asset_type != Enso_Asset_Type.File then
Panic.throw (Illegal_Argument.Error "`get_file_description` can only be called on File assets.")
Utils.http_request_as_json HTTP_Method.Get self.internal_uri
base_uri = URI.from self.internal_uri
uri = if want_presinged_url then base_uri.add_query_argument "presigned" "true" else base_uri
Utils.http_request_as_json HTTP_Method.Get uri

## PRIVATE
Fetches the basic information about an existing file from the Cloud.
Expand Down
Loading