From d1d595862598c2da921e040ef2c2893eb60fb89a Mon Sep 17 00:00:00 2001 From: Tom Forbes Date: Tue, 1 Aug 2023 22:12:06 +0100 Subject: [PATCH] Try this --- .github/workflows/run.yml | 2 +- sql/combine.sql | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 276abd60..2af697d8 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -145,7 +145,7 @@ jobs: - run: ls -la ${{ github.workspace }}/input/ - name: Combine - run: ${{ github.workspace }}/duckdb -echo -stats foo.db < ${{ github.workspace }}/sql/combine.sql + run: ${{ github.workspace }}/duckdb -echo foo.db < ${{ github.workspace }}/sql/combine.sql - run: ls -la ${{ github.workspace }}/*.parquet diff --git a/sql/combine.sql b/sql/combine.sql index 5defd397..d900f102 100644 --- a/sql/combine.sql +++ b/sql/combine.sql @@ -1,5 +1,6 @@ PRAGMA memory_limit='2GB'; PRAGMA threads=4; +SET enable_progress_bar=true; CREATE TABLE temp_table AS select regexp_extract(filename, '(\d+)\.parquet', 1)::USMALLINT as repository, * exclude (filename) FROM read_parquet('input/*.parquet', union_by_name = True, filename = true);