Skip to content

Commit

Permalink
Try this
Browse files Browse the repository at this point in the history
  • Loading branch information
orf committed Jul 30, 2023
1 parent af5b0bb commit 3f12a7d
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 18 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/unique_python_files.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,14 @@ jobs:
- name: Ingest
run: |
mkdir combined/
find dataset/ -name '*.parquet' | parallel -j 1 --xargs -n4 poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql --per-thread-output --output=parquet --threads=2 combined/{#}.parquet {}
find dataset/ -name '*.parquet' | parallel -j 1 --xargs -n4 poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files.prql --output=parquet --threads=2 combined/{#}.parquet {}
- name: List
run: ls combined/

- name: Combine
run: |
poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files_combine.prql --per-thread-output --output=parquet --memory=3 --threads=2 unique-python-files.parquet combined/*.parquet
poetry run pypi-data run-sql ${{ github.workspace }}/sql/unique_python_files_combine.prql --output=parquet --memory=3 --threads=2 unique-python-files.parquet combined/*.parquet
#
# - name: Gets latest created release info
Expand Down
11 changes: 2 additions & 9 deletions sql/unique_python_files.prql
Original file line number Diff line number Diff line change
@@ -1,13 +1,6 @@
prql target:sql.duckdb

let any_value = column -> s"any_value({column})"

from (read_parquet $1)
filter path ~= "\\.py$"
filter s"suffix({path}, '.py')"
filter skip_reason == ""
select {hash, path}
group {hash} (
aggregate {
path = any_value(path)
})
sort {hash}
select {s"distinct on({hash}) {hash}", path}
9 changes: 2 additions & 7 deletions sql/unique_python_files_combine.prql
Original file line number Diff line number Diff line change
@@ -1,10 +1,5 @@
prql target:sql.duckdb

let any_value = column -> s"any_value({column})"

from (read_parquet $1)
select {hash, path}
group {hash} (
aggregate {
path = any_value(path)
})
select {s"distinct on({hash}) {hash}", path}

0 comments on commit 3f12a7d

Please sign in to comment.