[query] Set QUAL to missing when it's "." (#14697)

When we switched from using htsjdk to do most of our VCF parsing to using our own parser, we kept the behavior of setting missing QUAL to a sentinel value. This isn't necessary (or correct) for hail to do. The type of QUAL is optional float64 and we should preserve the missing value. Technically speaking this represents a backwards compatibility change. However, I think this is a bug in my attempt in the past to keep bug-for-bug compatibility with htsjdk rather than parsing the value into what people would expect.
hail-is · Sep 20, 2024 · 13a9aee · 13a9aee
1 parent 9b8e322
commit 13a9aee
Show file tree

Hide file tree

Showing 52 changed files with 36 additions and 12 deletions.
diff --git a/hail/python/hail/methods/impex.py b/hail/python/hail/methods/impex.py
@@ -2883,13 +2883,13 @@ def import_vcf(
 
  >>> ds = hl.import_vcf('data/missing-values-in-array-fields.vcf', array_elements_required=False)
  >>> ds.show(n_rows=1, n_cols=1, include_row_fields=True)
- +---------------+------------+------+-----------+----------+--------------+
- | locus | alleles | rsid |  qual | filters | info.A |
- +---------------+------------+------+-----------+----------+--------------+
- | locus<GRCh37> | array<str> | str |  float64 | set<str> | array<int32> |
- +---------------+------------+------+-----------+----------+--------------+
- | 1:123456 | ["A","C"] | NA | -1.00e+01 | NA | [1,NA] |
- +---------------+------------+------+-----------+----------+--------------+
+ +---------------+------------+------+---------+----------+--------------+
+ | locus | alleles | rsid | qual | filters | info.A |
+ +---------------+------------+------+---------+----------+--------------+
+ | locus<GRCh37> | array<str> | str | float64 | set<str> | array<int32> |
+ +---------------+------------+------+---------+----------+--------------+
+ | 1:123456 | ["A","C"] | NA |  NA | NA | [1,NA] |
+ +---------------+------------+------+---------+----------+--------------+
  <BLANKLINE>
  +------------------+----------------+----------------+--------------+
  | info.B | info.C | info.D | 'SAMPLE1'.GT |

diff --git a/hail/src/main/scala/is/hail/io/vcf/LoadVCF.scala b/hail/src/main/scala/is/hail/io/vcf/LoadVCF.scala
@@ -1618,7 +1618,7 @@ object LoadVCF {
  if (c.hasQual) {
  val qstr = l.parseString()
  if (qstr == ".")
- rvb.addDouble(-10.0)
+ rvb.setMissing()
  else
  rvb.addDouble(qstr.toDouble)
  } else

diff --git a/hail/src/test/resources/ex.vcf.mt/README.txt b/hail/src/test/resources/ex.vcf.mt/README.txt
@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
- Written with version 0.2.12-4a66ad88153e
- Created at 2019/04/11 16:23:26
+ Written with version 0.2.132-8889c53db85c
+ Created at 2024/09/20 14:36:43
diff --git a/hail/src/test/resources/ex.vcf.mt/cols/README.txt b/hail/src/test/resources/ex.vcf.mt/cols/README.txt
@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+ Written with version 0.2.132-8889c53db85c
+ Created at 2024/09/20 14:36:43
diff --git a/hail/src/test/resources/ex.vcf.mt/cols/metadata.json.gz b/hail/src/test/resources/ex.vcf.mt/cols/metadata.json.gz
diff --git a/hail/src/test/resources/ex.vcf.mt/cols/rows/metadata.json.gz b/hail/src/test/resources/ex.vcf.mt/cols/rows/metadata.json.gz
diff --git a/hail/src/test/resources/ex.vcf.mt/cols/rows/parts/part-0 b/hail/src/test/resources/ex.vcf.mt/cols/rows/parts/part-0
diff --git a/hail/src/test/resources/ex.vcf.mt/entries/README.txt b/hail/src/test/resources/ex.vcf.mt/entries/README.txt
@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+ Written with version 0.2.132-8889c53db85c
+ Created at 2024/09/20 14:36:44
diff --git a/hail/src/test/resources/ex.vcf.mt/entries/metadata.json.gz b/hail/src/test/resources/ex.vcf.mt/entries/metadata.json.gz
diff --git a/hail/src/test/resources/ex.vcf.mt/entries/rows/metadata.json.gz b/hail/src/test/resources/ex.vcf.mt/entries/rows/metadata.json.gz
diff --git a/.../resources/ex.vcf.mt/entries/rows/parts/part-0-5-0-0-f674fb97-ae67-a4d4-5278-eaff0f6f8031 b/.../resources/ex.vcf.mt/entries/rows/parts/part-0-5-0-0-f674fb97-ae67-a4d4-5278-eaff0f6f8031
diff --git a/...c/test/resources/ex.vcf.mt/entries/rows/parts/part-0-ed5e014f-8074-4d32-9705-63b024fc1162 b/...c/test/resources/ex.vcf.mt/entries/rows/parts/part-0-ed5e014f-8074-4d32-9705-63b024fc1162
diff --git a/hail/src/test/resources/ex.vcf.mt/globals/README.txt b/hail/src/test/resources/ex.vcf.mt/globals/README.txt
@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+ Written with version 0.2.132-8889c53db85c
+ Created at 2024/09/20 14:36:43
diff --git a/hail/src/test/resources/ex.vcf.mt/globals/globals/metadata.json.gz b/hail/src/test/resources/ex.vcf.mt/globals/globals/metadata.json.gz
diff --git a/hail/src/test/resources/ex.vcf.mt/globals/globals/parts/part-0 b/hail/src/test/resources/ex.vcf.mt/globals/globals/parts/part-0
diff --git a/hail/src/test/resources/ex.vcf.mt/globals/metadata.json.gz b/hail/src/test/resources/ex.vcf.mt/globals/metadata.json.gz
diff --git a/hail/src/test/resources/ex.vcf.mt/globals/rows/metadata.json.gz b/hail/src/test/resources/ex.vcf.mt/globals/rows/metadata.json.gz
diff --git a/hail/src/test/resources/ex.vcf.mt/globals/rows/parts/part-0 b/hail/src/test/resources/ex.vcf.mt/globals/rows/parts/part-0
diff --git a/.../src/test/resources/ex.vcf.mt/index/part-0-ed5e014f-8074-4d32-9705-63b024fc1162.idx/index b/.../src/test/resources/ex.vcf.mt/index/part-0-ed5e014f-8074-4d32-9705-63b024fc1162.idx/index
diff --git a/...esources/ex.vcf.mt/index/part-0-ed5e014f-8074-4d32-9705-63b024fc1162.idx/metadata.json.gz b/...esources/ex.vcf.mt/index/part-0-ed5e014f-8074-4d32-9705-63b024fc1162.idx/metadata.json.gz
diff --git a/hail/src/test/resources/ex.vcf.mt/metadata.json.gz b/hail/src/test/resources/ex.vcf.mt/metadata.json.gz
diff --git a/hail/src/test/resources/ex.vcf.mt/rows/README.txt b/hail/src/test/resources/ex.vcf.mt/rows/README.txt
@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+ Written with version 0.2.132-8889c53db85c
+ Created at 2024/09/20 14:36:44
diff --git a/hail/src/test/resources/ex.vcf.mt/rows/metadata.json.gz b/hail/src/test/resources/ex.vcf.mt/rows/metadata.json.gz
diff --git a/hail/src/test/resources/ex.vcf.mt/rows/rows/metadata.json.gz b/hail/src/test/resources/ex.vcf.mt/rows/rows/metadata.json.gz
diff --git a/...est/resources/ex.vcf.mt/rows/rows/parts/part-0-5-0-0-f674fb97-ae67-a4d4-5278-eaff0f6f8031 b/...est/resources/ex.vcf.mt/rows/rows/parts/part-0-5-0-0-f674fb97-ae67-a4d4-5278-eaff0f6f8031
diff --git a/.../src/test/resources/ex.vcf.mt/rows/rows/parts/part-0-ed5e014f-8074-4d32-9705-63b024fc1162 b/.../src/test/resources/ex.vcf.mt/rows/rows/parts/part-0-ed5e014f-8074-4d32-9705-63b024fc1162
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/README.txt b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/README.txt
@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
- Written with version 0.2.12-4a66ad88153e
- Created at 2019/04/11 13:36:42
+ Written with version 0.2.132-8889c53db85c
+ Created at 2024/09/20 14:37:08
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/cols/README.txt b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/cols/README.txt
@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+ Written with version 0.2.132-8889c53db85c
+ Created at 2024/09/20 14:37:08
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/cols/metadata.json.gz b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/cols/metadata.json.gz
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/cols/rows/metadata.json.gz b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/cols/rows/metadata.json.gz
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/cols/rows/parts/part-0 b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/cols/rows/parts/part-0
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/entries/README.txt b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/entries/README.txt
@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+ Written with version 0.2.132-8889c53db85c
+ Created at 2024/09/20 14:37:08
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/entries/metadata.json.gz b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/entries/metadata.json.gz
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/entries/rows/metadata.json.gz b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/entries/rows/metadata.json.gz
diff --git a/...HG00096.g.vcf.gz.mt/entries/rows/parts/part-0-13-0-0-eb337ab9-da88-0427-70ae-3a3355377840 b/...HG00096.g.vcf.gz.mt/entries/rows/parts/part-0-13-0-0-eb337ab9-da88-0427-70ae-3a3355377840
diff --git a/.../gvcfs/HG00096.g.vcf.gz.mt/entries/rows/parts/part-0-cce3bb95-cfb5-49f4-b53e-0965d8a827cf b/.../gvcfs/HG00096.g.vcf.gz.mt/entries/rows/parts/part-0-cce3bb95-cfb5-49f4-b53e-0965d8a827cf
diff --git a/...HG00096.g.vcf.gz.mt/entries/rows/parts/part-1-13-1-0-c25ae343-a1a1-dba9-c4a1-fb32848b334b b/...HG00096.g.vcf.gz.mt/entries/rows/parts/part-1-13-1-0-c25ae343-a1a1-dba9-c4a1-fb32848b334b
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/README.txt b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/README.txt
@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+ Written with version 0.2.132-8889c53db85c
+ Created at 2024/09/20 14:37:08
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/globals/metadata.json.gz b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/globals/metadata.json.gz
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/globals/parts/part-0 b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/globals/parts/part-0
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/metadata.json.gz b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/metadata.json.gz
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/rows/metadata.json.gz b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/rows/metadata.json.gz
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/rows/parts/part-0 b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/globals/rows/parts/part-0
diff --git a/...ces/gvcfs/HG00096.g.vcf.gz.mt/index/part-0-cce3bb95-cfb5-49f4-b53e-0965d8a827cf.idx/index b/...ces/gvcfs/HG00096.g.vcf.gz.mt/index/part-0-cce3bb95-cfb5-49f4-b53e-0965d8a827cf.idx/index
diff --git a/...G00096.g.vcf.gz.mt/index/part-0-cce3bb95-cfb5-49f4-b53e-0965d8a827cf.idx/metadata.json.gz b/...G00096.g.vcf.gz.mt/index/part-0-cce3bb95-cfb5-49f4-b53e-0965d8a827cf.idx/metadata.json.gz
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/metadata.json.gz b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/metadata.json.gz
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/rows/README.txt b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/rows/README.txt
@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+ Written with version 0.2.132-8889c53db85c
+ Created at 2024/09/20 14:37:08
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/rows/metadata.json.gz b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/rows/metadata.json.gz
diff --git a/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/rows/rows/metadata.json.gz b/hail/src/test/resources/gvcfs/HG00096.g.vcf.gz.mt/rows/rows/metadata.json.gz
diff --git a/...fs/HG00096.g.vcf.gz.mt/rows/rows/parts/part-0-13-0-0-eb337ab9-da88-0427-70ae-3a3355377840 b/...fs/HG00096.g.vcf.gz.mt/rows/rows/parts/part-0-13-0-0-eb337ab9-da88-0427-70ae-3a3355377840
diff --git a/...ces/gvcfs/HG00096.g.vcf.gz.mt/rows/rows/parts/part-0-cce3bb95-cfb5-49f4-b53e-0965d8a827cf b/...ces/gvcfs/HG00096.g.vcf.gz.mt/rows/rows/parts/part-0-cce3bb95-cfb5-49f4-b53e-0965d8a827cf
diff --git a/...fs/HG00096.g.vcf.gz.mt/rows/rows/parts/part-1-13-1-0-c25ae343-a1a1-dba9-c4a1-fb32848b334b b/...fs/HG00096.g.vcf.gz.mt/rows/rows/parts/part-1-13-1-0-c25ae343-a1a1-dba9-c4a1-fb32848b334b