(WIP) feat: make text formatting preserve invalid values by coercing …

…them to string by default

TODO: test invalid wrapping for time

site/docs/invaliddata.md

@@ -5,7 +5,7 @@ title: Modes for Handling Invalid Data
 permalink: /docs/invalid-data.html
 ---
 
-This page discusses modes in Vega-Lite for handling invalid data (`null` and `NaN` in continuous scales).
+This page discusses modes in Vega-Lite for handling invalid data (`null` and `NaN` in continuous scales *without* defined output for invalid values in `config.scale.invalid`).
 
 The main configurations are [`mark.invalid`](#mark) and [`config.scale.invalid`](#scale). In addition, you can use [other Vega-Lite features including conditional encodings, layering, or window transform to handle invalid and missing data](#other).
 

src/axis.ts

@@ -565,7 +565,8 @@ const AXIS_PROPERTIES_INDEX: Flag<keyof Axis<any>> = {
   ...COMMON_AXIS_PROPERTIES_INDEX,
   style: 1,
   labelExpr: 1,
-  encoding: 1
+  encoding: 1,
+  formatInvalid: 1
 };
 
 export function isAxisProperty(prop: string): prop is keyof Axis<any> {

src/channeldef.ts

@@ -398,7 +398,7 @@ export interface DatumDef<
   // `F extends RepeatRef` probably should be `RepeatRef extends F` but there is likely a bug in TS.
 }
 
-export interface FormatMixins {
+export interface FormatMixins<FT extends string = 'number' | 'time' | string> {
   /**
    * When used with the default `"number"` and `"time"` format type, the text formatting pattern for labels of guides (axes, legends, headers) and text marks.
    *
@@ -420,7 +420,18 @@ export interface FormatMixins {
    * - `"time"` for temporal fields and ordinal and nominal fields with `timeUnit`.
    * - `"number"` for quantitative fields as well as ordinal and nominal fields without `timeUnit`.
    */
-  formatType?: 'number' | 'time' | string;
+  formatType?: FT;
+
+  /**
+   * Configure if the specified `format` and `formatType` should handle invalid values
+   * (nulls and NaNs on continuous scales *without* defined output for invalid values in `config.scale.invalid`).
+   *
+   * - `false` (Default) -- the invalid values will be coerced to strings via Vega Expression's `toString` without running the format function. Thus, null values will show up as `"null"`.
+   * - `true` -- the invalid value will be formatted as defined in the specified `format` function.  If no custom format is specified and `formatInvalid` is `true`, by default d3-format (Vega-Lite's default formatter) will format null values as `"0"`.
+   *
+   * @hidden
+   */
+  formatInvalid?: boolean;
 }
 
 export type StringDatumDef<F extends Field = string> = DatumDef<F> & FormatMixins;
@@ -974,14 +985,14 @@ export function defaultTitle(fieldDef: FieldDefBase<string>, config: Config) {
   return titleFormatter(fieldDef, config);
 }
 
-export function getFormatMixins(fieldDef: TypedFieldDef<string> | DatumDef) {
+export function getFormatMixins(fieldDef: TypedFieldDef<string> | DatumDef): FormatMixins {
   if (isStringFieldOrDatumDef(fieldDef)) {
-    const {format, formatType} = fieldDef;
-    return {format, formatType};
+    const {format, formatType, formatInvalid} = fieldDef;
+    return {format, formatType, formatInvalid};
   } else {
     const guide = getGuide(fieldDef) ?? {};
-    const {format, formatType} = guide;
-    return {format, formatType};
+    const {format, formatType, formatInvalid} = guide;
+    return {format, formatType, formatInvalid};
   }
 }
 

src/compile/axis/encode.ts

@@ -9,15 +9,14 @@ export function labels(model: UnitModel, channel: PositionScaleChannel, specifie
   const fieldOrDatumDef =
     getFieldOrDatumDef<string>(encoding[channel]) ?? getFieldOrDatumDef(encoding[getSecondaryRangeChannel(channel)]);
   const axis = model.axis(channel) || {};
-  const {format, formatType} = axis;
+  const {format, formatType, formatInvalid} = axis;
 
   if (isCustomFormatType(formatType)) {
     return {
       text: formatCustomType({
         fieldOrDatumDef,
         field: 'datum.value',
-        format,
-        formatType,
+        formatMixins: {format, formatType, formatInvalid},
         config
       }),
       ...specifiedLabelsSpec
@@ -33,8 +32,11 @@ export function labels(model: UnitModel, channel: PositionScaleChannel, specifie
           text: formatCustomType({
             fieldOrDatumDef,
             field: 'datum.value',
-            format: config.normalizedNumberFormat,
-            formatType: config.normalizedNumberFormatType,
+            formatMixins: {
+              format: config.normalizedNumberFormat,
+              formatType: config.normalizedNumberFormatType,
+              formatInvalid: config.normalizedNumberFormatInvalid
+            },
             config
           }),
           ...specifiedLabelsSpec
@@ -44,8 +46,11 @@ export function labels(model: UnitModel, channel: PositionScaleChannel, specifie
           text: formatCustomType({
             fieldOrDatumDef,
             field: 'datum.value',
-            format: config.numberFormat,
-            formatType: config.numberFormatType,
+            formatMixins: {
+              format: config.numberFormat,
+              formatType: config.numberFormatType,
+              formatInvalid: config.numberFormatInvalid
+            },
             config
           }),
           ...specifiedLabelsSpec
@@ -62,8 +67,11 @@ export function labels(model: UnitModel, channel: PositionScaleChannel, specifie
         text: formatCustomType({
           fieldOrDatumDef,
           field: 'datum.value',
-          format: config.timeFormat,
-          formatType: config.timeFormatType,
+          formatMixins: {
+            format: config.timeFormat,
+            formatType: config.timeFormatType,
+            formatInvalid: config.timeFormatInvalid
+          },
           config
         }),
         ...specifiedLabelsSpec

src/compile/data/bin.ts

@@ -25,7 +25,7 @@ function rangeFormula(model: ModelWithField, fieldDef: TypedFieldDef<string>, ch
 
     return {
       formulaAs: vgField(fieldDef, {binSuffix: 'range', forAs: true}),
-      formula: binFormatExpression(startField, endField, guide.format, guide.formatType, config)
+      formula: binFormatExpression(startField, endField, guide, config)
     };
   }
   return {};

src/compile/format.ts

@@ -5,6 +5,7 @@ import {
   channelDefType,
   DatumDef,
   FieldDef,
+  FormatMixins,
   isFieldDef,
   isFieldOrDatumDefForTimeFormat,
   isPositionFieldOrDatumDef,
@@ -21,7 +22,9 @@ import {isSignalRef} from '../vega.schema';
 import {TimeUnit} from './../timeunit';
 import {datumDefToExpr} from './mark/encode/valueref';
 
-export function isCustomFormatType(formatType: string) {
+export type CustomFormatType = Exclude<string, 'number' | 'time'>;
+
+export function isCustomFormatType(formatType: string): formatType is CustomFormatType {
   return formatType && formatType !== 'number' && formatType !== 'time';
 }
 
@@ -33,24 +36,23 @@ export const BIN_RANGE_DELIMITER = ' \u2013 ';
 
 export function formatSignalRef({
   fieldOrDatumDef,
-  format,
-  formatType,
+  formatMixins,
   expr,
   normalizeStack,
   config
 }: {
   fieldOrDatumDef: FieldDef<string> | DatumDef<string>;
-  format: string | Dict<unknown>;
-  formatType: string;
+  formatMixins: FormatMixins;
   expr?: 'datum' | 'parent' | 'datum.datum';
   normalizeStack?: boolean;
   config: Config;
 }) {
+  const {formatType, formatInvalid} = formatMixins;
+  let {format} = formatMixins;
   if (isCustomFormatType(formatType)) {
     return formatCustomType({
       fieldOrDatumDef,
-      format,
-      formatType,
+      formatMixins: {...formatMixins, formatType},
       expr,
       config
     });
@@ -64,16 +66,22 @@ export function formatSignalRef({
       if (normalizeStack && config.normalizedNumberFormatType)
         return formatCustomType({
           fieldOrDatumDef,
-          format: config.normalizedNumberFormat,
-          formatType: config.normalizedNumberFormatType,
+          formatMixins: {
+            format: config.normalizedNumberFormat,
+            formatType: config.normalizedNumberFormatType,
+            formatInvalid: config.normalizedNumberFormatInvalid
+          },
           expr,
           config
         });
       if (config.numberFormatType) {
         return formatCustomType({
           fieldOrDatumDef,
-          format: config.numberFormat,
-          formatType: config.numberFormatType,
+          formatMixins: {
+            format: config.numberFormat,
+            formatType: config.numberFormatType,
+            formatInvalid: config.numberFormatInvalid
+          },
           expr,
           config
         });
@@ -87,8 +95,7 @@ export function formatSignalRef({
     ) {
       return formatCustomType({
         fieldOrDatumDef,
-        format: config.timeFormat,
-        formatType: config.timeFormatType,
+        formatMixins,
         expr,
         config
       });
@@ -111,17 +118,24 @@ export function formatSignalRef({
   if (isFieldDef(fieldOrDatumDef) && isBinning(fieldOrDatumDef.bin)) {
     const endField = vgField(fieldOrDatumDef, {expr, binSuffix: 'end'});
     return {
-      signal: binFormatExpression(field, endField, format, formatType, config)
+      signal: binFormatExpression(field, endField, {format, formatType, formatInvalid}, config)
     };
   } else if (format || channelDefType(fieldOrDatumDef) === 'quantitative') {
-    return {
-      signal: `${formatExpr(field, format)}`
-    };
+    const signal = wrapFormatExprToHandleInvalidValues({
+      mainFormatExpr: builtInFormatExpr(field, format),
+      fieldExpr: field,
+      formatInvalid
+    });
+    return {signal};
   } else {
-    return {signal: `isValid(${field}) ? ${field} : ""+${field}`};
+    return {signal: toStringExpr(field)};
   }
 }
 
+function toStringExpr(expr: string) {
+  return `"" + ${expr}`;
+}
+
 function fieldToFormat(
   fieldOrDatumDef: FieldDef<string> | DatumDef<string>,
   expr: 'datum' | 'parent' | 'datum.datum',
@@ -143,16 +157,14 @@ function fieldToFormat(
 
 export function formatCustomType({
   fieldOrDatumDef,
-  format,
-  formatType,
   expr,
   normalizeStack,
   config,
+  formatMixins,
   field
 }: {
   fieldOrDatumDef: FieldDef<string> | DatumDef<string>;
-  format: string | Dict<unknown>;
-  formatType: string;
+  formatMixins: FormatMixins<CustomFormatType>;
   expr?: 'datum' | 'parent' | 'datum.datum';
   normalizeStack?: boolean;
   config: Config;
@@ -167,10 +179,16 @@ export function formatCustomType({
   ) {
     const endField = vgField(fieldOrDatumDef, {expr, binSuffix: 'end'});
     return {
-      signal: binFormatExpression(field, endField, format, formatType, config)
+      signal: binFormatExpression(field, endField, formatMixins, config)
     };
   }
-  return {signal: customFormatExpr(formatType, field, format)};
+  const {format, formatType, formatInvalid} = formatMixins;
+  const signal = wrapFormatExprToHandleInvalidValues({
+    mainFormatExpr: customFormatExpr(formatType, field, format),
+    fieldExpr: field,
+    formatInvalid
+  });
+  return {signal};
 }
 
 export function guideFormat(
@@ -289,31 +307,61 @@ export function timeFormat({
   return omitTimeFormatConfig ? undefined : config.timeFormat;
 }
 
-function formatExpr(field: string, format: string) {
+function builtInFormatExpr(field: string, format: string) {
   return `format(${field}, "${format || ''}")`;
 }
 
-function binNumberFormatExpr(field: string, format: string | Dict<unknown>, formatType: string, config: Config) {
+/**
+ * return format expr for number bin's start/end
+ */
+function binExtentFormatExpr(field: string, format: string | Dict<unknown>, formatType: string, config: Config) {
   if (isCustomFormatType(formatType)) {
     return customFormatExpr(formatType, field, format);
   }
+  format = (isString(format) ? format : undefined) ?? config.numberFormat;
+  return builtInFormatExpr(field, format);
+}
 
-  return formatExpr(field, (isString(format) ? format : undefined) ?? config.numberFormat);
+function wrapFormatExprToHandleInvalidValues({
+  fieldExpr,
+  mainFormatExpr,
+  formatInvalid
+}: {
+  formatInvalid: FormatMixins['formatInvalid'];
+  mainFormatExpr: string;
+  fieldExpr: string;
+}): string {
+  if (!formatInvalid) {
+    return `${fieldValidPredicate(fieldExpr, false)} ? ${toStringExpr(fieldExpr)} : ${mainFormatExpr}`;
+  }
+  return mainFormatExpr;
 }
 
 export function binFormatExpression(
   startField: string,
   endField: string,
-  format: string | Dict<unknown>,
-  formatType: string,
+  {format, formatType, formatInvalid}: FormatMixins,
   config: Config
 ): string {
   if (format === undefined && formatType === undefined && config.customFormatTypes && config.numberFormatType) {
-    return binFormatExpression(startField, endField, config.numberFormat, config.numberFormatType, config);
+    return binFormatExpression(
+      startField,
+      endField,
+      {
+        format: config.numberFormat,
+        formatType: config.numberFormatType,
+        formatInvalid: config.normalizedNumberFormatInvalid
+      },
+      config
+    );
   }
-  const start = binNumberFormatExpr(startField, format, formatType, config);
-  const end = binNumberFormatExpr(endField, format, formatType, config);
-  return `${fieldValidPredicate(startField, false)} ? "null" : ${start} + "${BIN_RANGE_DELIMITER}" + ${end}`;
+  const start = binExtentFormatExpr(startField, format, formatType, config);
+  const end = binExtentFormatExpr(endField, format, formatType, config);
+  return wrapFormatExprToHandleInvalidValues({
+    formatInvalid,
+    fieldExpr: startField,
+    mainFormatExpr: `${start} + "${BIN_RANGE_DELIMITER}" + ${end}`
+  });
 }
 
 /**

src/compile/header/assemble.ts

@@ -122,7 +122,7 @@ export function assembleLabelTitle(
   channel: FacetChannel,
   config: Config<SignalRef>
 ) {
-  const {format, formatType, labelAngle, labelAnchor, labelOrient, labelExpr} = getHeaderProperties(
+  const {format, formatType, formatInvalid, labelAngle, labelAnchor, labelOrient, labelExpr} = getHeaderProperties(
     ['format', 'formatType', 'labelAngle', 'labelAnchor', 'labelOrient', 'labelExpr'],
     facetFieldDef.header,
     config,
@@ -131,8 +131,7 @@ export function assembleLabelTitle(
 
   const titleTextExpr = formatSignalRef({
     fieldOrDatumDef: facetFieldDef,
-    format,
-    formatType,
+    formatMixins: {format, formatType, formatInvalid},
     expr: 'parent',
     config
   }).signal;