feat: allow spreading one-to-many and many-to-many embedded resources

The selected columns in the embedded resources are aggregated into arrays

CHANGELOG.md

@@ -12,6 +12,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
  - #3607, Log to stderr when the JWT secret is less than 32 characters long - @laurenceisla
  - #2858, Performance improvements when calling RPCs via GET using indexes in more cases - @wolfgangwalther
  - #3560, Log resolved host in "Listening on ..." messages - @develop7
+ - #3041, Allow spreading one-to-many and many-to-many embedded resources - @laurenceisla
+ + The selected columns in the embedded resources are aggregated into arrays
 
 ### Fixed
 

src/PostgREST/ApiRequest/Types.hs

@@ -86,7 +86,6 @@ data ApiRequestError
  | PutLimitNotAllowedError
  | QueryParamError QPError
  | RelatedOrderNotToOne Text Text
- | SpreadNotToOne Text Text
  | UnacceptableFilter Text
  | UnacceptableSchema [Text]
  | UnsupportedMethod ByteString
@@ -145,7 +144,7 @@ type Cast = Text
 type Alias = Text
 type Hint = Text
 
-data AggregateFunction = Sum | Avg | Max | Min | Count
+data AggregateFunction = Sum | Avg | Max | Min | Count | ArrayAgg
  deriving (Show, Eq)
 
 data EmbedParam

src/PostgREST/Error.hs

@@ -80,7 +80,6 @@ instance PgrstError ApiRequestError where
  status PutLimitNotAllowedError = HTTP.status400
  status QueryParamError{} = HTTP.status400
  status RelatedOrderNotToOne{} = HTTP.status400
- status SpreadNotToOne{} = HTTP.status400
  status UnacceptableFilter{} = HTTP.status400
  status UnacceptableSchema{} = HTTP.status406
  status UnsupportedMethod{} = HTTP.status405
@@ -176,12 +175,6 @@ instance JSON.ToJSON ApiRequestError where
  (Just $ JSON.String $ "'" <> origin <> "' and '" <> target <> "' do not form a many-to-one or one-to-one relationship")
  Nothing
 
- toJSON (SpreadNotToOne origin target) = toJsonPgrstError
- ApiRequestErrorCode19
- ("A spread operation on '" <> target <> "' is not possible")
- (Just $ JSON.String $ "'" <> origin <> "' and '" <> target <> "' do not form a many-to-one or one-to-one relationship")
- Nothing
-
  toJSON (UnacceptableFilter target) = toJsonPgrstError
  ApiRequestErrorCode20
  ("Bad operator on the '" <> target <> "' embedded resource")
@@ -629,7 +622,7 @@ data ErrorCode
  | ApiRequestErrorCode16
  | ApiRequestErrorCode17
  | ApiRequestErrorCode18
- | ApiRequestErrorCode19
+ -- | ApiRequestErrorCode19 -- no longer used (used to be mapped to SpreadNotToOne)
  | ApiRequestErrorCode20
  | ApiRequestErrorCode21
  | ApiRequestErrorCode22
@@ -678,7 +671,6 @@ buildErrorCode code = case code of
  ApiRequestErrorCode16 -> "PGRST116"
  ApiRequestErrorCode17 -> "PGRST117"
  ApiRequestErrorCode18 -> "PGRST118"
- ApiRequestErrorCode19 -> "PGRST119"
  ApiRequestErrorCode20 -> "PGRST120"
  ApiRequestErrorCode21 -> "PGRST121"
  ApiRequestErrorCode22 -> "PGRST122"

src/PostgREST/Plan.hs

@@ -336,9 +336,9 @@ readPlan qi@QualifiedIdentifier{..} AppConfig{configDbMaxRows, configDbAggregate
  validateAggFunctions configDbAggregates =<<
  addRelSelects =<<
  addNullEmbedFilters =<<
- validateSpreadEmbeds =<<
  addRelatedOrders =<<
  addAliases =<<
+ addArrayAggToManySpread =<<
  expandStars ctx =<<
  addRels qiSchema (iAction apiRequest) dbRelationships Nothing =<<
  addLogicTrees ctx apiRequest =<<
@@ -352,7 +352,7 @@ initReadRequest ctx@ResolverContext{qi=QualifiedIdentifier{..}} =
  foldr (treeEntry rootDepth) $ Node defReadPlan{from=qi ctx, relName=qiName, depth=rootDepth} []
  where
  rootDepth = 0
- defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing False [] rootDepth
+ defReadPlan = ReadPlan [] (QualifiedIdentifier mempty mempty) Nothing [] [] allRange mempty Nothing [] Nothing mempty Nothing Nothing False False [] rootDepth
  treeEntry :: Depth -> Tree SelectItem -> ReadPlanTree -> ReadPlanTree
  treeEntry depth (Node si fldForest) (Node q rForest) =
  let nxtDepth = succ depth in
@@ -417,13 +417,14 @@ knownColumnsInContext ResolverContext{..} =
 -- | Expand "select *" into explicit field names of the table in the following situations:
 -- * When there are data representations present.
 -- * When there is an aggregate function in a given ReadPlan or its parent.
+-- * When the ReadPlan is a spread embed nested inside a to-many spread relationship (array aggregate).
 expandStars :: ResolverContext -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
 expandStars ctx rPlanTree = Right $ expandStarsForReadPlan False rPlanTree
  where
  expandStarsForReadPlan :: Bool -> ReadPlanTree -> ReadPlanTree
  expandStarsForReadPlan hasAgg (Node rp@ReadPlan{select, from=fromQI, fromAlias=alias} children) =
  let
- newHasAgg = hasAgg || any (isJust . csAggFunction) select
+ newHasAgg = hasAgg || any (isJust . csAggFunction) select || spreadRelIsNestedInToMany rp
  newCtx = adjustContext ctx fromQI alias
  newRPlan = expandStarsForTable newCtx newHasAgg rp
  in Node newRPlan (map (expandStarsForReadPlan newHasAgg) children)
@@ -474,18 +475,18 @@ treeRestrictRange maxRows _ request = pure $ nodeRestrictRange maxRows <$> reque
 addRels :: Schema -> Action -> RelationshipsMap -> Maybe ReadPlanTree -> ReadPlanTree -> Either ApiRequestError ReadPlanTree
 addRels schema action allRels parentNode (Node rPlan@ReadPlan{relName,relHint,relAlias,depth} forest) =
  case parentNode of
- Just (Node ReadPlan{from=parentNodeQi, fromAlias=parentAlias} _) ->
+ Just (Node pr@ReadPlan{from=parentNodeQi, fromAlias=parentAlias} _) ->
  let
  newReadPlan = (\r ->
  let newAlias = Just (qiName (relForeignTable r) <> "_" <> show depth)
  aggAlias = qiName (relTable r) <> "_" <> fromMaybe relName relAlias <> "_" <> show depth in
  case r of
  Relationship{relCardinality=M2M _} -> -- m2m does internal implicit joins that don't need aliasing
- rPlan{from=relForeignTable r, relToParent=Just r, relAggAlias=aggAlias, relJoinConds=getJoinConditions Nothing parentAlias r}
+ rPlan{from=relForeignTable r, relToParent=Just r, relAggAlias=aggAlias, relJoinConds=getJoinConditions Nothing parentAlias r, relIsInToManySpread=spreadRelIsNestedInToMany pr}
  ComputedRelationship{} ->
- rPlan{from=relForeignTable r, relToParent=Just r{relTableAlias=maybe (relTable r) (QualifiedIdentifier mempty) parentAlias}, relAggAlias=aggAlias, fromAlias=newAlias}
+ rPlan{from=relForeignTable r, relToParent=Just r{relTableAlias=maybe (relTable r) (QualifiedIdentifier mempty) parentAlias}, relAggAlias=aggAlias, fromAlias=newAlias, relIsInToManySpread=spreadRelIsNestedInToMany pr}
  _ ->
- rPlan{from=relForeignTable r, relToParent=Just r, relAggAlias=aggAlias, fromAlias=newAlias, relJoinConds=getJoinConditions newAlias parentAlias r}
+ rPlan{from=relForeignTable r, relToParent=Just r, relAggAlias=aggAlias, fromAlias=newAlias, relJoinConds=getJoinConditions newAlias parentAlias r, relIsInToManySpread=spreadRelIsNestedInToMany pr}
  ) <$> rel
  origin = if depth == 1 -- Only on depth 1 we check if the root(depth 0) has an alias so the sourceCTEName alias can be found as a relationship
  then fromMaybe (qiName parentNodeQi) parentAlias
@@ -509,6 +510,10 @@ addRels schema action allRels parentNode (Node rPlan@ReadPlan{relName,relHint,re
  updateForest :: Maybe ReadPlanTree -> Either ApiRequestError [ReadPlanTree]
  updateForest rq = addRels schema action allRels rq `traverse` forest
 
+spreadRelIsNestedInToMany :: ReadPlan -> Bool
+spreadRelIsNestedInToMany ReadPlan{relIsSpread, relToParent, relIsInToManySpread} =
+ relIsSpread && (relIsInToManySpread || Just False == (relIsToOne <$> relToParent))
+
 getJoinConditions :: Maybe Alias -> Maybe Alias -> Relationship -> [JoinCondition]
 getJoinConditions _ _ ComputedRelationship{} = []
 getJoinConditions tblAlias parentAlias Relationship{relTable=qi,relForeignTable=fQi,relCardinality=card} =
@@ -616,6 +621,22 @@ findRel schema allRels origin target hint =
  )
  ) $ fromMaybe mempty $ HM.lookup (QualifiedIdentifier schema origin, schema) allRels
 
+-- Add ArrayAgg aggregates to selected fields that do not have other aggregates and:
+-- * Are selected inside a to-many spread relationship
+-- * Are selected inside a to-one spread relationship but are nested inside a to-many spread relationship at any level
+addArrayAggToManySpread :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
+addArrayAggToManySpread (Node rp@ReadPlan{select} forest) =
+ let newForest = addArrayAggToManySpread `traverse` forest
+ newSelects
+ | shouldAddArrayAgg = fieldToArrayAgg <$> select
+ | otherwise = select
+ in Node rp { select = newSelects } <$> newForest
+ where
+ shouldAddArrayAgg = spreadRelIsNestedInToMany rp
+ fieldToArrayAgg field
+ | isJust $ csAggFunction field = field
+ | otherwise = field { csAggFunction = Just ArrayAgg, csAlias = newAlias (csAlias field) (cfName $ csField field) }
+ newAlias alias fieldName = maybe (Just fieldName) pure alias
 
 addRelSelects :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
 addRelSelects node@(Node rp forest)
@@ -628,11 +649,12 @@ addRelSelects node@(Node rp forest)
 generateRelSelectField :: ReadPlanTree -> Maybe RelSelectField
 generateRelSelectField (Node rp@ReadPlan{relToParent=Just _, relAggAlias, relIsSpread = True} _) =
  Just $ Spread { rsSpreadSel = generateSpreadSelectFields rp, rsAggAlias = relAggAlias }
-generateRelSelectField (Node ReadPlan{relToParent=Just rel, select, relName, relAlias, relAggAlias, relIsSpread = False} forest) =
+generateRelSelectField (Node ReadPlan{relToParent=Just rel, select, relName, relAlias, relAggAlias, relIsSpread = False, relIsInToManySpread} forest) =
  Just $ JsonEmbed { rsEmbedMode, rsSelName, rsAggAlias = relAggAlias, rsEmptyEmbed }
  where
  rsSelName = fromMaybe relName relAlias
- rsEmbedMode = if relIsToOne rel then JsonObject else JsonArray
+ -- If the JsonEmbed is nested in a to-many spread relationship, it will be aggregated at the top. That's why we treat it as `JsonObject`.
+ rsEmbedMode = if relIsToOne rel || relIsInToManySpread then JsonObject else JsonArray
  rsEmptyEmbed = hasOnlyNullEmbed (null select) forest
  hasOnlyNullEmbed = foldr checkIfNullEmbed
  checkIfNullEmbed :: ReadPlanTree -> Bool -> Bool
@@ -641,7 +663,7 @@ generateRelSelectField (Node ReadPlan{relToParent=Just rel, select, relName, rel
 generateRelSelectField _ = Nothing
 
 generateSpreadSelectFields :: ReadPlan -> [SpreadSelectField]
-generateSpreadSelectFields ReadPlan{select, relSelect} =
+generateSpreadSelectFields rp@ReadPlan{select, relSelect} =
  -- We combine the select and relSelect fields into a single list of SpreadSelectField.
  selectSpread ++ relSelectSpread
  where
@@ -653,7 +675,9 @@ generateSpreadSelectFields ReadPlan{select, relSelect} =
  relSelectSpread = concatMap relSelectToSpread relSelect
  relSelectToSpread :: RelSelectField -> [SpreadSelectField]
  relSelectToSpread (JsonEmbed{rsSelName}) =
- [SpreadSelectField { ssSelName = rsSelName, ssSelAggFunction = Nothing, ssSelAggCast = Nothing, ssSelAlias = Nothing }]
+ -- The regular embeds that are nested inside spread to-many relationships are also aggregated in an array
+ let (aggFun, alias) = if spreadRelIsNestedInToMany rp then (Just ArrayAgg, Just rsSelName) else (Nothing, Nothing) in
+ [SpreadSelectField { ssSelName = rsSelName, ssSelAggFunction = aggFun, ssSelAggCast = Nothing, ssSelAlias = alias }]
  relSelectToSpread (Spread{rsSpreadSel}) =
  rsSpreadSel
 
@@ -815,7 +839,7 @@ addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
 -- relName = "projects",
 -- relToParent = Nothing,
 -- relJoinConds = [],
--- relAlias = Nothing, relAggAlias = "clients_projects_1", relHint = Nothing, relJoinType = Nothing, relIsSpread = False, depth = 1,
+-- relAlias = Nothing, relAggAlias = "clients_projects_1", relHint = Nothing, relJoinType = Nothing, relIsSpread = False, relIsInToManySpread = False, depth = 1,
 -- relSelect = []
 -- },
 -- subForest = []
@@ -841,7 +865,7 @@ addRelatedOrders (Node rp@ReadPlan{order,from} forest) = do
 -- )
 -- ],
 -- order = [], range_ = fullRange, relName = "clients", relToParent = Nothing, relJoinConds = [], relAlias = Nothing, relAggAlias = "", relHint = Nothing,
--- relJoinType = Nothing, relIsSpread = False, depth = 0,
+-- relJoinType = Nothing, relIsSpread = False, relIsInToManySpread = False, depth = 0,
 -- relSelect = []
 -- },
 -- subForest = subForst
@@ -906,15 +930,6 @@ resolveLogicTree ctx (Expr b op lts) = CoercibleExpr b op (map (resolveLogicTree
 resolveFilter :: ResolverContext -> Filter -> CoercibleFilter
 resolveFilter ctx (Filter fld opExpr) = CoercibleFilter{field=resolveQueryInputField ctx fld, opExpr=opExpr}
 
--- Validates that spread embeds are only done on to-one relationships
-validateSpreadEmbeds :: ReadPlanTree -> Either ApiRequestError ReadPlanTree
-validateSpreadEmbeds (Node rp@ReadPlan{relToParent=Nothing} forest) = Node rp <$> validateSpreadEmbeds `traverse` forest
-validateSpreadEmbeds (Node rp@ReadPlan{relIsSpread,relToParent=Just rel,relName} forest) = do
- validRP <- if relIsSpread && not (relIsToOne rel)
- then Left $ SpreadNotToOne (qiName $ relTable rel) relName -- TODO using relTable is not entirely right because ReadPlan might have an alias, need to store the parent alias on ReadPlan
- else Right rp
- Node validRP <$> validateSpreadEmbeds `traverse` forest
-
 -- Find a Node of the Tree and apply a function to it
 updateNode :: (a -> ReadPlanTree -> ReadPlanTree) -> (EmbedPath, a) -> Either ApiRequestError ReadPlanTree -> Either ApiRequestError ReadPlanTree
 updateNode f ([], a) rr = f a <$> rr

src/PostgREST/Plan/ReadPlan.hs

@@ -29,22 +29,23 @@ data JoinCondition =
  deriving (Eq, Show)
 
 data ReadPlan = ReadPlan
- { select :: [CoercibleSelectField]
- , from :: QualifiedIdentifier
- , fromAlias :: Maybe Alias
- , where_ :: [CoercibleLogicTree]
- , order :: [CoercibleOrderTerm]
- , range_ :: NonnegRange
- , relName :: NodeName
- , relToParent :: Maybe Relationship
- , relJoinConds :: [JoinCondition]
- , relAlias :: Maybe Alias
- , relAggAlias :: Alias
- , relHint :: Maybe Hint
- , relJoinType :: Maybe JoinType
- , relIsSpread :: Bool
- , relSelect :: [RelSelectField]
- , depth :: Depth
+ { select :: [CoercibleSelectField]
+ , from :: QualifiedIdentifier
+ , fromAlias :: Maybe Alias
+ , where_ :: [CoercibleLogicTree]
+ , order :: [CoercibleOrderTerm]
+ , range_ :: NonnegRange
+ , relName :: NodeName
+ , relToParent :: Maybe Relationship
+ , relJoinConds :: [JoinCondition]
+ , relAlias :: Maybe Alias
+ , relAggAlias :: Alias
+ , relHint :: Maybe Hint
+ , relJoinType :: Maybe JoinType
+ , relIsSpread :: Bool
+ , relIsInToManySpread :: Bool
+ , relSelect :: [RelSelectField]
+ , depth :: Depth
  -- ^ used for aliasing
  }
  deriving (Eq, Show)

src/PostgREST/Query/SqlFragment.hs

@@ -252,7 +252,7 @@ pgFmtCallUnary :: Text -> SQL.Snippet -> SQL.Snippet
 pgFmtCallUnary f x = SQL.sql (encodeUtf8 f) <> "(" <> x <> ")"
 
 pgFmtField :: QualifiedIdentifier -> CoercibleField -> SQL.Snippet
-pgFmtField table CoercibleField{cfFullRow=True} = fromQi table
+pgFmtField table CoercibleField{cfFullRow=True} = pgFmtIdent (qiName table)
 pgFmtField table CoercibleField{cfName=fn, cfJsonPath=[]} = pgFmtColumn table fn
 pgFmtField table CoercibleField{cfName=fn, cfToJson=doToJson, cfJsonPath=jp} | doToJson = "to_jsonb(" <> pgFmtColumn table fn <> ")" <> pgFmtJsonPath jp
  | otherwise = pgFmtColumn table fn <> pgFmtJsonPath jp
@@ -284,10 +284,12 @@ pgFmtApplyAggregate Nothing _ snippet = snippet
 pgFmtApplyAggregate (Just agg) aggCast snippet =
  pgFmtApplyCast aggCast aggregatedSnippet
  where
- convertAggFunction :: AggregateFunction -> SQL.Snippet
- -- Convert from e.g. Sum (the data type) to SUM
  convertAggFunction = SQL.sql . BS.map toUpper . BS.pack . show
- aggregatedSnippet = convertAggFunction agg <> "(" <> snippet <> ")"
+ aggregatedSnippet = case agg of
+ -- TODO: NULLIF(...,'{null}') does not take into consideration a case with a single element with a null value.
+ -- See https://github.com/PostgREST/postgrest/pull/3640#issuecomment-2334996466
+ ArrayAgg -> "COALESCE(NULLIF(array_agg(" <> snippet <> "),'{null}'),'{}')"
+ a -> convertAggFunction a <> "(" <> snippet <> ")"
 
 pgFmtApplyCast :: Maybe Cast -> SQL.Snippet -> SQL.Snippet
 pgFmtApplyCast Nothing snippet = snippet