From 81a8095bfb1fbab4b7768d26eacefbc7f1007510 Mon Sep 17 00:00:00 2001 From: Sinclair Chen Date: Wed, 26 Jun 2024 13:56:43 -0700 Subject: [PATCH] simplify weekly portfolio update --- .../src/scheduled/weekly-portfolio-updates.ts | 177 ++++++------------ backend/shared/src/create-notification.ts | 12 +- 2 files changed, 60 insertions(+), 129 deletions(-) diff --git a/backend/functions/src/scheduled/weekly-portfolio-updates.ts b/backend/functions/src/scheduled/weekly-portfolio-updates.ts index 6f25a91d77..4457c87a53 100644 --- a/backend/functions/src/scheduled/weekly-portfolio-updates.ts +++ b/backend/functions/src/scheduled/weekly-portfolio-updates.ts @@ -1,32 +1,26 @@ import * as functions from 'firebase-functions' import { sum } from 'lodash' - import { getUsersContractMetricsOrderedByProfit } from 'common/supabase/contract-metrics' import { createWeeklyPortfolioUpdateNotification } from 'shared/create-notification' import { createSupabaseClient, createSupabaseDirectClient, - SupabaseClient, } from 'shared/supabase/init' -import { getUser, getUsers, log } from 'shared/utils' +import { log } from 'shared/utils' import { secrets } from 'common/secrets' import { bulkInsert } from 'shared/supabase/utils' -import { APIError } from 'common/api/utils' - import * as dayjs from 'dayjs' import { Row } from 'common/supabase/utils' import { ContractMetric } from 'common/contract-metric' -import { convertPrivateUser } from 'common/supabase/users' const now = new Date() -const time = now.getTime() const getDate = () => dayjs(now).format('YYYY-MM-DD') const USERS_TO_SAVE = 300 // Saving metrics should work until our users are greater than USERS_TO_SAVE * 2*60 users export const saveWeeklyContractMetrics = functions - .runWith({ memory: '4GB', secrets, timeoutSeconds: 60 }) + .runWith({ secrets, timeoutSeconds: 60 }) // every minute for 2 hours Friday 4am PT (UTC -08:00) .pubsub.schedule('* 13-14 * * 5') .timeZone('Etc/UTC') @@ -35,7 +29,7 @@ export const saveWeeklyContractMetrics = functions }) export const sendWeeklyPortfolioUpdate = functions - .runWith({ memory: '8GB', secrets, timeoutSeconds: 540 }) + .runWith({ secrets, timeoutSeconds: 540 }) // every Friday at 12pm PT (UTC -08:00) .pubsub.schedule('0 20 * * 5') .timeZone('Etc/UTC') @@ -44,53 +38,27 @@ export const sendWeeklyPortfolioUpdate = functions }) export const saveWeeklyContractMetricsInternal = async () => { + const pg = createSupabaseDirectClient() const db = createSupabaseClient() // users who have disabled browser notifications for profit/loss updates won't be able to see their portfolio updates in the past - const privateUsersQuery = await db - .from('private_users') - .select('id') - .contains( - `data->'notificationPreferences'->'profit_loss_updates'`, - 'browser' - ) - - if (privateUsersQuery.error) { - throw new APIError( - 500, - 'Error getting private users: ', - privateUsersQuery.error + const userIds = await pg.map( + `select p.id from private_users p + where p.data->'notificationPreferences'->'profit_loss_updates' ? 'browser' + and not exists ( + select 1 from weekly_update w where w.user_id = p.id and w.range_end = $1 ) - } - const privateUsers = privateUsersQuery.data - - const alreadyUpdatedQuery = await db - .from('weekly_update') - .select('user_id') - .eq('range_end', getDate()) - - if (alreadyUpdatedQuery.error) { - throw new APIError( - 500, - 'Error getting already updated users: ', - alreadyUpdatedQuery.error - ) - } - - const alreadyUpdated = alreadyUpdatedQuery.data.map((r) => r.user_id) - - log('already updated users', alreadyUpdated.length, 'at', time) - // filter out the users who have already had their weekly update saved - const usersToSave = privateUsers - .filter((user) => !alreadyUpdated.includes(user.id)) - .slice(0, USERS_TO_SAVE) + limit $2`, + [getDate(), USERS_TO_SAVE], + (data) => data.id + ) - log('usersToSave', usersToSave.length) - if (usersToSave.length === 0) return + log('usersToSave', userIds.length) + if (userIds.length === 0) return // TODO: try out the new rpc call const usersToContractMetrics = await getUsersContractMetricsOrderedByProfit( - usersToSave.map((u) => u.id), + userIds, db, 'week' ) @@ -99,96 +67,63 @@ export const saveWeeklyContractMetricsInternal = async () => { return } - const results = await Promise.all( - usersToSave.map(async (privateUser) => { - const user = await getUser(privateUser.id) - const contractMetrics = usersToContractMetrics[privateUser.id] - return { - contract_metrics: contractMetrics, - user_id: privateUser.id, - profit: - user?.profitCached.weekly ?? - sum(contractMetrics.map((m) => m.from?.week.profit ?? 0)), - range_end: getDate(), - } - }) + const userProfits = await pg.manyOrNone<{ + id: string + profit: number | undefined + }>( + `select id, (data->'profitCached'->'weekly')::numeric as profit + from users + where id = any($1)`, + [userIds] ) - const pg = createSupabaseDirectClient() + const results = userIds.map((id) => { + const profit = userProfits.find((u) => u.id === id)?.profit + const contractMetrics = usersToContractMetrics[id] + return { + contract_metrics: contractMetrics, + user_id: id, + profit: + profit ?? sum(contractMetrics.map((m) => m.from?.week.profit ?? 0)), + range_end: getDate(), + } + }) + await bulkInsert(pg, 'weekly_update', results) - log('saved weekly contract metrics for users:', usersToSave.length) + log('saved weekly contract metrics for users:', userIds.length) } export const sendWeeklyPortfolioUpdateNotifications = async () => { - const db = createSupabaseClient() - - // get all users who have opted in to weekly portfolio updates - const privateUsersQuery = await db - .from('private_users') - .select() - .contains( - `data->'notificationPreferences'->'profit_loss_updates'`, - 'browser' - ) + const pg = createSupabaseDirectClient() - if (privateUsersQuery.error) { - throw new APIError( - 500, - 'Error getting private users: ', - privateUsersQuery.error - ) - } - const privateUsers = privateUsersQuery.data.map(convertPrivateUser) + const now = getDate() - const userData = await getUsers(privateUsers.map((u) => u.id)) - const usernameById = Object.fromEntries( - userData.map((u) => [u.id, u.username]) + const data = await pg.manyOrNone< + Pick, 'id'> & + Pick, 'profit' | 'range_end' | 'contract_metrics'> & + Pick, 'username'> + >( + `select p.id, w.profit, w.range_end, w.contract_metrics, u.username + from private_users p join weekly_update w on w.user_id = p.id join users u on u.id = p.id + where p.data->'notificationPreferences'->'profit_loss_updates' ? 'browser' + and w.range_end = $1`, + [now] ) - log('users to send weekly portfolio updates to', privateUsers.length) - let count = 0 - const now = getDate() + + log('users to send weekly portfolio updates to', data.length) + await Promise.all( - privateUsers.map(async (privateUser) => { - const data = await getUsersWeeklyUpdate(db, privateUser.id, now) - if (!data) return - const { profit, range_end, contract_metrics } = data + data.map(async ({ id, profit, range_end, contract_metrics, username }) => { const contractMetrics = contract_metrics as ContractMetric[] - // Don't send update if there are no contracts - count++ - if (count % 100 === 0) - log('sent weekly portfolio updates to', count, '/', privateUsers.length) if (contractMetrics.length === 0) return + await createWeeklyPortfolioUpdateNotification( - privateUser, - usernameById[privateUser.id], + id, + username, profit, range_end ) }) ) } - -const getUsersWeeklyUpdate = async ( - db: SupabaseClient, - userId: string, - rangeEnd: string -) => { - const { data, error } = await db - .from('weekly_update') - .select('*') - .eq('user_id', userId) - .eq('range_end', rangeEnd) - .order('created_time', { ascending: false }) - .limit(1) - - if (error) { - console.error(error) - return - } - if (!data.length) { - return - } - - return data[0] as Row<'weekly_update'> -} diff --git a/backend/shared/src/create-notification.ts b/backend/shared/src/create-notification.ts index 49f5fb5c9d..fc8e55c745 100644 --- a/backend/shared/src/create-notification.ts +++ b/backend/shared/src/create-notification.ts @@ -1512,23 +1512,19 @@ export const createMarketClosedNotification = async ( contract ) } + +// assumes user has notification for this enabled export const createWeeklyPortfolioUpdateNotification = async ( - privateUser: PrivateUser, + userId: string, userUsername: string, weeklyProfit: number, rangeEndDateSlug: string ) => { - const { sendToBrowser } = getNotificationDestinationsForUser( - privateUser, - 'profit_loss_updates' - ) - if (!sendToBrowser) return - const id = rangeEndDateSlug + 'weekly_portfolio_update' const notification: Notification = { id, - userId: privateUser.id, + userId: userId, reason: 'profit_loss_updates', createdTime: Date.now(), isSeen: false,