DataFlow backfill - disabled but most of the functionality is there

src/api/draftSpecs.ts

@@ -42,7 +42,8 @@ export const createDraftSpec = (
  catalogName: string,
  draftSpec: any,
  specType?: Entity | null,
- lastPubId?: string | null
+ lastPubId?: string | null,
+ noResponse?: boolean
 ) => {
  let matchData: CreateMatchData = {
  draft_id: draftId,
@@ -55,7 +56,7 @@ export const createDraftSpec = (
  matchData = { ...matchData, expect_pub_id: lastPubId };
  }
 
- return insertSupabase(TABLES.DRAFT_SPECS, matchData);
+ return insertSupabase(TABLES.DRAFT_SPECS, matchData, noResponse);
 };
 
 export const modifyDraftSpec = (

src/api/liveSpecsExt.ts

@@ -206,7 +206,6 @@ export interface LiveSpecsExtQuery_DetailsForm {
  spec_type: Entity;
  spec: any;
  data_plane_id: string;
- detail: string | null;
  connector_tag_id: string;
  connector_image_name: string;
  connector_image_tag: string;
@@ -219,7 +218,6 @@ const DETAILS_FORM_QUERY = `
  spec_type,
  spec,
  data_plane_id,
- detail,
  connector_tag_id,
  connector_image_name,
  connector_image_tag,
@@ -372,6 +370,14 @@ const getLiveSpecsByLiveSpecId = async (liveSpecId: string) => {
  return data;
 };
 
+const getLiveSpecSpec = (liveSpecId: string) => {
+ return supabaseClient
+ .from(TABLES.LIVE_SPECS_EXT)
+ .select(`spec`)
+ .eq('id', liveSpecId)
+ .single();
+};
+
 const getLiveSpecShards = (tenant: string, entityType: Entity) => {
  return supabaseClient
  .from(TABLES.LIVE_SPECS_EXT)
@@ -380,6 +386,44 @@ const getLiveSpecShards = (tenant: string, entityType: Entity) => {
  .eq('spec_type', entityType);
 };
 
+const liveSpecsExtRelatedColumns = ['catalog_name', 'reads_from', 'id'];
+export const liveSpecsExtRelatedQuery = liveSpecsExtRelatedColumns.join(',');
+export interface LiveSpecsExt_Related {
+ catalog_name: string;
+ reads_from: string[];
+ id: string;
+}
+// const getLiveSpecsRelatedToMaterialization = async (
+// collectionNames: string[]
+// ) => {
+// const limiter = pLimit(3);
+// const promises = [];
+// let index = 0;
+
+// // TODO (retry) promise generator
+// const promiseGenerator = (idx: number) => {
+// return supabaseClient
+// .from(TABLES.LIVE_SPECS_EXT)
+// .select(liveSpecsExtRelatedQuery)
+// .eq('spec_type', 'materialization')
+// .overlaps(
+// 'reads_from',
+// collectionNames.slice(idx, idx + CHUNK_SIZE)
+// )
+// .returns<LiveSpecsExt_Related[]>();
+// };
+
+// while (index < collectionNames.length) {
+// const prom = promiseGenerator(index);
+// promises.push(limiter(() => prom));
+// index = index + CHUNK_SIZE;
+// }
+
+// const response = await Promise.all(promises);
+// const errors = response.filter((r) => r.error);
+// return errors[0] ?? response[0];
+// };
+
 export {
  getLiveSpecs_captures,
  getLiveSpecs_collections,
@@ -392,4 +436,5 @@ export {
  getLiveSpecsByConnectorId,
  getLiveSpecsByLiveSpecId,
  getLiveSpecShards,
+ getLiveSpecSpec,
 };

src/components/editor/Bindings/Backfill/index.tsx

@@ -10,8 +10,8 @@ import {
  useBinding_currentCollection,
  useBinding_currentBindingUUID,
  useBinding_setBackfilledBindings,
- useBinding_collections_count,
  useBinding_backfillSupported,
+ useBinding_enabledCollections_count,
 } from 'stores/Binding/hooks';
 import {
  useFormStateStore_isActive,
@@ -36,7 +36,7 @@ function Backfill({ description, bindingIndex = -1 }: BackfillProps) {
  // Binding Store
  const currentCollection = useBinding_currentCollection();
  const currentBindingUUID = useBinding_currentBindingUUID();
- const collectionsCount = useBinding_collections_count();
+ const collectionsCount = useBinding_enabledCollections_count();
  const allBindingsDisabled = useBinding_allBindingsDisabled();
 
  const backfillAllBindings = useBinding_backfillAllBindings();
@@ -199,10 +199,11 @@ function Backfill({ description, bindingIndex = -1 }: BackfillProps) {
  ) : null}
  </Stack>
 
- {/*TODO (data flow reset)*/}
- {/* {bindingIndex === -1 && workflow === 'capture_edit' ? (
+ {/*TODO (data flow reset)
+ {bindingIndex === -1 && workflow === 'capture_edit' ? (
  <BackfillDataFlowOption />
- ) : null}*/}
+ ) : null}
+ */}
  </Box>
  );
 }

src/components/editor/Bindings/index.tsx

@@ -110,11 +110,9 @@ function BindingsMultiEditor({
  {workflow === 'capture_edit' ||
  workflow === 'materialization_edit' ? (
  <Backfill
- description={
- <FormattedMessage
- id={`workflows.collectionSelector.manualBackfill.message.${entityType}.allBindings`}
- />
- }
+ description={intl.formatMessage({
+ id: `workflows.collectionSelector.manualBackfill.message.${entityType}.allBindings`,
+ })}
  />
  ) : null}
  </Stack>

src/components/editor/Shards/Disable/Warning.tsx

@@ -3,50 +3,61 @@ import { FormattedMessage } from 'react-intl';
 import { useEntityType } from 'context/EntityContext';
 
 import AlertBox from 'components/shared/AlertBox';
+import useGlobalSearchParams, {
+ GlobalSearchParams,
+} from 'hooks/searchParams/useGlobalSearchParams';
 
 function ShardsDisableWarning() {
+ const forcedToEnable = useGlobalSearchParams<number>(
+ GlobalSearchParams.FORCED_SHARD_ENABLE
+ );
+
  const entityType = useEntityType();
 
- return (
- <Box
- sx={{
- 'mb': 2,
- '& .MuiAlertTitle-root': {
- textTransform: 'capitalize',
- },
- }}
- >
- <AlertBox
- short
- severity="warning"
- title={
- <FormattedMessage
- id="workflows.disable.autoEnabledAlert.title"
- values={{
- entityType,
- }}
- />
- }
+ if (forcedToEnable === 0) {
+ return (
+ <Box
+ sx={{
+ 'mb': 2,
+ '& .MuiAlertTitle-root': {
+ textTransform: 'capitalize',
+ },
+ }}
  >
- <Box>
- <FormattedMessage
- id="workflows.disable.autoEnabledAlert.message"
- values={{
- entityType,
- }}
- />
- </Box>
- <Box>
- <FormattedMessage
- id="workflows.disable.autoEnabledAlert.instructions"
- values={{
- entityType,
- }}
- />
- </Box>
- </AlertBox>
- </Box>
- );
+ <AlertBox
+ short
+ severity="warning"
+ title={
+ <FormattedMessage
+ id="workflows.disable.autoEnabledAlert.title"
+ values={{
+ entityType,
+ }}
+ />
+ }
+ >
+ <Box>
+ <FormattedMessage
+ id="workflows.disable.autoEnabledAlert.message"
+ values={{
+ entityType,
+ }}
+ />
+ </Box>
+ <Box>
+ <FormattedMessage
+ id="workflows.disable.autoEnabledAlert.instructions"
+ values={{
+ entityType,
+ }}
+ />
+ </Box>
+ </AlertBox>
+ </Box>
+ );
+ }
+
+ return null;
 }
 
 export default ShardsDisableWarning;

src/components/editor/Shards/Disable/index.tsx

@@ -2,17 +2,11 @@ import { Stack, Typography } from '@mui/material';
 import { FormattedMessage } from 'react-intl';
 import { useEntityType } from 'context/EntityContext';
 import { useEditorStore_persistedDraftId } from 'components/editor/Store/hooks';
-import useGlobalSearchParams, {
- GlobalSearchParams,
-} from 'hooks/searchParams/useGlobalSearchParams';
+
 import ShardsDisableForm from './Form';
 import ShardsDisableWarning from './Warning';
 
 function ShardsDisable() {
- const forcedToEnable = useGlobalSearchParams(
- GlobalSearchParams.FORCED_SHARD_ENABLE
- );
-
  const entityType = useEntityType();
 
  const draftId = useEditorStore_persistedDraftId();
@@ -23,7 +17,7 @@ function ShardsDisable() {
 
  return (
  <>
- {forcedToEnable ? <ShardsDisableWarning /> : null}
+ <ShardsDisableWarning />
 
  <Stack spacing={1} sx={{ mb: 2 }}>
  <Typography variant="formSectionHeader">

src/components/home/dashboard/EntityStatOverview/StatOverview.tsx

@@ -1,15 +1,10 @@
 import { Stack, Typography } from '@mui/material';
 import { PostgrestError } from '@supabase/postgrest-js';
-import { authenticatedRoutes } from 'app/routes';
 import LinkWrapper from 'components/shared/LinkWrapper';
-import {
- semiTransparentBackground_blue,
- semiTransparentBackground_purple,
- semiTransparentBackground_teal,
-} from 'context/Theme';
-import { CloudDownload, CloudUpload, DatabaseScript } from 'iconoir-react';
+
 import { FormattedMessage, useIntl } from 'react-intl';
 import { Entity } from 'types';
+import { ENTITY_SETTINGS } from 'utils/entity-utils';
 import ActiveEntityCount from './ActiveEntityCount';
 import Statistic from './Statistic';
 
@@ -21,56 +16,6 @@ interface Props {
  monthlyUsageLoading?: boolean;
 }
 
-const getEntityPageURLPath = (entityType: string): string => {
- if (entityType === 'collection') {
- return authenticatedRoutes.collections.fullPath;
- }
-
- if (entityType === 'capture') {
- return authenticatedRoutes.captures.fullPath;
- }
-
- return authenticatedRoutes.materializations.fullPath;
-};
-
-const getTitleId = (entityType: string): string => {
- if (entityType === 'collection') {
- return 'terms.collections';
- }
-
- if (entityType === 'capture') {
- return 'terms.sources';
- }
-
- return 'terms.destinations';
-};
-
-const getBackgroundColor = (
- entityType: string
-): { dark: string; light: string } => {
- if (entityType === 'collection') {
- return semiTransparentBackground_blue;
- }
-
- if (entityType === 'capture') {
- return semiTransparentBackground_teal;
- }
-
- return semiTransparentBackground_purple;
-};
-
-const getEntityIcon = (entityType: string) => {
- if (entityType === 'collection') {
- return DatabaseScript;
- }
-
- if (entityType === 'capture') {
- return CloudUpload;
- }
-
- return CloudDownload;
-};
-
 export default function StatOverview({
  entityType,
  monthlyUsage,
@@ -80,11 +25,12 @@ export default function StatOverview({
 }: Props) {
  const intl = useIntl();
 
- const route = getEntityPageURLPath(entityType);
- const titleId = getTitleId(entityType);
-
- const background = getBackgroundColor(entityType);
- const Icon = getEntityIcon(entityType);
+ const {
+ routes: { viewAll },
+ termId: titleId,
+ background,
+ Icon,
+ } = ENTITY_SETTINGS[entityType];
 
  return (
  <Stack
@@ -117,7 +63,7 @@ export default function StatOverview({
  </Typography>
  </Stack>
 
- <LinkWrapper link={route}>
+ <LinkWrapper link={viewAll}>
  <FormattedMessage id="cta.goToAll" />
  </LinkWrapper>
  </Stack>