DataFlow backfill - disabled but most of the functionality is there

src/api/draftSpecs.ts

@@ -42,7 +42,8 @@ export const createDraftSpec = (
     catalogName: string,
     draftSpec: any,
     specType?: Entity | null,
-    lastPubId?: string | null
+    lastPubId?: string | null,
+    noResponse?: boolean
 ) => {
     let matchData: CreateMatchData = {
         draft_id: draftId,
@@ -55,7 +56,7 @@ export const createDraftSpec = (
         matchData = { ...matchData, expect_pub_id: lastPubId };
     }
 
-    return insertSupabase(TABLES.DRAFT_SPECS, matchData);
+    return insertSupabase(TABLES.DRAFT_SPECS, matchData, noResponse);
 };
 
 export const modifyDraftSpec = (

src/api/liveSpecsExt.ts

@@ -206,7 +206,6 @@ export interface LiveSpecsExtQuery_DetailsForm {
     spec_type: Entity;
     spec: any;
     data_plane_id: string;
-    detail: string | null;
     connector_tag_id: string;
     connector_image_name: string;
     connector_image_tag: string;
@@ -219,7 +218,6 @@ const DETAILS_FORM_QUERY = `
     spec_type,
     spec,
     data_plane_id,
-    detail,
     connector_tag_id,
     connector_image_name,
     connector_image_tag,
@@ -372,6 +370,14 @@ const getLiveSpecsByLiveSpecId = async (liveSpecId: string) => {
     return data;
 };
 
+const getLiveSpecSpec = (liveSpecId: string) => {
+    return supabaseClient
+        .from(TABLES.LIVE_SPECS_EXT)
+        .select(`spec`)
+        .eq('id', liveSpecId)
+        .single();
+};
+
 const getLiveSpecShards = (tenant: string, entityType: Entity) => {
     return supabaseClient
         .from(TABLES.LIVE_SPECS_EXT)
@@ -380,6 +386,44 @@ const getLiveSpecShards = (tenant: string, entityType: Entity) => {
         .eq('spec_type', entityType);
 };
 
+const liveSpecsExtRelatedColumns = ['catalog_name', 'reads_from', 'id'];
+export const liveSpecsExtRelatedQuery = liveSpecsExtRelatedColumns.join(',');
+export interface LiveSpecsExt_Related {
+    catalog_name: string;
+    reads_from: string[];
+    id: string;
+}
+// const getLiveSpecsRelatedToMaterialization = async (
+//     collectionNames: string[]
+// ) => {
+//     const limiter = pLimit(3);
+//     const promises = [];
+//     let index = 0;
+
+//     // TODO (retry) promise generator
+//     const promiseGenerator = (idx: number) => {
+//         return supabaseClient
+//             .from(TABLES.LIVE_SPECS_EXT)
+//             .select(liveSpecsExtRelatedQuery)
+//             .eq('spec_type', 'materialization')
+//             .overlaps(
+//                 'reads_from',
+//                 collectionNames.slice(idx, idx + CHUNK_SIZE)
+//             )
+//             .returns<LiveSpecsExt_Related[]>();
+//     };
+
+//     while (index < collectionNames.length) {
+//         const prom = promiseGenerator(index);
+//         promises.push(limiter(() => prom));
+//         index = index + CHUNK_SIZE;
+//     }
+
+//     const response = await Promise.all(promises);
+//     const errors = response.filter((r) => r.error);
+//     return errors[0] ?? response[0];
+// };
+
 export {
     getLiveSpecs_captures,
     getLiveSpecs_collections,
@@ -392,4 +436,5 @@ export {
     getLiveSpecsByConnectorId,
     getLiveSpecsByLiveSpecId,
     getLiveSpecShards,
+    getLiveSpecSpec,
 };

src/components/editor/Bindings/Backfill/BackfillCount.tsx

@@ -4,7 +4,7 @@ import { useMemo } from 'react';
 import { useIntl } from 'react-intl';
 import {
     useBinding_backfilledBindings_count,
-    useBinding_collections_count,
+    useBinding_enabledCollections_count,
 } from 'stores/Binding/hooks';
 import { BackfillCountProps } from './types';
 
@@ -13,7 +13,7 @@ function BackfillCount({ disabled }: BackfillCountProps) {
     const entityType = useEntityType();
 
     const backfillCount = useBinding_backfilledBindings_count();
-    const bindingsTotal = useBinding_collections_count();
+    const bindingsTotal = useBinding_enabledCollections_count();
 
     // Only reason noBackfill is in here is because we are already running the memo on backfillCount change
     const [noBackfill, itemType_backfill, itemType_bindings] = useMemo(() => {

src/components/editor/Bindings/Backfill/index.tsx

@@ -10,8 +10,8 @@ import {
     useBinding_currentCollection,
     useBinding_currentBindingUUID,
     useBinding_setBackfilledBindings,
-    useBinding_collections_count,
     useBinding_backfillSupported,
+    useBinding_enabledCollections_count,
 } from 'stores/Binding/hooks';
 import {
     useFormStateStore_isActive,
@@ -36,7 +36,7 @@ function Backfill({ description, bindingIndex = -1 }: BackfillProps) {
     // Binding Store
     const currentCollection = useBinding_currentCollection();
     const currentBindingUUID = useBinding_currentBindingUUID();
-    const collectionsCount = useBinding_collections_count();
+    const collectionsCount = useBinding_enabledCollections_count();
     const allBindingsDisabled = useBinding_allBindingsDisabled();
 
     const backfillAllBindings = useBinding_backfillAllBindings();
@@ -199,10 +199,11 @@ function Backfill({ description, bindingIndex = -1 }: BackfillProps) {
                 ) : null}
             </Stack>
 
-            {/*TODO (data flow reset)*/}
-            {/*            {bindingIndex === -1 && workflow === 'capture_edit' ? (
+            {/*TODO (data flow reset)
+            {bindingIndex === -1 && workflow === 'capture_edit' ? (
                 <BackfillDataFlowOption />
-            ) : null}*/}
+            ) : null}
+            */}
         </Box>
     );
 }

src/components/editor/Shards/Disable/Warning.tsx

@@ -3,50 +3,61 @@ import { FormattedMessage } from 'react-intl';
 import { useEntityType } from 'context/EntityContext';
 
 import AlertBox from 'components/shared/AlertBox';
+import useGlobalSearchParams, {
+    GlobalSearchParams,
+} from 'hooks/searchParams/useGlobalSearchParams';
 
 function ShardsDisableWarning() {
+    const forcedToEnable = useGlobalSearchParams<number>(
+        GlobalSearchParams.FORCED_SHARD_ENABLE
+    );
+
     const entityType = useEntityType();
 
-    return (
-        <Box
-            sx={{
-                'mb': 2,
-                '& .MuiAlertTitle-root': {
-                    textTransform: 'capitalize',
-                },
-            }}
-        >
-            <AlertBox
-                short
-                severity="warning"
-                title={
-                    <FormattedMessage
-                        id="workflows.disable.autoEnabledAlert.title"
-                        values={{
-                            entityType,
-                        }}
-                    />
-                }
+    if (forcedToEnable === 0) {
+        return (
+            <Box
+                sx={{
+                    'mb': 2,
+                    '& .MuiAlertTitle-root': {
+                        textTransform: 'capitalize',
+                    },
+                }}
             >
-                <Box>
-                    <FormattedMessage
-                        id="workflows.disable.autoEnabledAlert.message"
-                        values={{
-                            entityType,
-                        }}
-                    />
-                </Box>
-                <Box>
-                    <FormattedMessage
-                        id="workflows.disable.autoEnabledAlert.instructions"
-                        values={{
-                            entityType,
-                        }}
-                    />
-                </Box>
-            </AlertBox>
-        </Box>
-    );
+                <AlertBox
+                    short
+                    severity="warning"
+                    title={
+                        <FormattedMessage
+                            id="workflows.disable.autoEnabledAlert.title"
+                            values={{
+                                entityType,
+                            }}
+                        />
+                    }
+                >
+                    <Box>
+                        <FormattedMessage
+                            id="workflows.disable.autoEnabledAlert.message"
+                            values={{
+                                entityType,
+                            }}
+                        />
+                    </Box>
+                    <Box>
+                        <FormattedMessage
+                            id="workflows.disable.autoEnabledAlert.instructions"
+                            values={{
+                                entityType,
+                            }}
+                        />
+                    </Box>
+                </AlertBox>
+            </Box>
+        );
+    }
+
+    return null;
 }
 
 export default ShardsDisableWarning;

src/components/editor/Shards/Disable/index.tsx

@@ -2,17 +2,11 @@ import { Stack, Typography } from '@mui/material';
 import { FormattedMessage } from 'react-intl';
 import { useEntityType } from 'context/EntityContext';
 import { useEditorStore_persistedDraftId } from 'components/editor/Store/hooks';
-import useGlobalSearchParams, {
-    GlobalSearchParams,
-} from 'hooks/searchParams/useGlobalSearchParams';
+
 import ShardsDisableForm from './Form';
 import ShardsDisableWarning from './Warning';
 
 function ShardsDisable() {
-    const forcedToEnable = useGlobalSearchParams(
-        GlobalSearchParams.FORCED_SHARD_ENABLE
-    );
-
     const entityType = useEntityType();
 
     const draftId = useEditorStore_persistedDraftId();
@@ -23,7 +17,7 @@ function ShardsDisable() {
 
     return (
         <>
-            {forcedToEnable ? <ShardsDisableWarning /> : null}
+            <ShardsDisableWarning />
 
             <Stack spacing={1} sx={{ mb: 2 }}>
                 <Typography variant="formSectionHeader">

src/components/home/dashboard/EntityStatOverview/StatOverview.tsx

@@ -1,15 +1,10 @@
 import { Stack, Typography } from '@mui/material';
 import { PostgrestError } from '@supabase/postgrest-js';
-import { authenticatedRoutes } from 'app/routes';
 import LinkWrapper from 'components/shared/LinkWrapper';
-import {
-    semiTransparentBackground_blue,
-    semiTransparentBackground_purple,
-    semiTransparentBackground_teal,
-} from 'context/Theme';
-import { CloudDownload, CloudUpload, DatabaseScript } from 'iconoir-react';
+
 import { FormattedMessage, useIntl } from 'react-intl';
 import { Entity } from 'types';
+import { ENTITY_SETTINGS } from 'utils/entity-utils';
 import ActiveEntityCount from './ActiveEntityCount';
 import Statistic from './Statistic';
 
@@ -21,56 +16,6 @@ interface Props {
     monthlyUsageLoading?: boolean;
 }
 
-const getEntityPageURLPath = (entityType: string): string => {
-    if (entityType === 'collection') {
-        return authenticatedRoutes.collections.fullPath;
-    }
-
-    if (entityType === 'capture') {
-        return authenticatedRoutes.captures.fullPath;
-    }
-
-    return authenticatedRoutes.materializations.fullPath;
-};
-
-const getTitleId = (entityType: string): string => {
-    if (entityType === 'collection') {
-        return 'terms.collections';
-    }
-
-    if (entityType === 'capture') {
-        return 'terms.sources';
-    }
-
-    return 'terms.destinations';
-};
-
-const getBackgroundColor = (
-    entityType: string
-): { dark: string; light: string } => {
-    if (entityType === 'collection') {
-        return semiTransparentBackground_blue;
-    }
-
-    if (entityType === 'capture') {
-        return semiTransparentBackground_teal;
-    }
-
-    return semiTransparentBackground_purple;
-};
-
-const getEntityIcon = (entityType: string) => {
-    if (entityType === 'collection') {
-        return DatabaseScript;
-    }
-
-    if (entityType === 'capture') {
-        return CloudUpload;
-    }
-
-    return CloudDownload;
-};
-
 export default function StatOverview({
     entityType,
     monthlyUsage,
@@ -80,11 +25,12 @@ export default function StatOverview({
 }: Props) {
     const intl = useIntl();
 
-    const route = getEntityPageURLPath(entityType);
-    const titleId = getTitleId(entityType);
-
-    const background = getBackgroundColor(entityType);
-    const Icon = getEntityIcon(entityType);
+    const {
+        routes: { viewAll },
+        termId: titleId,
+        background,
+        Icon,
+    } = ENTITY_SETTINGS[entityType];
 
     return (
         <Stack
@@ -117,7 +63,7 @@ export default function StatOverview({
                     </Typography>
                 </Stack>
 
-                <LinkWrapper link={route}>
+                <LinkWrapper link={viewAll}>
                     <FormattedMessage id="cta.goToAll" />
                 </LinkWrapper>
             </Stack>