DataFlow backfill - disabled but most of the functionality is there (#…

…1261) * Turning everything back on * Moving steps stuff into a manual machine state in a store * Getting actions buttons wired up Calling intl for translations * Moving more stuff into content Moving label into store to reduce duplication Adding a running handler into parent * checking first step * Updating typing a lot Switching to uses as single status property Getting capture disable wired up * typing tweaks handling errors a bit better updating enum * Storing off step definition * Updating key to keep consistent * Adding support for a context to store data... but this feels wrong * adding quick support to store off things in a keyed / machine way * cleaning up handled as we are not using this * Forcing to pass an index when updating a step * quick hack and typing update * Checking locked to skip showing the load bar Closing when "back" button is clicked on first step No longer making steps options just defaulting to empty Just setting show to false is enough for closing dialog Checking the shallow `onFirstStep` to reduce renders Moving context out to the main store Only running initialize once per hydration * Adding handled to get no response when an insert is done to reduce network Fetching live spec based on spec id * Storing stuff to re-enable the capture Wiring up most of the steps (without waiting for no data) * marking valid so the next step works * Starting to wire up the last step * Updating ChipList to: Support passing in 'sx' so we can override styling when needed Scrolling the element down a little bit when all items are shown * Making code a bit safer * Typing for chiplist changes * styling the options better so the collections don't overlap the option below * Reset the state when leaving the flow * Probably best to just store the publication status as a whole Making consts for important keys * Adding a "review selections" step as an idea Starting work on handling errors better Trying to get publication populated correctly * Refactoring to reduce need of functions * Resetting when closing as doing it in store is not working * Moving hydrator within modal so it is always reset when opening and closing * Breaking out the review table * Adding more plural support * Updating content as requested * styling tweaks * Putting routes nested Moving the show prompt flag over to the form state store Linking to the capture and materialization details Have LinkWrapper handle newWindow links Allowing path overrides on the details navigator * Sharing the link to details Making code more null/empty * A bit better error handling and messaging Removing some filler content Adding support for valid and error to be handled by the store * clearing our errors when success * Using related collections list Allowing related collections to open in a new window Changing query to just find materializations with Source Capture Updating content to use the right terms Removing log token from the state * Making sure we only count the enabled bindings in the list * cleaning up hook that is not used * Fixing the warning to display correctly Moving the hook fetch into the component to reduce extra renders * Making the save and publish logging and warning shared Moving the LogDialog into a component folder * Moved to a component fold * Updating handling Showing proper error icon in stepper * Ended up not needing the hook Starting to figure out how to process skipping steps * Updating messaging * Not show logs right away Moving files around a bit to reduce nesting Move the loading indicator into the selector * Adding a uuid so make things a bit more traceable * Adding a LR event to try to make these sessions easier to find * Cleaning up comments Adding event to store to make it consistent * Cleaning up comments * Adding some handling for draft errors * Wiring up to exit the flow when complete Adding in a real time to prevent build from breaking Removing draft errors from alert * Updating details message * Disabling all the new stuff so this can be merged * Removing `detail` from the forms. * Removing code that is not needed * PR: comments * PR: commenting out function that isn't used * PR: commenting out to disable this * Making code a bit simpler Typing the ref Including the current top height when scrolling * Getting the chips aligned * PR: putting original count back in * Updating content to be more clear Switching out format components * PR: typo and small content change * PR: content typo
estuary · Sep 25, 2024 · 7bbb7d6 · 7bbb7d6
1 parent fe4cd04
commit 7bbb7d6
Show file tree

Hide file tree

Showing 89 changed files with 1,928 additions and 761 deletions.
diff --git a/src/api/draftSpecs.ts b/src/api/draftSpecs.ts
@@ -42,7 +42,8 @@ export const createDraftSpec = (
     catalogName: string,
     draftSpec: any,
     specType?: Entity | null,
-    lastPubId?: string | null
+    lastPubId?: string | null,
+    noResponse?: boolean
 ) => {
     let matchData: CreateMatchData = {
         draft_id: draftId,
@@ -55,7 +56,7 @@ export const createDraftSpec = (
         matchData = { ...matchData, expect_pub_id: lastPubId };
     }
 
-    return insertSupabase(TABLES.DRAFT_SPECS, matchData);
+    return insertSupabase(TABLES.DRAFT_SPECS, matchData, noResponse);
 };
 
 export const modifyDraftSpec = (

diff --git a/src/api/liveSpecsExt.ts b/src/api/liveSpecsExt.ts
@@ -206,7 +206,6 @@ export interface LiveSpecsExtQuery_DetailsForm {
     spec_type: Entity;
     spec: any;
     data_plane_id: string;
-    detail: string | null;
     connector_tag_id: string;
     connector_image_name: string;
     connector_image_tag: string;
@@ -219,7 +218,6 @@ const DETAILS_FORM_QUERY = `
     spec_type,
     spec,
     data_plane_id,
-    detail,
     connector_tag_id,
     connector_image_name,
     connector_image_tag,
@@ -372,6 +370,14 @@ const getLiveSpecsByLiveSpecId = async (liveSpecId: string) => {
     return data;
 };
 
+const getLiveSpecSpec = (liveSpecId: string) => {
+    return supabaseClient
+        .from(TABLES.LIVE_SPECS_EXT)
+        .select(`spec`)
+        .eq('id', liveSpecId)
+        .single();
+};
+
 const getLiveSpecShards = (tenant: string, entityType: Entity) => {
     return supabaseClient
         .from(TABLES.LIVE_SPECS_EXT)
@@ -380,6 +386,44 @@ const getLiveSpecShards = (tenant: string, entityType: Entity) => {
         .eq('spec_type', entityType);
 };
 
+const liveSpecsExtRelatedColumns = ['catalog_name', 'reads_from', 'id'];
+export const liveSpecsExtRelatedQuery = liveSpecsExtRelatedColumns.join(',');
+export interface LiveSpecsExt_Related {
+    catalog_name: string;
+    reads_from: string[];
+    id: string;
+}
+// const getLiveSpecsRelatedToMaterialization = async (
+//     collectionNames: string[]
+// ) => {
+//     const limiter = pLimit(3);
+//     const promises = [];
+//     let index = 0;
+
+//     // TODO (retry) promise generator
+//     const promiseGenerator = (idx: number) => {
+//         return supabaseClient
+//             .from(TABLES.LIVE_SPECS_EXT)
+//             .select(liveSpecsExtRelatedQuery)
+//             .eq('spec_type', 'materialization')
+//             .overlaps(
+//                 'reads_from',
+//                 collectionNames.slice(idx, idx + CHUNK_SIZE)
+//             )
+//             .returns<LiveSpecsExt_Related[]>();
+//     };
+
+//     while (index < collectionNames.length) {
+//         const prom = promiseGenerator(index);
+//         promises.push(limiter(() => prom));
+//         index = index + CHUNK_SIZE;
+//     }
+
+//     const response = await Promise.all(promises);
+//     const errors = response.filter((r) => r.error);
+//     return errors[0] ?? response[0];
+// };
+
 export {
     getLiveSpecs_captures,
     getLiveSpecs_collections,
@@ -392,4 +436,5 @@ export {
     getLiveSpecsByConnectorId,
     getLiveSpecsByLiveSpecId,
     getLiveSpecShards,
+    getLiveSpecSpec,
 };
diff --git a/src/components/editor/Bindings/Backfill/index.tsx b/src/components/editor/Bindings/Backfill/index.tsx
@@ -10,8 +10,8 @@ import {
     useBinding_currentCollection,
     useBinding_currentBindingUUID,
     useBinding_setBackfilledBindings,
-    useBinding_collections_count,
     useBinding_backfillSupported,
+    useBinding_enabledCollections_count,
 } from 'stores/Binding/hooks';
 import {
     useFormStateStore_isActive,
@@ -36,7 +36,7 @@ function Backfill({ description, bindingIndex = -1 }: BackfillProps) {
     // Binding Store
     const currentCollection = useBinding_currentCollection();
     const currentBindingUUID = useBinding_currentBindingUUID();
-    const collectionsCount = useBinding_collections_count();
+    const collectionsCount = useBinding_enabledCollections_count();
     const allBindingsDisabled = useBinding_allBindingsDisabled();
 
     const backfillAllBindings = useBinding_backfillAllBindings();
@@ -199,10 +199,11 @@ function Backfill({ description, bindingIndex = -1 }: BackfillProps) {
                 ) : null}
             </Stack>
 
-            {/*TODO (data flow reset)*/}
-            {/*            {bindingIndex === -1 && workflow === 'capture_edit' ? (
+            {/*TODO (data flow reset)
+            {bindingIndex === -1 && workflow === 'capture_edit' ? (
                 <BackfillDataFlowOption />
-            ) : null}*/}
+            ) : null}
+            */}
         </Box>
     );
 }

diff --git a/src/components/editor/Bindings/index.tsx b/src/components/editor/Bindings/index.tsx
@@ -110,11 +110,9 @@ function BindingsMultiEditor({
                 {workflow === 'capture_edit' ||
                 workflow === 'materialization_edit' ? (
                     <Backfill
-                        description={
-                            <FormattedMessage
-                                id={`workflows.collectionSelector.manualBackfill.message.${entityType}.allBindings`}
-                            />
-                        }
+                        description={intl.formatMessage({
+                            id: `workflows.collectionSelector.manualBackfill.message.${entityType}.allBindings`,
+                        })}
                     />
                 ) : null}
             </Stack>

diff --git a/src/components/editor/Shards/Disable/Warning.tsx b/src/components/editor/Shards/Disable/Warning.tsx
@@ -3,50 +3,61 @@ import { FormattedMessage } from 'react-intl';
 import { useEntityType } from 'context/EntityContext';
 
 import AlertBox from 'components/shared/AlertBox';
+import useGlobalSearchParams, {
+    GlobalSearchParams,
+} from 'hooks/searchParams/useGlobalSearchParams';
 
 function ShardsDisableWarning() {
+    const forcedToEnable = useGlobalSearchParams<number>(
+        GlobalSearchParams.FORCED_SHARD_ENABLE
+    );
+
     const entityType = useEntityType();
 
-    return (
-        <Box
-            sx={{
-                'mb': 2,
-                '& .MuiAlertTitle-root': {
-                    textTransform: 'capitalize',
-                },
-            }}
-        >
-            <AlertBox
-                short
-                severity="warning"
-                title={
-                    <FormattedMessage
-                        id="workflows.disable.autoEnabledAlert.title"
-                        values={{
-                            entityType,
-                        }}
-                    />
-                }
+    if (forcedToEnable === 0) {
+        return (
+            <Box
+                sx={{
+                    'mb': 2,
+                    '& .MuiAlertTitle-root': {
+                        textTransform: 'capitalize',
+                    },
+                }}
             >
-                <Box>
-                    <FormattedMessage
-                        id="workflows.disable.autoEnabledAlert.message"
-                        values={{
-                            entityType,
-                        }}
-                    />
-                </Box>
-                <Box>
-                    <FormattedMessage
-                        id="workflows.disable.autoEnabledAlert.instructions"
-                        values={{
-                            entityType,
-                        }}
-                    />
-                </Box>
-            </AlertBox>
-        </Box>
-    );
+                <AlertBox
+                    short
+                    severity="warning"
+                    title={
+                        <FormattedMessage
+                            id="workflows.disable.autoEnabledAlert.title"
+                            values={{
+                                entityType,
+                            }}
+                        />
+                    }
+                >
+                    <Box>
+                        <FormattedMessage
+                            id="workflows.disable.autoEnabledAlert.message"
+                            values={{
+                                entityType,
+                            }}
+                        />
+                    </Box>
+                    <Box>
+                        <FormattedMessage
+                            id="workflows.disable.autoEnabledAlert.instructions"
+                            values={{
+                                entityType,
+                            }}
+                        />
+                    </Box>
+                </AlertBox>
+            </Box>
+        );
+    }
+
+    return null;
 }
 
 export default ShardsDisableWarning;
diff --git a/src/components/editor/Shards/Disable/index.tsx b/src/components/editor/Shards/Disable/index.tsx
@@ -2,17 +2,11 @@ import { Stack, Typography } from '@mui/material';
 import { FormattedMessage } from 'react-intl';
 import { useEntityType } from 'context/EntityContext';
 import { useEditorStore_persistedDraftId } from 'components/editor/Store/hooks';
-import useGlobalSearchParams, {
-    GlobalSearchParams,
-} from 'hooks/searchParams/useGlobalSearchParams';
+
 import ShardsDisableForm from './Form';
 import ShardsDisableWarning from './Warning';
 
 function ShardsDisable() {
-    const forcedToEnable = useGlobalSearchParams(
-        GlobalSearchParams.FORCED_SHARD_ENABLE
-    );
-
     const entityType = useEntityType();
 
     const draftId = useEditorStore_persistedDraftId();
@@ -23,7 +17,7 @@ function ShardsDisable() {
 
     return (
         <>
-            {forcedToEnable ? <ShardsDisableWarning /> : null}
+            <ShardsDisableWarning />
 
             <Stack spacing={1} sx={{ mb: 2 }}>
                 <Typography variant="formSectionHeader">

diff --git a/src/components/home/dashboard/EntityStatOverview/StatOverview.tsx b/src/components/home/dashboard/EntityStatOverview/StatOverview.tsx
@@ -1,15 +1,10 @@
 import { Stack, Typography } from '@mui/material';
 import { PostgrestError } from '@supabase/postgrest-js';
-import { authenticatedRoutes } from 'app/routes';
 import LinkWrapper from 'components/shared/LinkWrapper';
-import {
-    semiTransparentBackground_blue,
-    semiTransparentBackground_purple,
-    semiTransparentBackground_teal,
-} from 'context/Theme';
-import { CloudDownload, CloudUpload, DatabaseScript } from 'iconoir-react';
+
 import { FormattedMessage, useIntl } from 'react-intl';
 import { Entity } from 'types';
+import { ENTITY_SETTINGS } from 'utils/entity-utils';
 import ActiveEntityCount from './ActiveEntityCount';
 import Statistic from './Statistic';
 
@@ -21,56 +16,6 @@ interface Props {
     monthlyUsageLoading?: boolean;
 }
 
-const getEntityPageURLPath = (entityType: string): string => {
-    if (entityType === 'collection') {
-        return authenticatedRoutes.collections.fullPath;
-    }
-
-    if (entityType === 'capture') {
-        return authenticatedRoutes.captures.fullPath;
-    }
-
-    return authenticatedRoutes.materializations.fullPath;
-};
-
-const getTitleId = (entityType: string): string => {
-    if (entityType === 'collection') {
-        return 'terms.collections';
-    }
-
-    if (entityType === 'capture') {
-        return 'terms.sources';
-    }
-
-    return 'terms.destinations';
-};
-
-const getBackgroundColor = (
-    entityType: string
-): { dark: string; light: string } => {
-    if (entityType === 'collection') {
-        return semiTransparentBackground_blue;
-    }
-
-    if (entityType === 'capture') {
-        return semiTransparentBackground_teal;
-    }
-
-    return semiTransparentBackground_purple;
-};
-
-const getEntityIcon = (entityType: string) => {
-    if (entityType === 'collection') {
-        return DatabaseScript;
-    }
-
-    if (entityType === 'capture') {
-        return CloudUpload;
-    }
-
-    return CloudDownload;
-};
-
 export default function StatOverview({
     entityType,
     monthlyUsage,
@@ -80,11 +25,12 @@ export default function StatOverview({
 }: Props) {
     const intl = useIntl();
 
-    const route = getEntityPageURLPath(entityType);
-    const titleId = getTitleId(entityType);
-
-    const background = getBackgroundColor(entityType);
-    const Icon = getEntityIcon(entityType);
+    const {
+        routes: { viewAll },
+        termId: titleId,
+        background,
+        Icon,
+    } = ENTITY_SETTINGS[entityType];
 
     return (
         <Stack
@@ -117,7 +63,7 @@ export default function StatOverview({
                     </Typography>
                 </Stack>
 
-                <LinkWrapper link={route}>
+                <LinkWrapper link={viewAll}>
                     <FormattedMessage id="cta.goToAll" />
                 </LinkWrapper>
             </Stack>