From b0cb903d93b18b60202809e89d755e9c8730fcc4 Mon Sep 17 00:00:00 2001 From: Travis Jenkins Date: Fri, 10 Jan 2025 13:51:34 -0500 Subject: [PATCH 1/9] Adding a script to help make developing flow web changes easier --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 966704ebc..d2d1b188f 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "installDataPlane": "npm install 'https://gitpkg.now.sh/estuary/data-plane-gateway/client/dist?main'", "beta_logRocketIntegrity": "node --env-file .env ./scripts/logRocketIntegirty", "generate-flow-types": "cd deps/flow && ./fetch-flow.sh && ./flow-bin/flowctl json-schema | ./flow-bin/flowctl schemalate typescript --name Catalog --hoist-definitions > flow.d.ts", - "generate-supabase-types": "node ./scripts/generateSupabaseTypes" + "generate-supabase-types": "node ./scripts/generateSupabaseTypes", + "use-local-web-flow": "./deps/flow-web/hack-in-local-copy.sh" }, "dependencies": { "@date-fns/utc": "^1.0.0", From 2b4e833b80a620669de8db18f45dae823b019b11 Mon Sep 17 00:00:00 2001 From: Travis Jenkins Date: Fri, 10 Jan 2025 14:02:17 -0500 Subject: [PATCH 2/9] Adding a call to the new WASM endpoint to fetch pointers Storing those pointers in the store Then updating checking support by using shared hook and useShallow Broke up the binding store code a bit to help IDE slowness --- deps/flow-web/hack-in-local-copy.sh | 38 ++++ deps/flow/hack-in-flow-web.sh | 3 + .../AddSourceCaptureToSpecButton.tsx | 13 +- .../SourceCaptureChipOptionalSettings.tsx | 9 +- .../Entity/AddDialog/OptionalSettings.tsx | 9 +- src/services/ajv.ts | 53 ++--- src/stores/Binding/Store.ts | 185 ++---------------- src/stores/Binding/hooks.ts | 13 ++ src/stores/Binding/shared.ts | 156 +++++++++++++++ src/stores/Binding/types.ts | 4 +- src/stores/SourceCapture/hooks.ts | 9 +- 11 files changed, 267 insertions(+), 225 deletions(-) create mode 100755 deps/flow-web/hack-in-local-copy.sh create mode 100644 deps/flow/hack-in-flow-web.sh diff --git a/deps/flow-web/hack-in-local-copy.sh b/deps/flow-web/hack-in-local-copy.sh new file mode 100755 index 000000000..91237853a --- /dev/null +++ b/deps/flow-web/hack-in-local-copy.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Check if exactly one argument is passed +if [ $# -ne 1 ]; then + echo "Please provide the path to flow (ex: /home/travis/code/flow)" + exit 1 +fi + +# The directory path from the argument +SOURCE_DIR="$1/crates/flow-web/pkg/" + +# Check if the directory exists +if [ ! -d "$SOURCE_DIR" ]; then + echo "Error: '$SOURCE_DIR' is not a valid directory." + exit 1 +fi + +# Destination directory +DEST_DIR="node_modules/@estuary/flow-web" + +# Create the destination directory if it doesn't exist +mkdir -p "$DEST_DIR" + +# Copy the directory to the destination +cp -r "$SOURCE_DIR"/* "$DEST_DIR" + +echo "Directory '$SOURCE_DIR' has been copied to '$DEST_DIR'." + +# Path to the directory to delete +DIR_TO_DELETE="node_modules/.vite/deps" + +# Check if the directory exists, and delete it if it does +if [ -d "$DIR_TO_DELETE" ]; then + rm -r "$DIR_TO_DELETE" + echo "Directory '$DIR_TO_DELETE' has been deleted." +else + echo "Directory '$DIR_TO_DELETE' does not exist." +fi \ No newline at end of file diff --git a/deps/flow/hack-in-flow-web.sh b/deps/flow/hack-in-flow-web.sh new file mode 100644 index 000000000..504f309de --- /dev/null +++ b/deps/flow/hack-in-flow-web.sh @@ -0,0 +1,3 @@ + + +node_modules/%40estuary/flow-web \ No newline at end of file diff --git a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx index e243949f9..68e679316 100644 --- a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx +++ b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx @@ -2,8 +2,10 @@ import { Button } from '@mui/material'; import { AddCollectionDialogCTAProps } from 'components/shared/Entity/types'; import invariableStores from 'context/Zustand/invariableStores'; import { FormattedMessage } from 'react-intl'; -import { useBinding_prefillResourceConfigs } from 'stores/Binding/hooks'; -import { useBindingStore } from 'stores/Binding/Store'; +import { + useBinding_prefillResourceConfigs, + useBinding_sourceCaptureFlags, +} from 'stores/Binding/hooks'; import { useSourceCaptureStore } from 'stores/SourceCapture/Store'; import { SourceCaptureDef } from 'types'; import { useStore } from 'zustand'; @@ -19,13 +21,10 @@ function AddSourceCaptureToSpecButton({ toggle }: AddCollectionDialogCTAProps) { const { existingSourceCapture, updateDraft } = useSourceCapture(); - const [ + const { sourceCaptureDeltaUpdatesSupported, sourceCaptureTargetSchemaSupported, - ] = useBindingStore((state) => [ - state.sourceCaptureDeltaUpdatesSupported, - state.sourceCaptureTargetSchemaSupported, - ]); + } = useBinding_sourceCaptureFlags(); const [sourceCapture, setSourceCapture, deltaUpdates, targetSchema] = useSourceCaptureStore((state) => [ diff --git a/src/components/materialization/SourceCapture/SourceCaptureChipOptionalSettings.tsx b/src/components/materialization/SourceCapture/SourceCaptureChipOptionalSettings.tsx index 307d1a65f..4db4e24ba 100644 --- a/src/components/materialization/SourceCapture/SourceCaptureChipOptionalSettings.tsx +++ b/src/components/materialization/SourceCapture/SourceCaptureChipOptionalSettings.tsx @@ -1,4 +1,4 @@ -import { useBindingStore } from 'stores/Binding/Store'; +import { useBinding_sourceCaptureFlags } from 'stores/Binding/hooks'; import { useSourceCaptureStore_sourceCaptureDefinition } from 'stores/SourceCapture/hooks'; import SourceCaptureChipOption from './SourceCaptureChipOption'; @@ -6,13 +6,10 @@ function SourceCaptureChipOptionalSettings() { const sourceCaptureDefinition = useSourceCaptureStore_sourceCaptureDefinition(); - const [ + const { sourceCaptureDeltaUpdatesSupported, sourceCaptureTargetSchemaSupported, - ] = useBindingStore((state) => [ - state.sourceCaptureDeltaUpdatesSupported, - state.sourceCaptureTargetSchemaSupported, - ]); + } = useBinding_sourceCaptureFlags(); if ( (sourceCaptureDeltaUpdatesSupported || diff --git a/src/components/shared/Entity/AddDialog/OptionalSettings.tsx b/src/components/shared/Entity/AddDialog/OptionalSettings.tsx index 6117de32c..e9e9eefc8 100644 --- a/src/components/shared/Entity/AddDialog/OptionalSettings.tsx +++ b/src/components/shared/Entity/AddDialog/OptionalSettings.tsx @@ -2,18 +2,15 @@ import { Stack, Typography } from '@mui/material'; import DeltaUpdates from 'components/editor/Bindings/DeltaUpdates'; import SchemaMode from 'components/editor/Bindings/SchemaMode'; import { useIntl } from 'react-intl'; -import { useBindingStore } from 'stores/Binding/Store'; +import { useBinding_sourceCaptureFlags } from 'stores/Binding/hooks'; function OptionalSettings() { const intl = useIntl(); - const [ + const { sourceCaptureDeltaUpdatesSupported, sourceCaptureTargetSchemaSupported, - ] = useBindingStore((state) => [ - state.sourceCaptureDeltaUpdatesSupported, - state.sourceCaptureTargetSchemaSupported, - ]); + } = useBinding_sourceCaptureFlags(); if ( !sourceCaptureDeltaUpdatesSupported && diff --git a/src/services/ajv.ts b/src/services/ajv.ts index 25cbf9716..de8c5f1ba 100644 --- a/src/services/ajv.ts +++ b/src/services/ajv.ts @@ -1,3 +1,4 @@ +import { get_resource_config_pointers } from '@estuary/flow-web'; import { createAjv } from '@jsonforms/core'; import { isEmpty } from 'lodash'; import { Annotations } from 'types/jsonforms'; @@ -114,42 +115,28 @@ export function createJSONFormDefaults( return { data, errors }; } -// TODO (web flow wasm) This should be fetched with WASM code -// waiting on https://github.com/estuary/flow/issues/1760 +// TODO (web flow wasm - source capture) +// Maybe we can get this type from wasm? export interface ResourceConfigPointers { - [Annotations.defaultResourceConfigName]?: boolean; - [Annotations.targetSchema]?: boolean; - [Annotations.deltaUpdates]?: boolean; + ['x_collection_name']: string | undefined; + ['x_schema_name']: string | undefined; + ['x_delta_updates']: string | undefined; } -export const findKeysInObject = ( - obj: Record, - keysToFind: string[], - results: ResourceConfigPointers = {} -): ResourceConfigPointers => { - // Iterate over each key in the object - for (const key in obj) { - if (obj.hasOwnProperty(key)) { - const value = obj[key]; - - // Check if the key is one of the keys we're searching for - if (keysToFind.includes(key)) { - results[key] = value; - } - - // If the value is an object, recurse into it - if (value && typeof value === 'object') { - findKeysInObject(value, keysToFind, results); - } +export const getResourceConfigPointers = ( + schema: any +): ResourceConfigPointers | null => { + try { + const response = get_resource_config_pointers(schema); + + if (!response.pointers) { + return null; } - } - return results; -}; + return response.pointers; -export const getResourceConfigPointers = (schema: any) => - findKeysInObject(schema, [ - Annotations.defaultResourceConfigName, - Annotations.targetSchema, - Annotations.deltaUpdates, - ]); + // eslint-disable-next-line @typescript-eslint/no-implicit-any-catch + } catch (e: any) { + return null; + } +}; diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index 459bd4145..3efc126f3 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -1,12 +1,8 @@ -import { PostgrestError } from '@supabase/postgrest-js'; -import { getDraftSpecsByDraftId } from 'api/draftSpecs'; import { getLiveSpecsById_writesTo, getLiveSpecsByLiveSpecId, - getSchema_Resource, } from 'api/hydration'; import { GlobalSearchParams } from 'hooks/searchParams/useGlobalSearchParams'; -import { LiveSpecsExtQuery } from 'hooks/useLiveSpecsExt'; import produce from 'immer'; import { difference, @@ -24,28 +20,29 @@ import { getResourceConfigPointers, } from 'services/ajv'; import { logRocketEvent } from 'services/shared'; -import { BASE_ERROR } from 'services/supabase'; +import { stringifyJSON } from 'services/stringify'; import { CustomEvents } from 'services/types'; -import { - getInitialHydrationData, - getStoreWithHydrationSettings, -} from 'stores/extensions/Hydration'; +import { getStoreWithHydrationSettings } from 'stores/extensions/Hydration'; import { BindingStoreNames } from 'stores/names'; -import { Entity, Schema } from 'types'; import { getDereffedSchema, hasLength } from 'utils/misc-utils'; import { devtoolsOptions } from 'utils/store-utils'; -import { formatCaptureInterval, parsePostgresInterval } from 'utils/time-utils'; +import { parsePostgresInterval } from 'utils/time-utils'; import { getBackfillCounter, getBindingIndex } from 'utils/workflow-utils'; import { POSTGRES_INTERVAL_RE } from 'validation'; import { create, StoreApi } from 'zustand'; import { devtools, NamedSet } from 'zustand/middleware'; import { getCollectionNames, + getInitialMiscData, + getInitialStoreData, + hydrateConnectorTagDependentState, + hydrateSpecificationDependentState, initializeAndGenerateUUID, initializeBinding, initializeCurrentBinding, populateResourceConfigErrors, sortResourceConfigs, + STORE_KEY, whatChanged, } from './shared'; import { @@ -59,151 +56,6 @@ import { } from './slices/TimeTravel'; import { BindingMetadata, BindingState, ResourceConfig } from './types'; -const STORE_KEY = 'Bindings'; - -const hydrateConnectorTagDependentState = async ( - connectorTagId: string, - get: StoreApi['getState'] -): Promise => { - if (!hasLength(connectorTagId)) { - return null; - } - - const { data, error } = await getSchema_Resource(connectorTagId); - - if (error) { - get().setHydrationErrorsExist(true); - } else if (data?.resource_spec_schema) { - const schema = data.resource_spec_schema as unknown as Schema; - await get().setResourceSchema(schema); - - get().setBackfillSupported(!Boolean(data.disable_backfill)); - } - - return data; -}; - -const hydrateSpecificationDependentState = async ( - defaultInterval: string | null | undefined, - entityType: Entity, - fallbackInterval: string | null, - get: StoreApi['getState'], - liveSpec: LiveSpecsExtQuery['spec'], - searchParams: URLSearchParams -): Promise => { - const draftId = searchParams.get(GlobalSearchParams.DRAFT_ID); - - if (draftId) { - const { data: draftSpecs, error } = await getDraftSpecsByDraftId( - draftId, - entityType - ); - - if (error || !draftSpecs || draftSpecs.length === 0) { - return ( - error ?? { - ...BASE_ERROR, - message: `An issue was encountered fetching the drafted specification for this ${entityType}`, - } - ); - } - - get().prefillBindingDependentState( - entityType, - liveSpec.bindings, - draftSpecs[0].spec.bindings - ); - - const targetInterval = draftSpecs[0].spec?.interval; - - get().setCaptureInterval( - targetInterval - ? formatCaptureInterval(targetInterval) - : fallbackInterval, - defaultInterval - ); - - get().setSpecOnIncompatibleSchemaChange( - draftSpecs[0].spec?.onIncompatibleSchemaChange - ); - } else { - get().prefillBindingDependentState(entityType, liveSpec.bindings); - - get().setCaptureInterval( - liveSpec?.interval ?? fallbackInterval, - defaultInterval - ); - - get().setSpecOnIncompatibleSchemaChange( - liveSpec?.onIncompatibleSchemaChange - ); - } - - return null; -}; - -const getInitialBindingData = (): Pick< - BindingState, - 'bindingErrorsExist' | 'bindings' | 'currentBinding' -> => ({ - bindingErrorsExist: false, - bindings: {}, - currentBinding: null, -}); - -const getInitialMiscData = (): Pick< - BindingState, - | 'backfilledBindings' - | 'backfillAllBindings' - | 'backfillDataFlow' - | 'backfillDataFlowTarget' - | 'backfillSupported' - | 'captureInterval' - | 'collectionsRequiringRediscovery' - | 'defaultCaptureInterval' - | 'discoveredCollections' - | 'evolvedCollections' - | 'onIncompatibleSchemaChange' - | 'rediscoveryRequired' - | 'resourceConfigErrorsExist' - | 'resourceConfigErrors' - | 'resourceConfigs' - | 'resourceSchema' - | 'restrictedDiscoveredCollections' - | 'serverUpdateRequired' - | 'sourceCaptureDeltaUpdatesSupported' - | 'sourceCaptureTargetSchemaSupported' -> => ({ - backfillAllBindings: false, - backfillDataFlowTarget: null, - backfillDataFlow: false, - backfillSupported: true, - backfilledBindings: [], - captureInterval: null, - collectionsRequiringRediscovery: [], - defaultCaptureInterval: null, - discoveredCollections: [], - evolvedCollections: [], - onIncompatibleSchemaChange: undefined, - rediscoveryRequired: false, - resourceConfigErrorsExist: false, - resourceConfigErrors: [], - resourceConfigs: {}, - resourceSchema: {}, - restrictedDiscoveredCollections: [], - serverUpdateRequired: false, - sourceCaptureDeltaUpdatesSupported: false, - sourceCaptureTargetSchemaSupported: false, -}); - -const getInitialStoreData = () => ({ - ...getInitialBindingData(), - ...getInitialFieldSelectionData(), - ...getInitialHydrationData(), - ...getInitialMiscData(), - ...getInitialTimeTravelData(), -}); - const getInitialState = ( set: NamedSet, get: StoreApi['getState'] @@ -941,16 +793,19 @@ const getInitialState = ( // TODO (web flow wasm - source capture - possible perf improvement) // Us calling `getResourceConfigPointers` here means we end up going // through the schema multiple times. Once here and twice when we go through - // it to generate a UI schema. That does NOT set anything in a store and probably - // should never set anything in a store directly. + // it to generate a UI schema. When generating the UI Schema we have never set + // anything in a store and probably should never do that... maybe. // Might not be a huge deal to do this twice but something to think about. - const pointers = getResourceConfigPointers(resolved); - state.sourceCaptureDeltaUpdatesSupported = Boolean( - pointers['x-delta-updates'] - ); - state.sourceCaptureTargetSchemaSupported = Boolean( - pointers['x-schema-name'] - ); + const resourceConfigPointers = getResourceConfigPointers({ + spec: stringifyJSON(resolved), + }); + + if (!resourceConfigPointers) { + state.setHydrationErrorsExist(true); + return; + } + + state.resourceConfigPointers = resourceConfigPointers; }), false, 'Resource Schema Set' diff --git a/src/stores/Binding/hooks.ts b/src/stores/Binding/hooks.ts index a044c832d..a0ace30a8 100644 --- a/src/stores/Binding/hooks.ts +++ b/src/stores/Binding/hooks.ts @@ -1,3 +1,4 @@ +import { hasLength } from 'utils/misc-utils'; import { useShallow } from 'zustand/react/shallow'; import { getCollectionNames, @@ -339,3 +340,15 @@ export const useBinding_collectionsBeingBackfilled = () => }); }) ); + +export const useBinding_sourceCaptureFlags = () => + useBindingStore( + useShallow((state) => ({ + sourceCaptureDeltaUpdatesSupported: hasLength( + state.resourceConfigPointers?.x_delta_updates + ), + sourceCaptureTargetSchemaSupported: hasLength( + state.resourceConfigPointers?.x_schema_name + ), + })) + ); diff --git a/src/stores/Binding/shared.ts b/src/stores/Binding/shared.ts index f6458b9d5..8d38b4540 100644 --- a/src/stores/Binding/shared.ts +++ b/src/stores/Binding/shared.ts @@ -1,6 +1,19 @@ +import { PostgrestError } from '@supabase/postgrest-js'; +import { getDraftSpecsByDraftId } from 'api/draftSpecs'; +import { getSchema_Resource } from 'api/hydration'; +import { GlobalSearchParams } from 'hooks/searchParams/useGlobalSearchParams'; +import { LiveSpecsExtQuery } from 'hooks/useLiveSpecsExt'; import { difference, intersection } from 'lodash'; +import { BASE_ERROR } from 'services/supabase'; +import { getInitialHydrationData } from 'stores/extensions/Hydration'; import { populateErrors } from 'stores/utils'; +import { Entity, Schema } from 'types'; +import { hasLength } from 'utils/misc-utils'; +import { formatCaptureInterval } from 'utils/time-utils'; import { getCollectionName, getDisableProps } from 'utils/workflow-utils'; +import { StoreApi } from 'zustand'; +import { getInitialFieldSelectionData } from './slices/FieldSelection'; +import { getInitialTimeTravelData } from './slices/TimeTravel'; import { Bindings, BindingState, @@ -196,3 +209,146 @@ export const initializeAndGenerateUUID = ( UUID, }; }; + +export const STORE_KEY = 'Bindings'; + +export const hydrateConnectorTagDependentState = async ( + connectorTagId: string, + get: StoreApi['getState'] +): Promise => { + if (!hasLength(connectorTagId)) { + return null; + } + + const { data, error } = await getSchema_Resource(connectorTagId); + + if (error) { + get().setHydrationErrorsExist(true); + } else if (data?.resource_spec_schema) { + const schema = data.resource_spec_schema as unknown as Schema; + await get().setResourceSchema(schema); + + get().setBackfillSupported(!Boolean(data.disable_backfill)); + } + + return data; +}; + +export const hydrateSpecificationDependentState = async ( + defaultInterval: string | null | undefined, + entityType: Entity, + fallbackInterval: string | null, + get: StoreApi['getState'], + liveSpec: LiveSpecsExtQuery['spec'], + searchParams: URLSearchParams +): Promise => { + const draftId = searchParams.get(GlobalSearchParams.DRAFT_ID); + + if (draftId) { + const { data: draftSpecs, error } = await getDraftSpecsByDraftId( + draftId, + entityType + ); + + if (error || !draftSpecs || draftSpecs.length === 0) { + return ( + error ?? { + ...BASE_ERROR, + message: `An issue was encountered fetching the drafted specification for this ${entityType}`, + } + ); + } + + get().prefillBindingDependentState( + entityType, + liveSpec.bindings, + draftSpecs[0].spec.bindings + ); + + const targetInterval = draftSpecs[0].spec?.interval; + + get().setCaptureInterval( + targetInterval + ? formatCaptureInterval(targetInterval) + : fallbackInterval, + defaultInterval + ); + + get().setSpecOnIncompatibleSchemaChange( + draftSpecs[0].spec?.onIncompatibleSchemaChange + ); + } else { + get().prefillBindingDependentState(entityType, liveSpec.bindings); + + get().setCaptureInterval( + liveSpec?.interval ?? fallbackInterval, + defaultInterval + ); + + get().setSpecOnIncompatibleSchemaChange( + liveSpec?.onIncompatibleSchemaChange + ); + } + + return null; +}; + +const getInitialBindingData = (): Pick< + BindingState, + 'bindingErrorsExist' | 'bindings' | 'currentBinding' +> => ({ + bindingErrorsExist: false, + bindings: {}, + currentBinding: null, +}); + +export const getInitialMiscData = (): Pick< + BindingState, + | 'backfilledBindings' + | 'backfillAllBindings' + | 'backfillDataFlow' + | 'backfillDataFlowTarget' + | 'backfillSupported' + | 'captureInterval' + | 'collectionsRequiringRediscovery' + | 'defaultCaptureInterval' + | 'discoveredCollections' + | 'evolvedCollections' + | 'onIncompatibleSchemaChange' + | 'rediscoveryRequired' + | 'resourceConfigErrorsExist' + | 'resourceConfigErrors' + | 'resourceConfigs' + | 'resourceSchema' + | 'restrictedDiscoveredCollections' + | 'serverUpdateRequired' + | 'resourceConfigPointers' +> => ({ + backfillAllBindings: false, + backfillDataFlowTarget: null, + backfillDataFlow: false, + backfillSupported: true, + backfilledBindings: [], + captureInterval: null, + collectionsRequiringRediscovery: [], + defaultCaptureInterval: null, + discoveredCollections: [], + evolvedCollections: [], + onIncompatibleSchemaChange: undefined, + rediscoveryRequired: false, + resourceConfigErrorsExist: false, + resourceConfigErrors: [], + resourceConfigs: {}, + resourceSchema: {}, + restrictedDiscoveredCollections: [], + serverUpdateRequired: false, + resourceConfigPointers: undefined, +}); + +export const getInitialStoreData = () => ({ + ...getInitialBindingData(), + ...getInitialFieldSelectionData(), + ...getInitialHydrationData(), + ...getInitialMiscData(), + ...getInitialTimeTravelData(), +}); diff --git a/src/stores/Binding/types.ts b/src/stores/Binding/types.ts index e93afeafc..d95271045 100644 --- a/src/stores/Binding/types.ts +++ b/src/stores/Binding/types.ts @@ -2,6 +2,7 @@ import { EvolvedCollections } from 'api/evolutions'; import { BooleanString } from 'components/shared/buttons/types'; import { LiveSpecsExt_MaterializeOrTransform } from 'hooks/useLiveSpecsExt'; import { DurationObjectUnits } from 'luxon'; +import { ResourceConfigPointers } from 'services/ajv'; import { CallSupabaseResponse } from 'services/supabase'; import { StoreWithHydration } from 'stores/extensions/Hydration'; import { Entity, EntityWorkflow, JsonFormsData, Schema } from 'types'; @@ -108,8 +109,7 @@ export interface BindingState setBackfillSupported: (val: BindingState['backfillSupported']) => void; // Control sourceCapture optional settings - sourceCaptureTargetSchemaSupported: boolean; - sourceCaptureDeltaUpdatesSupported: boolean; + resourceConfigPointers?: ResourceConfigPointers; // Capture interval captureInterval: string | null; diff --git a/src/stores/SourceCapture/hooks.ts b/src/stores/SourceCapture/hooks.ts index aed5418b4..602a79ce4 100644 --- a/src/stores/SourceCapture/hooks.ts +++ b/src/stores/SourceCapture/hooks.ts @@ -1,18 +1,15 @@ import { useCallback } from 'react'; -import { useBindingStore } from 'stores/Binding/Store'; +import { useBinding_sourceCaptureFlags } from 'stores/Binding/hooks'; import { SourceCaptureDef } from 'types'; import { useShallow } from 'zustand/react/shallow'; import { useSourceCaptureStore } from './Store'; export const useSourceCaptureStore_sourceCaptureDefinition = (): SourceCaptureDef | null => { - const [ + const { sourceCaptureDeltaUpdatesSupported, sourceCaptureTargetSchemaSupported, - ] = useBindingStore((state) => [ - state.sourceCaptureDeltaUpdatesSupported, - state.sourceCaptureTargetSchemaSupported, - ]); + } = useBinding_sourceCaptureFlags(); return useSourceCaptureStore( useShallow((state) => { From c3c3e470734fa75f71e68164515d0edb701e0232 Mon Sep 17 00:00:00 2001 From: Travis Jenkins Date: Mon, 13 Jan 2025 13:45:49 -0500 Subject: [PATCH 3/9] Basic typing to keep things more strict Making calls to fetch default binding settings Adding in support to pass through default binding settings --- .../AddSourceCaptureToSpecButton.tsx | 6 +++- src/services/ajv.ts | 17 ++++++---- src/stores/Binding/Store.ts | 33 ++++++++++++++----- src/stores/Binding/types.ts | 11 +++++-- src/types/index.ts | 5 +++ 5 files changed, 55 insertions(+), 17 deletions(-) diff --git a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx index 68e679316..8ff766c33 100644 --- a/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx +++ b/src/components/materialization/SourceCapture/AddSourceCaptureToSpecButton.tsx @@ -78,7 +78,11 @@ function AddSourceCaptureToSpecButton({ toggle }: AddCollectionDialogCTAProps) { setSourceCapture(updatedSourceCapture.capture); if (selectedRow?.writes_to) { - prefillResourceConfigs(selectedRow.writes_to, true); + prefillResourceConfigs( + selectedRow.writes_to, + true, + updatedSourceCapture + ); } } diff --git a/src/services/ajv.ts b/src/services/ajv.ts index de8c5f1ba..46bb9639e 100644 --- a/src/services/ajv.ts +++ b/src/services/ajv.ts @@ -1,6 +1,7 @@ import { get_resource_config_pointers } from '@estuary/flow-web'; import { createAjv } from '@jsonforms/core'; import { isEmpty } from 'lodash'; +import { DefaultAjvResponse, Schema } from 'types'; import { Annotations } from 'types/jsonforms'; import { stripPathing } from 'utils/misc-utils'; @@ -83,17 +84,21 @@ function defaultResourceSchema(resourceSchema: any, collection: string) { } } +export function createJSONFormDefaults(jsonSchema: any): DefaultAjvResponse; export function createJSONFormDefaults( jsonSchema: any, - collection?: string -): { - data: any; - errors: any[]; -} { + collection: string, + dataDefaults: Schema +): DefaultAjvResponse; +export function createJSONFormDefaults( + jsonSchema: any, + collection?: string, + dataDefaults?: Schema +): DefaultAjvResponse { // We start with an empty object, and then validate it to set any default values. // Note that this requires all parent properties to also specify a `default` in the json // schema. - const data = {}; + const data = dataDefaults ?? {}; // Get the schema we want. // If collection then we want to generate a Resource Schema diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index 3efc126f3..23319aa51 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -1,3 +1,4 @@ +import { update_materialization_resource_spec } from '@estuary/flow-web'; import { getLiveSpecsById_writesTo, getLiveSpecsByLiveSpecId, @@ -131,7 +132,8 @@ const getInitialState = ( modifiedResourceConfigs[bindingUUID] = { ...createJSONFormDefaults( state.resourceSchema, - collectionName + collectionName, + {} ), meta: { collectionName, bindingIndex }, }; @@ -366,7 +368,7 @@ const getInitialState = ( ); }, - prefillResourceConfigs: (targetCollections, disableOmit) => { + prefillResourceConfigs: (targetCollections, disableOmit, sourceCapture) => { set( produce((state: BindingState) => { const collections = getCollectionNames(state.resourceConfigs); @@ -402,16 +404,31 @@ const getInitialState = ( initializeBinding(state, collectionName, bindingUUID); + let prefilledData = {}; + if (sourceCapture) { + // If we have a sourceCapture then we should use those settings to have WASM + // produce some default data. This prefills certain settings the same way the + // backend would when new bindings are added. + const defaultSchema = + update_materialization_resource_spec({ + source_capture: sourceCapture, + resource_spec: {}, + resource_spec_pointers: + state.resourceConfigPointers, + collection_name: collectionName, + }); + + // TODO (web flow wasm - source capture) + // We need to do some better error handling here + prefilledData = JSON.parse(defaultSchema); + } + const jsonFormDefaults = createJSONFormDefaults( state.resourceSchema, - collectionName + collectionName, + prefilledData ); - // TODO (web flow wasm - source capture) - // Will want to merge / update / something with WASM here into the defaultConfig.data - // once we add the ability in WASM AND we wire up the ability for a user to make - // mass changes to their binding settings here - state.resourceConfigs[bindingUUID] = { ...jsonFormDefaults, meta: { diff --git a/src/stores/Binding/types.ts b/src/stores/Binding/types.ts index d95271045..de2bd9526 100644 --- a/src/stores/Binding/types.ts +++ b/src/stores/Binding/types.ts @@ -5,7 +5,13 @@ import { DurationObjectUnits } from 'luxon'; import { ResourceConfigPointers } from 'services/ajv'; import { CallSupabaseResponse } from 'services/supabase'; import { StoreWithHydration } from 'stores/extensions/Hydration'; -import { Entity, EntityWorkflow, JsonFormsData, Schema } from 'types'; +import { + Entity, + EntityWorkflow, + JsonFormsData, + Schema, + SourceCaptureDef, +} from 'types'; import { StoreWithFieldSelection } from './slices/FieldSelection'; import { StoreWithTimeTravel } from './slices/TimeTravel'; @@ -137,7 +143,8 @@ export interface BindingState // and bindings are added to the specification via the collection selector. prefillResourceConfigs: ( targetCollections: string[], - disableOmit?: boolean + disableOmit?: boolean, + sourceCapture?: SourceCaptureDef ) => void; // The combination of resource config store actions, `updateResourceConfig` and diff --git a/src/types/index.ts b/src/types/index.ts index 96a27b777..fd2ddb5e8 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -299,6 +299,11 @@ export interface LiveSpecsExtBaseQuery extends LiveSpecsExtBareMinimum { spec: any; } +export interface DefaultAjvResponse { + data: any; + errors: any[]; +} + export interface MarketPlaceVerifyResponse { data: any; error: any; From 7cf225ae89230f89d3df9f0d60bfad82a5f6fbbf Mon Sep 17 00:00:00 2001 From: Travis Jenkins Date: Tue, 14 Jan 2025 09:08:09 -0500 Subject: [PATCH 4/9] Wrapping calls in a try catch to be a bit safer --- src/services/ajv.ts | 34 ++++++++++++++++++++++++++++++++-- src/stores/Binding/Store.ts | 25 +++++++++---------------- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/src/services/ajv.ts b/src/services/ajv.ts index 46bb9639e..0ad0a552a 100644 --- a/src/services/ajv.ts +++ b/src/services/ajv.ts @@ -1,7 +1,10 @@ -import { get_resource_config_pointers } from '@estuary/flow-web'; +import { + get_resource_config_pointers, + update_materialization_resource_spec, +} from '@estuary/flow-web'; import { createAjv } from '@jsonforms/core'; import { isEmpty } from 'lodash'; -import { DefaultAjvResponse, Schema } from 'types'; +import { DefaultAjvResponse, Schema, SourceCaptureDef } from 'types'; import { Annotations } from 'types/jsonforms'; import { stripPathing } from 'utils/misc-utils'; @@ -145,3 +148,30 @@ export const getResourceConfigPointers = ( return null; } }; + +export const updateMaterializationResourceSpec = ( + source_capture: SourceCaptureDef, + resource_spec_pointers: ResourceConfigPointers, + collection_name: string +) => { + try { + // TODO (web flow wasm - source capture) + // We need to do some better error handling here + const response = update_materialization_resource_spec({ + source_capture, + resource_spec: {}, + resource_spec_pointers, + collection_name, + }); + + if (!response) { + return null; + } + + return JSON.parse(response); + + // eslint-disable-next-line @typescript-eslint/no-implicit-any-catch + } catch (e: any) { + return null; + } +}; diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index 23319aa51..e59fbdcbd 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -1,4 +1,3 @@ -import { update_materialization_resource_spec } from '@estuary/flow-web'; import { getLiveSpecsById_writesTo, getLiveSpecsByLiveSpecId, @@ -19,6 +18,7 @@ import { import { createJSONFormDefaults, getResourceConfigPointers, + updateMaterializationResourceSpec, } from 'services/ajv'; import { logRocketEvent } from 'services/shared'; import { stringifyJSON } from 'services/stringify'; @@ -404,29 +404,22 @@ const getInitialState = ( initializeBinding(state, collectionName, bindingUUID); - let prefilledData = {}; - if (sourceCapture) { + let prefilledData; + if (sourceCapture && state.resourceConfigPointers) { // If we have a sourceCapture then we should use those settings to have WASM // produce some default data. This prefills certain settings the same way the // backend would when new bindings are added. - const defaultSchema = - update_materialization_resource_spec({ - source_capture: sourceCapture, - resource_spec: {}, - resource_spec_pointers: - state.resourceConfigPointers, - collection_name: collectionName, - }); - - // TODO (web flow wasm - source capture) - // We need to do some better error handling here - prefilledData = JSON.parse(defaultSchema); + prefilledData = updateMaterializationResourceSpec( + sourceCapture, + state.resourceConfigPointers, + collectionName + ); } const jsonFormDefaults = createJSONFormDefaults( state.resourceSchema, collectionName, - prefilledData + prefilledData ?? {} ); state.resourceConfigs[bindingUUID] = { From ec6c34b6e542b635043b2ee7b3002f63542fa95b Mon Sep 17 00:00:00 2001 From: Travis Jenkins Date: Tue, 14 Jan 2025 14:24:09 -0500 Subject: [PATCH 5/9] Changing variable name as WASM handles these now Renaming functions to make a bit more sense --- src/services/ajv.ts | 16 ++++++++-------- src/stores/Binding/Store.ts | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/services/ajv.ts b/src/services/ajv.ts index 0ad0a552a..2e8fc8681 100644 --- a/src/services/ajv.ts +++ b/src/services/ajv.ts @@ -149,19 +149,19 @@ export const getResourceConfigPointers = ( } }; -export const updateMaterializationResourceSpec = ( - source_capture: SourceCaptureDef, - resource_spec_pointers: ResourceConfigPointers, - collection_name: string +export const generateMaterializationResourceSpec = ( + sourceCapture: SourceCaptureDef, + resourceSpecPointers: ResourceConfigPointers, + collectionName: string ) => { try { // TODO (web flow wasm - source capture) // We need to do some better error handling here const response = update_materialization_resource_spec({ - source_capture, - resource_spec: {}, - resource_spec_pointers, - collection_name, + sourceCapture, + resourceSpecPointers, + collectionName, + resourceSpec: {}, }); if (!response) { diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index e59fbdcbd..e70231420 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -18,7 +18,7 @@ import { import { createJSONFormDefaults, getResourceConfigPointers, - updateMaterializationResourceSpec, + generateMaterializationResourceSpec, } from 'services/ajv'; import { logRocketEvent } from 'services/shared'; import { stringifyJSON } from 'services/stringify'; @@ -409,7 +409,7 @@ const getInitialState = ( // If we have a sourceCapture then we should use those settings to have WASM // produce some default data. This prefills certain settings the same way the // backend would when new bindings are added. - prefilledData = updateMaterializationResourceSpec( + prefilledData = generateMaterializationResourceSpec( sourceCapture, state.resourceConfigPointers, collectionName From a8d70a1456428771cac1a9fae3a7eb708832b881 Mon Sep 17 00:00:00 2001 From: Travis Jenkins Date: Tue, 14 Jan 2025 14:43:46 -0500 Subject: [PATCH 6/9] Making sure all the fields are there. Probably need to work this into rust somehow --- src/services/ajv.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/services/ajv.ts b/src/services/ajv.ts index 2e8fc8681..8ba30dcc3 100644 --- a/src/services/ajv.ts +++ b/src/services/ajv.ts @@ -131,6 +131,24 @@ export interface ResourceConfigPointers { ['x_delta_updates']: string | undefined; } +export const prepareSourceCaptureForServer = (arg: SourceCaptureDef) => { + const response = { + ...arg, + }; + + // These are optional locally and in the spec. + // But when calling WASM we want to make sure they're always there + if (!response.deltaUpdates) { + response.deltaUpdates = false; + } + + if (!response.targetSchema) { + response.targetSchema = 'leaveEmpty'; + } + + return response; +}; + export const getResourceConfigPointers = ( schema: any ): ResourceConfigPointers | null => { @@ -158,10 +176,10 @@ export const generateMaterializationResourceSpec = ( // TODO (web flow wasm - source capture) // We need to do some better error handling here const response = update_materialization_resource_spec({ - sourceCapture, resourceSpecPointers, collectionName, resourceSpec: {}, + sourceCapture: prepareSourceCaptureForServer(sourceCapture), }); if (!response) { From 1d986f5c54bdc7113880602c284f9a3b4edf72d6 Mon Sep 17 00:00:00 2001 From: Travis Jenkins Date: Tue, 14 Jan 2025 15:05:26 -0500 Subject: [PATCH 7/9] Can just pass in the json and the wasm will handle the conversion --- src/stores/Binding/Store.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/stores/Binding/Store.ts b/src/stores/Binding/Store.ts index e70231420..026823a7d 100644 --- a/src/stores/Binding/Store.ts +++ b/src/stores/Binding/Store.ts @@ -21,7 +21,6 @@ import { generateMaterializationResourceSpec, } from 'services/ajv'; import { logRocketEvent } from 'services/shared'; -import { stringifyJSON } from 'services/stringify'; import { CustomEvents } from 'services/types'; import { getStoreWithHydrationSettings } from 'stores/extensions/Hydration'; import { BindingStoreNames } from 'stores/names'; @@ -807,7 +806,7 @@ const getInitialState = ( // anything in a store and probably should never do that... maybe. // Might not be a huge deal to do this twice but something to think about. const resourceConfigPointers = getResourceConfigPointers({ - spec: stringifyJSON(resolved), + spec: resolved, }); if (!resourceConfigPointers) { From 447b6a3fce0b6fd4d1511a203990452b18c33c26 Mon Sep 17 00:00:00 2001 From: Travis Jenkins Date: Thu, 23 Jan 2025 13:52:38 -0500 Subject: [PATCH 8/9] Adding web flow for license Using a more proper term for the command --- .licensee.json | 5 ++++- package.json | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.licensee.json b/.licensee.json index 3ba253cb3..548e21a20 100644 --- a/.licensee.json +++ b/.licensee.json @@ -36,7 +36,10 @@ "spdx-ranges": "2.1.1", "// Public Domain": "0.0.0", - "jsonify": "0.0.1" + "jsonify": "0.0.1", + + "// BSL": "0.0.0", + "flow-web": "0.4.0" }, "ignore": [{ "prefix": "@estuary" }] } diff --git a/package.json b/package.json index d2d1b188f..5f59ef4b5 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "beta_logRocketIntegrity": "node --env-file .env ./scripts/logRocketIntegirty", "generate-flow-types": "cd deps/flow && ./fetch-flow.sh && ./flow-bin/flowctl json-schema | ./flow-bin/flowctl schemalate typescript --name Catalog --hoist-definitions > flow.d.ts", "generate-supabase-types": "node ./scripts/generateSupabaseTypes", - "use-local-web-flow": "./deps/flow-web/hack-in-local-copy.sh" + "hack-in-local-web-flow": "./deps/flow-web/hack-in-local-copy.sh" }, "dependencies": { "@date-fns/utc": "^1.0.0", From 22b5b74436a9ecd9f345d6bd45df3ff1915f6f44 Mon Sep 17 00:00:00 2001 From: Travis Jenkins Date: Thu, 23 Jan 2025 14:31:12 -0500 Subject: [PATCH 9/9] updating message --- deps/flow-web/hack-in-local-copy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/flow-web/hack-in-local-copy.sh b/deps/flow-web/hack-in-local-copy.sh index 91237853a..49023116b 100755 --- a/deps/flow-web/hack-in-local-copy.sh +++ b/deps/flow-web/hack-in-local-copy.sh @@ -2,7 +2,7 @@ # Check if exactly one argument is passed if [ $# -ne 1 ]; then - echo "Please provide the path to flow (ex: /home/travis/code/flow)" + echo "Please provide the path to flow (ex: /home/user/repos/flow)" exit 1 fi