From a3cbbc23164ecb7d06c992f7f8d1eef544fcf2dd Mon Sep 17 00:00:00 2001
From: Julian Bilcke <julian.bilcke@huggingface.co>
Date: Thu, 10 Oct 2024 18:40:24 +0200
Subject: [PATCH] working to add gen3 and kling through fal.ai

---
 .../app/api/resolve/providers/falai/index.ts  | 57 +++++++++++++++++++
 .../workflows/falai/defaultWorkflows.ts       | 39 +++++++++++++
 2 files changed, 96 insertions(+)

diff --git a/packages/app/src/app/api/resolve/providers/falai/index.ts b/packages/app/src/app/api/resolve/providers/falai/index.ts
index 88b943bd..c1ea0b8e 100644
--- a/packages/app/src/app/api/resolve/providers/falai/index.ts
+++ b/packages/app/src/app/api/resolve/providers/falai/index.ts
@@ -298,6 +298,63 @@ export async function resolveSegment(
         }
       }
 
+      segment.assetUrl = result?.video?.url || ''
+    } else if (model === 'fal-ai/kling-video/v1/standard/text-to-video') {
+      if (!request.prompts.image.positive) {
+        throw new Error(`cannot generate a video without a storyboard prompt`)
+      }
+
+      const result = (await fal.run(model, {
+        input: {
+          ...getWorkflowInputValues(request.settings.videoGenerationWorkflow),
+
+          sync_mode: true,
+          enable_safety_checker:
+            request.settings.censorNotForAllAudiencesContent,
+        },
+      })) as FalAiVideoResponse
+
+      if (request.settings.censorNotForAllAudiencesContent) {
+        if (
+          Array.isArray(result.has_nsfw_concepts) &&
+          result.has_nsfw_concepts.includes(true)
+        ) {
+          throw new Error(
+            `The generated content has been filtered according to your safety settings`
+          )
+        }
+      }
+
+      segment.assetUrl = result?.video?.url || ''
+    }  else if (model === 'fal-ai/runway-gen3/turbo/image-to-video') {
+      const hasPromptOrImage = request.prompts.image.positive || request.prompts.video.image
+      if (!hasPromptOrImage) {
+        throw new Error(`cannot generate a video without a storyboard prompt or a storybaord text`)
+      }
+
+      const result = (await fal.run(model, {
+        input: {
+          ...getWorkflowInputValues(request.settings.videoGenerationWorkflow),
+
+          image_url: request.prompts.video.image,
+
+          sync_mode: true,
+          enable_safety_checker:
+            request.settings.censorNotForAllAudiencesContent,
+        },
+      })) as FalAiVideoResponse
+
+      if (request.settings.censorNotForAllAudiencesContent) {
+        if (
+          Array.isArray(result.has_nsfw_concepts) &&
+          result.has_nsfw_concepts.includes(true)
+        ) {
+          throw new Error(
+            `The generated content has been filtered according to your safety settings`
+          )
+        }
+      }
+
       segment.assetUrl = result?.video?.url || ''
     } else {
       throw new Error(
diff --git a/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts b/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts
index a9727bc2..bc0fbb5d 100644
--- a/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts
+++ b/packages/app/src/services/editors/workflow-editor/workflows/falai/defaultWorkflows.ts
@@ -47,6 +47,45 @@ TODO: add those as well
 // -> we can create a ticket to fix this
 // ------------------------------------------------------------------------------
 export const defaultWorkflows: ClapWorkflow[] = [
+  {
+    id: 'falai://fal-ai/kling-video/v1/standard/text-to-video',
+    label: 'Kling 1.0',
+    description: '',
+    tags: ['video generation'],
+    author: '',
+    thumbnailUrl: '',
+    nonCommercial: true,
+
+    engine: ClapWorkflowEngine.REST_API,
+    provider: ClapWorkflowProvider.FALAI,
+    category: ClapWorkflowCategory.VIDEO_GENERATION,
+    data: 'fal-ai/kling-video/v1/standard/text-to-video',
+    schema: '',
+    inputFields: [genericPrompt],
+    inputValues: {
+      [genericPrompt.id]: genericPrompt.defaultValue,
+    },
+  },
+  {
+    id: 'falai://fal-ai/runway-gen3/turbo/image-to-video',
+    label: 'RunwayML Gen 3',
+    description: '',
+    tags: ['video generation'],
+    author: '',
+    thumbnailUrl: '',
+    nonCommercial: true,
+
+    engine: ClapWorkflowEngine.REST_API,
+    provider: ClapWorkflowProvider.FALAI,
+    category: ClapWorkflowCategory.VIDEO_GENERATION,
+    data: 'fal-ai/runway-gen3/turbo/image-to-video',
+    schema: '',
+    inputFields: [genericPrompt, genericImageUrl],
+    inputValues: {
+      [genericPrompt.id]: genericPrompt.defaultValue,
+      [genericImageUrl.id]: genericImageUrl.defaultValue,
+    },
+  },
   {
     id: 'falai://fal-ai/cogvideox-5b',
     label: 'CogVideoX-5b',