v2.18.10

changlog.md

@@ -1,8 +1,13 @@
 # 功能升级日志
 
-#  计划
+#  计划 
 # 2.18.10
 - 🐞 修复：视频按钮提示词，有个小错别字。 #424
+- 🐞 修复：claude-3-5-sonnet-20240620无法上传图片 #425 #427 #430
+- 🐞 修复：LUMA不支持竖屏9:16 这个对于短视频 不友好 官方是支持的 #431
+- 🐞 修复：其实cluade其他模型也是支持识图 #433
+- 😄 新增：LUMA视频 支持 `延展` 每 `延展` `+5s`
+- 😄 修复：LUMA不支持竖屏9:16 这个对于短视频 不友好 官方是支持的 #431
 
 # 2.18.9
 - 🐞 修复：luma download_url bug

package.json

@@ -1,6 +1,6 @@
 {
   "name": "chatgpt-web-midjourney-proxy",
-  "version": "2.18.9",
+  "version": "2.18.10",
   "private": false,
   "description": "ChatGPT Web Midjourney Proxy",
   "author": "Dooy <[email protected]>",
@@ -29,6 +29,8 @@
     "common:prepare": "husky install"
   },
   "dependencies": {
+    "@ffmpeg/ffmpeg": "^0.12.10",
+    "@ffmpeg/util": "^0.12.1",
     "@traptitech/markdown-it-katex": "^3.6.0",
     "@vueuse/core": "^9.13.0",
     "eventsource-parser": "^1.1.1",

src-tauri/tauri.conf.json

@@ -8,7 +8,7 @@
   },
   "package": {
     "productName": "ChatGPT-MJ",
-    "version": "2.18.9"
+    "version": "2.18.10"
   },
   "tauri": {
     "allowlist": {

src/api/mjapi.ts

@@ -368,14 +368,14 @@ export const canVisionModel= (model:string)=>{
     mlog('canVisionModel ',model );
     //['gpt-4-all','gpt-4-v'].indexOf(model)==-1 && model.indexOf('gpt-4-gizmo')==-1
     if( ['gpt-4-all','gpt-4-v','gpt-4v','gpt-3.5-net','gpt-4o-all'].indexOf(model)>-1 ) return true;
-    if(model.indexOf('gpt-4-gizmo')>-1 || model.indexOf('claude-3-opus')>-1 )return true; 
+    if(model.indexOf('gpt-4-gizmo')>-1 )return true;  // || model.indexOf('claude-3-opus')>-1cha
 
     return false;
 }
 export const isCanBase64Model=(model:string)=>{
     //gpt-4o
     //customVisionModel
-    let visionArr=['gemini-pro-vision','gpt-4o','gpt-4o-2024-05-13','gemini-pro-1.5','gpt-4-turbo','gpt-4-turbo-2024-04-09','gpt-4-vision-preview','luma-video','claude-3-5-sonnet-20240620', defaultVisionModel() ]
+    let visionArr=['gemini-pro-vision','gpt-4o','gpt-4o-2024-05-13','gemini-pro-1.5','gpt-4-turbo','gpt-4-turbo-2024-04-09','gpt-4-vision-preview','luma-video','claude-3-5-sonnet-20240620' ,'claude-3-sonnet-20240229','claude-3-opus-20240229', defaultVisionModel() ]
     if( homeStore.myData.session.customVisionModel ){ 
         homeStore.myData.session.customVisionModel.split(/[ ,]+/ig).map( (v:string)=>{
             visionArr.push( v.toLocaleLowerCase() )

src/api/mp4img.ts

@@ -0,0 +1,52 @@
+import { FFmpeg } from '@ffmpeg/ffmpeg'
+import type { LogEvent } from '@ffmpeg/ffmpeg/dist/esm/types'
+import { fetchFile, toBlobURL } from '@ffmpeg/util'
+import { mlog } from './mjapi'
+const baseURL = 'https://unpkg.com/@ffmpeg/[email protected]/dist/esm'
+
+ export async function myTestTranscode( vidoUrl:string) {
+      mlog('vidoUrl>> ', vidoUrl )
+     const ffmpeg = new FFmpeg()
+     //await ffmpeg.load();
+
+      //message.value = 'Loading ffmpeg-core.js'
+      ffmpeg.on('log', ({ message: msg }: LogEvent) => {
+        //message.value = msg
+        mlog('FFmpeg', msg)
+      })
+      mlog('FFmpegs url23 ' )
+      try{
+      await ffmpeg.load({
+        coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, 'text/javascript'),
+        wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, 'application/wasm'),
+        workerURL: await toBlobURL(`${baseURL}/ffmpeg-core.worker.js`, 'text/javascript')
+      })
+      }catch(e){
+         mlog('FFmpegs url3 er ', e  )
+      }
+
+        //await ffmpeg.load();
+        mlog('FFmpegs url3 ' )
+      //message.value = 'Start transcoding'
+      await ffmpeg.writeFile('input.mp4', await fetchFile(vidoUrl))
+      //await ffmpeg.exec(['-i', 'test.avi', 'test.mp4'])
+      await ffmpeg.exec(['-sseof', '-3', '-i', 'input.mp4', '-update', '1', '-q:v', '1', 'last_frame.jpg'])
+      //message.value = 'Complete transcoding'
+      const data = await ffmpeg.readFile( 'last_frame.jpg')
+      //const url = URL.createObjectURL(new Blob([data.buffer], { type: 'image/jpeg' }));
+      const url = URL.createObjectURL(new Blob([(data as Uint8Array).buffer],  { type: 'image/jpeg' } ))
+      mlog('FFmpeg url ', url )
+      // 创建一个临时的<a>元素
+        const a = document.createElement('a');
+        a.href = url;
+        a.download ='last_frame.jpg'; // 设置下载文件的名称
+
+        // 将 <a> 元素添加到 DOM，然后触发点击事件以启动下载
+        document.body.appendChild(a);
+        a.click();
+
+        // 触发下载后，从 DOM 中移除 <a> 元素，并释放 URL 对象
+        document.body.removeChild(a);
+        URL.revokeObjectURL(url);
+
+    }

src/locales/en-US.ts

@@ -429,7 +429,12 @@ export default {
     plsInput: "Please input content!",
     submitSuccess: "Submitted successfully!",
     process: "Video generating...",
-    repeat: "Get again"
+    repeat: "Get again",
+    pending: 'Status: Queued',
+    processing: 'Status: Processing',
+    download: 'Download',
+    extend: 'Extend'
+
   }
 
 

src/locales/zh-CN.ts

@@ -437,6 +437,12 @@ export default {
     submitSuccess:'已提交成功！',
     process:'视频生成中...',
     repeat:'重新获取',
+
+    pending:'状态:队列中',
+    processing:'状态:队列中',
+    download:'下载',
+    extend:'延展',
+
   }
 
 

src/views/luma/voInput.vue

@@ -1,19 +1,31 @@
 <script setup lang="ts">
-import { ref, computed, onMounted } from 'vue';
-import { NInput,NButton, useMessage,NTag } from 'naive-ui';
+import { ref, computed, onMounted, watch } from 'vue';
+import { NInput,NButton, useMessage,NTag ,NPopover} from 'naive-ui';
 import {SvgIcon} from '@/components/common'
 import { FeedLumaTask, lumaFetch, mlog, upImg } from '@/api';
 import { homeStore } from '@/store';
 import { t } from '@/locales';
+import { LumaMedia } from '@/api/lumaStore';
 
 const luma= ref({ "aspect_ratio": "16:9", "expand_prompt": true,  "image_url": "",  "user_prompt": "" });
 const st= ref({isDo:false})
 const ms = useMessage();
 const fsRef= ref() ;
+const exLuma= ref<LumaMedia>()
+
+const vf=[{s:'width: 100%; height: 100%;',label:'1:1'}
+,{s:'width: 100%; height: 75%;',label:'4:3'}
+,{s:'width: 75%; height: 100%;',label:'3:4'}
+,{s:'width: 100%; height: 50%;',label:'16:9'}
+,{s:'width: 50%; height: 100%;',label:'9:16'}
+ ];
+
 onMounted(() => {
     homeStore.setMyData({ms:ms})
 });
 
+
+
 const canPost = computed(() => {
     return luma.value.user_prompt!='' && !st.value.isDo
 })
@@ -25,7 +37,9 @@ const generate= async ()=>{
         return ;
     }
     try{
-        const d:any=  await lumaFetch('/generations/', luma.value);
+        let url= '/generations/';
+        if(exLuma.value) url= `/generations/${exLuma.value.id}/extend`
+        const d:any=  await lumaFetch(url, luma.value);
         mlog("d", d )
         if(d.id ) FeedLumaTask(d.id )
         else FeedLumaTask(d[0].id )
@@ -49,17 +63,55 @@ function selectFile(input:any){
 const clearInput = ()=>{
     luma.value.user_prompt= ''
     luma.value.image_url= ''
+    exLuma.value= undefined
 }
- 
+
+//luma.extend
+watch(()=>homeStore.myData.act, (n)=>{
+    if(n=='luma.extend'){
+        mlog("luma.extend", homeStore.myData.actData )
+        const s= homeStore.myData.actData as LumaMedia
+        exLuma.value= s 
+        // cs.value.continue_clip_id= s.id
+        // cs.value.continue_at= Math.ceil(s.metadata.duration/2) 
+    }
+});
 </script>
 
 <template>
 <div class="p-2"> 
-    <div>
+    <div class=" flex items-center justify-between space-x-1">
+            <template  v-for="(item,index) in vf" >
+            <section class="aspect-item flex-1 rounded border-2 dark:border-neutral-700 cursor-pointer"  :class="{'active':luma.aspect_ratio==item.label}"  @click="luma.aspect_ratio=item.label">
+                <div class="aspect-box-wrapper mx-auto my-2 flex h-5 w-5 items-center justify-center">
+                    <div class="aspect-box rounded border-2 dark:border-neutral-700" :style="item.s"></div>
+                </div>
+                <p class="mb-1 text-center text-sm">{{ item.label }}</p>
+            </section>
+            </template>
+
+    </div>
+    <div class="pt-1">
       <n-input v-model:value="luma.user_prompt" 
                 :placeholder="$t('video.descpls')"  type="textarea"  size="small"   
                 :autosize="{ minRows: 3, maxRows: 12  }"  />
     </div>
+    <div v-if="exLuma" class="pt-1">
+        <div class="flex justify-between items-center">
+            <div  >
+                <n-popover trigger="hover">
+                    <template #trigger>
+                    <div class="line-clamp-1">{{ $t('video.extend') }}:{{exLuma.prompt}}</div>
+                    </template>
+                    <div class=" max-w-[300px]">{{exLuma.prompt}}</div>
+                </n-popover>
+            </div>
+        </div>
+        <div class="relative flex items-center justify-center bg-white bg-opacity-10 rounded-[5px] overflow-hidden aspect-[16/8.85] ">
+                <video v-if="exLuma.video?.url|| exLuma.video?.download_url" :src="exLuma.video?.download_url? exLuma.video?.download_url:exLuma.video?.url" @error="$event.target.src=exLuma.video?.url" loop  playsinline  controls class="w-full h-full object-cover"></video>    
+        </div>
+
+    </div>
 
     <div class="pt-1">
         <div class="flex justify-between  items-end">
@@ -72,7 +124,7 @@ const clearInput = ()=>{
             </div>
             <div>
                 <div class="pb-1 text-right">
-                  <NTag v-if="luma.user_prompt!=''||luma.image_url!=''" type="success" size="small" round  ><span class="cursor-pointer" @click="clearInput()" >{{$t('video.clear')}}</span></NTag>
+                  <NTag v-if=" exLuma|| luma.user_prompt!=''||luma.image_url!=''" type="success" size="small" round  ><span class="cursor-pointer" @click="clearInput()" >{{$t('video.clear')}}</span></NTag>
                 </div>
                 <div>
                     <NButton :loading="st.isDo" type="primary" :disabled="!canPost" @click="generate()"><SvgIcon icon="ri:video-add-line"  /> {{$t('video.generate')}}</NButton> 

src/views/luma/voList.vue

@@ -1,10 +1,12 @@
 <script setup lang="ts">
 import { LumaMedia, lumaStore } from '@/api/lumaStore';
 import { computed, ref, watch } from 'vue';
-import {NEmpty ,NButton,NPopover} from 'naive-ui'
-import {FeedLumaTask} from '@/api';
+import {NEmpty ,NButton,NPopover, NTag,NButtonGroup } from 'naive-ui'
+import {FeedLumaTask, lumaFetch, mlog} from '@/api';
 import { homeStore } from '@/store';
 import {SvgIcon} from '@/components/common'
+import { myTestTranscode } from '@/api/mp4img';
+//import { myTestTranscode } from '@/api/mp4img';
 
 const st= ref({pIndex:-1});
 const list= ref<LumaMedia[]>([]);
@@ -16,6 +18,31 @@ const initLoad=()=>{
 const nowTime= computed(()=>{
     return new Date().getTime()
 })
+const FeedLumaTaskDown=async (id:string)=>{
+    //FeedLumaTask(id)
+    let d:any= await lumaFetch('/generations/'+id+'/download_video_url' );
+    mlog("d", d )
+    if(d.url) {
+        //window.open(d.url)
+        const link = document.createElement('a');
+        link.href =d.url;
+        link.download = id+".mp4";
+        link.target = '_blank';
+        document.body.appendChild(link);
+        link.click();
+        document.body.removeChild(link);
+    }
+}
+
+//getLastFrameBase64
+const extend= async(item:LumaMedia )=>{
+    
+    mlog("extend ", item ) 
+    homeStore.setMyData({act:"luma.extend", actData: item  })
+
+}
+
+
 watch(()=>homeStore.myData.act, (n)=>{
      if(n=='FeedLumaTask')  initLoad() 
 });
@@ -27,11 +54,15 @@ initLoad();
     <div  class="grid grid-cols-1 gap-5 md:grid-cols-2 lg:grid-cols-3">
         <div v-for="(item, index) in list" :key="index" class="relative" @mousemove="st.pIndex=index" @mouseout="st.pIndex=-1">
             <div class="relative flex items-center justify-center bg-white bg-opacity-10 rounded-[16px] overflow-hidden aspect-[16/8.85] ">
-                <video v-if="item.video?.url|| item.video?.download_url" :src="item.video?.download_url? item.video?.download_url:item.video?.url" loop  playsinline  :controls="st.pIndex==index" class="w-full h-full object-cover"></video>
+                <video v-if="item.video?.url|| item.video?.download_url" :src="item.video?.download_url? item.video?.download_url:item.video?.url" @error="$event.target.src=item.video?.url" loop  playsinline  :controls="st.pIndex==index" class="w-full h-full object-cover"></video>
                 <div class=" text-center" v-else>
 
                     <NButton  size="small" type="primary" @click="FeedLumaTask( item.id )"   v-if="!item.last_feed|| ((new Date().getTime())-item.last_feed)>20*1000" >{{$t('video.repeat')}}</NButton>
-                    <div class="pt-2" v-else>{{$t('video.process')}}{{ new Date(item.last_feed).toLocaleString() }}</div>
+                    <div class="pt-2" v-else>
+                        <div>{{$t('video.process')}}{{ new Date(item.last_feed).toLocaleString() }}</div>
+                        <div v-if="item.state=='pending'">{{ $t('video.pending') }}</div> 
+                        <div v-if="item.state=='processing'">{{ $t('video.processing') }}</div> 
+                    </div>
 
                 </div>
             </div>
@@ -45,8 +76,15 @@ initLoad();
                 </n-popover>
 
                 </div>
-                <div> 
-                     <a :href="item.video?.download_url? item.video?.download_url:item.video?.url" download  target="_blank" v-if="item.video?.url|| item.video?.download_url"  ><SvgIcon icon="mdi:download" class="cursor-pointer"/></a>
+                <div class="flex justify-end items-center pt-1"  v-if="item.video?.url|| item.video?.download_url"> 
+                     <!-- <span    @click="FeedLumaTaskDown( item.id )" class="cursor-pointer" ><SvgIcon icon="mdi:download" /></span> -->
+
+
+                      <n-button-group size="tiny">
+                        <n-button  size="tiny" round ghost   @click="FeedLumaTaskDown( item.id )"  ><SvgIcon icon="mdi:download" /> {{ $t('video.download') }}</n-button>
+                        <n-button   size="tiny"  round ghost  @click="extend( item )"  ><SvgIcon icon="ri:video-add-line" /> {{ $t('video.extend') }}</n-button>
+                      </n-button-group>
+                     <!-- <a :href="item.video?.download_url? item.video?.download_url:item.video?.url" download  target="_blank" v-if="item.video?.url|| item.video?.download_url"  ><SvgIcon icon="mdi:download" class="cursor-pointer"/></a> -->
                 </div>
             </div>
         </div>