diff --git a/api/video_flow.ts b/api/video_flow.ts index 9029be2..7cbbc58 100644 --- a/api/video_flow.ts +++ b/api/video_flow.ts @@ -292,11 +292,14 @@ export const applyRoleToShots = async (request: { character_replacements: CharacterReplacement[]; /** 是否等待完成 */ wait_for_completion?: boolean; + /** 数据缓存 */ + character_draft: string; }): Promise { if (response.successful) { console.log(`分镜 ${shot.id} 角色替换成功:`, response.data); diff --git a/app/service/Interaction/ShotService.ts b/app/service/Interaction/ShotService.ts index de32f96..b0e7c4d 100644 --- a/app/service/Interaction/ShotService.ts +++ b/app/service/Interaction/ShotService.ts @@ -1,5 +1,9 @@ import { useState, useCallback, useEffect } from "react"; -import { VideoSegmentEditUseCase } from "../usecase/ShotEditUsecase"; +import { + MatchedPerson, + RoleRecognitionResponse, + VideoSegmentEditUseCase, +} from "../usecase/ShotEditUsecase"; import { VideoSegmentEntity } from "../domain/Entities"; import { LensType, SimpleCharacter } from "../domain/valueObject"; import { getUploadToken, uploadToQiniu } from "@/api/common"; @@ -16,15 +20,15 @@ export interface UseShotService { videoSegments: VideoSegmentEntity[]; /** 当前选中的视频片段 */ selectedSegment: VideoSegmentEntity | null; - + /** 识别出的人物信息 */ + matched_persons: MatchedPerson[]; // 操作方法 /** 获取视频片段列表 */ getVideoSegmentList: (projectId: string) => Promise; /** 重新生成视频片段 */ - regenerateVideoSegment: ( - // roleReplaceParams?: { oldId: string; newId: string }[], - // sceneReplaceParams?: { oldId: string; newId: string }[] - ) => Promise; + regenerateVideoSegment: () => // roleReplaceParams?: { oldId: string; newId: string }[], + // sceneReplaceParams?: { oldId: string; newId: string }[] + Promise; /** AI优化视频内容 */ optimizeVideoContent: ( shotId: string, @@ -40,9 +44,19 @@ export interface UseShotService { /** 删除指定镜头 */ deleteLens: (lensName: string) => void; /** 获取视频当前帧并上传到七牛云 */ - filterRole: (video: HTMLVideoElement) => Promise; + filterRole: ( + video: HTMLVideoElement + ) => Promise; /** 设置角色简单数据 */ setSimpleCharacter: (characters: SimpleCharacter[]) => void; + /** 计算识别框 */ + calculateRecognitionBoxes: (containerElement: HTMLElement) => Array<{ + left: number; + top: number; + width: number; + height: number; + person_id: string; + }>; } /** @@ -54,9 +68,11 @@ export const useShotService = (): UseShotService => { // 响应式状态 const [loading, setLoading] = useState(false); const [videoSegments, setVideoSegments] = useState([]); - const [selectedSegment, setSelectedSegment] = useState(null); + const [selectedSegment, setSelectedSegment] = + useState(null); const [projectId, setProjectId] = useState(""); const [simpleCharacter, setSimpleCharacter] = useState([]); + const [matched_persons, setMatched_persons] = useState([]); // 轮询任务ID const [intervalId, setIntervalId] = useState(null); // UseCase实例 @@ -87,7 +103,6 @@ export const useShotService = (): UseShotService => { ); const setIntervalIdHandler = async (projectId: string): Promise => { - // 每次执行前先清除之前的定时器,确保只存在一个定时器 if (intervalId) { clearInterval(intervalId); @@ -98,11 +113,15 @@ export const useShotService = (): UseShotService => { try { const segments = await vidoEditUseCase.getVideoSegmentList(projectId); - setVideoSegments(prevSegments => { - const existingSegmentsMap = new Map(prevSegments.map(segment => [segment.id, segment])); - const segmentsToUpdate = segments.filter(segment => segment.id !== selectedSegment?.id); + setVideoSegments((prevSegments) => { + const existingSegmentsMap = new Map( + prevSegments.map((segment) => [segment.id, segment]) + ); + const segmentsToUpdate = segments.filter( + (segment) => segment.id !== selectedSegment?.id + ); - segmentsToUpdate.forEach(newSegment => { + segmentsToUpdate.forEach((newSegment) => { const existingSegment = existingSegmentsMap.get(newSegment.id); if (existingSegment) { @@ -113,7 +132,7 @@ export const useShotService = (): UseShotService => { sketchUrl: newSegment.sketchUrl, lens: newSegment.lens, updatedAt: newSegment.updatedAt, - loadingProgress: newSegment.loadingProgress + loadingProgress: newSegment.loadingProgress, }); } else { existingSegmentsMap.set(newSegment.id, newSegment); @@ -123,7 +142,7 @@ export const useShotService = (): UseShotService => { return Array.from(existingSegmentsMap.values()); }); } catch (error) { - console.error('定时获取视频片段列表失败:', error); + console.error("定时获取视频片段列表失败:", error); } }, 5000); @@ -148,22 +167,21 @@ export const useShotService = (): UseShotService => { * @param sceneReplaceParams 场景替换参数(可选) * @returns Promise 重新生成的视频片段 */ - const regenerateVideoSegment = useCallback( - async ( - ): Promise => { + const regenerateVideoSegment = + useCallback(async (): Promise => { try { setLoading(true); const regeneratedSegment = await vidoEditUseCase.regenerateVideoSegment( projectId, selectedSegment!.lens, - selectedSegment!.id, + selectedSegment!.id ); // 如果重新生成的是现有片段,更新列表中的对应项 if (selectedSegment) { - setVideoSegments(prev => - prev.map(segment => + setVideoSegments((prev) => + prev.map((segment) => segment.id === selectedSegment.id ? regeneratedSegment : segment ) ); @@ -176,9 +194,7 @@ export const useShotService = (): UseShotService => { } finally { setLoading(false); } - }, - [projectId, selectedSegment, vidoEditUseCase] - ); + }, [projectId, selectedSegment, vidoEditUseCase]); /** * AI优化视频内容 @@ -226,9 +242,12 @@ export const useShotService = (): UseShotService => { /** * 设置选中的视频片段 */ - const setSelectedSegmentHandler = useCallback((segment: VideoSegmentEntity | null): void => { - setSelectedSegment(segment); - }, []); + const setSelectedSegmentHandler = useCallback( + (segment: VideoSegmentEntity | null): void => { + setSelectedSegment(segment); + }, + [] + ); /** * 添加新镜头到选中的视频片段 @@ -250,13 +269,15 @@ export const useShotService = (): UseShotService => { // 创建更新后的片段 const updatedSegment: VideoSegmentEntity = { ...selectedSegment, - lens: [...selectedSegment.lens, newLens] + lens: [...selectedSegment.lens, newLens], }; // 批量更新状态,避免多次重渲染 setSelectedSegment(updatedSegment); - setVideoSegments(prev => { - const segmentIndex = prev.findIndex(segment => segment.id === selectedSegment.id); + setVideoSegments((prev) => { + const segmentIndex = prev.findIndex( + (segment) => segment.id === selectedSegment.id + ); if (segmentIndex === -1) return prev; const newSegments = [...prev]; @@ -269,34 +290,42 @@ export const useShotService = (): UseShotService => { * 删除指定镜头 * @param lensName 要删除的镜头名称 */ - const deleteLens = useCallback((lensName: string): void => { - if (!selectedSegment) { - console.warn("没有选中的视频片段,无法删除镜头"); - return; - } + const deleteLens = useCallback( + (lensName: string): void => { + if (!selectedSegment) { + console.warn("没有选中的视频片段,无法删除镜头"); + return; + } - // 过滤掉指定名称的镜头并重新排序 - const updatedLens = selectedSegment.lens - .filter(lens => lens.name !== lensName) - .map((lens, index) => new LensType(`镜头${index + 1}`, lens.script, lens.content)); + // 过滤掉指定名称的镜头并重新排序 + const updatedLens = selectedSegment.lens + .filter((lens) => lens.name !== lensName) + .map( + (lens, index) => + new LensType(`镜头${index + 1}`, lens.script, lens.content) + ); - // 创建更新后的片段 - const updatedSegment: VideoSegmentEntity = { - ...selectedSegment, - lens: updatedLens - }; + // 创建更新后的片段 + const updatedSegment: VideoSegmentEntity = { + ...selectedSegment, + lens: updatedLens, + }; - // 批量更新状态,避免多次重渲染 - setSelectedSegment(updatedSegment); - setVideoSegments(prev => { - const segmentIndex = prev.findIndex(segment => segment.id === selectedSegment.id); - if (segmentIndex === -1) return prev; + // 批量更新状态,避免多次重渲染 + setSelectedSegment(updatedSegment); + setVideoSegments((prev) => { + const segmentIndex = prev.findIndex( + (segment) => segment.id === selectedSegment.id + ); + if (segmentIndex === -1) return prev; - const newSegments = [...prev]; - newSegments[segmentIndex] = updatedSegment; - return newSegments; - }); - }, [selectedSegment]); + const newSegments = [...prev]; + newSegments[segmentIndex] = updatedSegment; + return newSegments; + }); + }, + [selectedSegment] + ); /** * 获取视频当前帧的画面,上传到七牛云,并返回七牛云的图片地址,然后调用接口识别出里面的人物信息,返回人物信息 @@ -305,69 +334,130 @@ export const useShotService = (): UseShotService => { * @param videoId 视频ID * @returns Promise 七牛云的图片地址 */ - const filterRole = useCallback(async ( - video: HTMLVideoElement, - ) => { - try { - // 创建canvas元素来截取视频帧 - const canvas = document.createElement('canvas'); - const ctx = canvas.getContext('2d'); - console.log(video); - video.crossOrigin = 'anonymous'; - if (!ctx) { - throw new Error('无法获取canvas上下文'); - } + const filterRole = useCallback( + async (video: HTMLVideoElement) => { + try { + // 创建canvas元素来截取视频帧 + const canvas = document.createElement("canvas"); + const ctx = canvas.getContext("2d"); + console.log(video); + video.crossOrigin = "anonymous"; + if (!ctx) { + throw new Error("无法获取canvas上下文"); + } - // 设置canvas尺寸为视频尺寸 - canvas.width = video.videoWidth; - canvas.height = video.videoHeight; + // 设置canvas尺寸为视频尺寸 + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; - // 将当前视频帧绘制到canvas上 - ctx.drawImage(video, 0, 0, canvas.width, canvas.height); + // 将当前视频帧绘制到canvas上 + ctx.drawImage(video, 0, 0, canvas.width, canvas.height); - // 将canvas转换为blob - const blob = await new Promise((resolve, reject) => { - canvas.toBlob((blob) => { - if (blob) { - resolve(blob); - } else { - reject(new Error('无法将canvas转换为blob')); - } - }, 'image/png'); - }); + // 将canvas转换为blob + const blob = await new Promise((resolve, reject) => { + canvas.toBlob((blob) => { + if (blob) { + resolve(blob); + } else { + reject(new Error("无法将canvas转换为blob")); + } + }, "image/png"); + }); - // 创建File对象 - const file = new File([blob], `frame_${Date.now()}.png`, { type: 'image/png' }); + // 创建File对象 + const file = new File([blob], `frame_${Date.now()}.png`, { + type: "image/png", + }); - // 获取上传token - const { token } = await getUploadToken(); + // 获取上传token + const { token } = await getUploadToken(); - // 上传到七牛云 - const imageUrl = await uploadToQiniu(file, token); - // 调用用例中的识别角色方法 - if (vidoEditUseCase) { + // 上传到七牛云 + const imageUrl = await uploadToQiniu(file, token); + // 调用用例中的识别角色方法 try { - const recognitionResult = await vidoEditUseCase.recognizeRoleFromImage( - projectId, - selectedSegment!.id, - imageUrl + const recognitionResult = + await vidoEditUseCase.recognizeRoleFromImage( + projectId, + selectedSegment!.id, + imageUrl + ); + console.log("角色识别结果:", recognitionResult); + setMatched_persons( + recognitionResult.recognition_result.data.matched_persons ); - console.log('角色识别结果:', recognitionResult); return recognitionResult; } catch (recognitionError) { - console.warn('角色识别失败,但图片上传成功:', recognitionError); + console.warn("角色识别失败,但图片上传成功:", recognitionError); } + } catch (error) { + console.error("获取视频帧失败:", error); + throw error; } - } catch (error) { - console.error('获取视频帧失败:', error); - throw error; - } - }, [projectId, selectedSegment, vidoEditUseCase]); + }, + [projectId, selectedSegment, vidoEditUseCase] + ); + + /** + * 计算识别框的属性 + * @description 根据DOM元素尺寸和匹配数据计算识别框的位置和尺寸 + * @param containerElement DOM容器元素 + * @returns 计算后的识别框属性数组 + */ + const calculateRecognitionBoxes = ( + containerElement: HTMLElement + ): Array<{ + /** 横向定位坐标 */ + left: number; + /** 纵向定位坐标 */ + top: number; + /** 宽度 */ + width: number; + /** 高度 */ + height: number; + /** 人物ID */ + person_id: string; + }> => { + // 获取容器元素的尺寸 + const containerRect = containerElement.getBoundingClientRect(); + const containerWidth = containerRect.width; + const containerHeight = containerRect.height; + + // 计算识别框属性 + return matched_persons + .map((person) => { + // 取出bbox信息 + const bbox = (person as any).bbox; + if (!bbox) return null; + + // 计算绝对坐标和尺寸(百分比转像素) + const left = (bbox.x || 0) * containerWidth; + const top = (bbox.y || 0) * containerHeight; + const width = (bbox.width || 0) * containerWidth; + const height = (bbox.height || 0) * containerHeight; + + return { + left, + top, + width, + height, + person_id: person.person_id + }; + }) + .filter(Boolean) as Array<{ + left: number; + top: number; + width: number; + height: number; + person_id: string; + }>; + }; return { // 响应式状态 loading, videoSegments, selectedSegment, + matched_persons, // 操作方法 getVideoSegmentList, regenerateVideoSegment, @@ -378,5 +468,6 @@ export const useShotService = (): UseShotService => { deleteLens, filterRole, setSimpleCharacter, + calculateRecognitionBoxes }; }; diff --git a/app/service/usecase/RoleEditUseCase.ts b/app/service/usecase/RoleEditUseCase.ts index 286b146..2cd8e85 100644 --- a/app/service/usecase/RoleEditUseCase.ts +++ b/app/service/usecase/RoleEditUseCase.ts @@ -24,6 +24,8 @@ export interface RoleResponse { highlights: string[]; /** 角色图片地址 */ image_path: string; + /**缓存 */ + character_draft: string; } /** @@ -73,6 +75,7 @@ export class RoleEditUseCase { const characters = newCharacterData.data || []; return characters.map((char, index) => { + const roleEntity: RoleEntity = { id: `role_${index + 1}`, name: char.character_name || '', @@ -101,6 +104,10 @@ export class RoleEditUseCase { } return projectRoleData.map((char, index) => { + if(char.character_draft){ + const roleEntity: RoleEntity = JSON.parse(char.character_draft); + return roleEntity; + } /** 角色实体对象 */ const roleEntity: RoleEntity = { id: `role_${index + 1}`, diff --git a/app/service/usecase/ShotEditUsecase.ts b/app/service/usecase/ShotEditUsecase.ts index ba4a7eb..a2013d5 100644 --- a/app/service/usecase/ShotEditUsecase.ts +++ b/app/service/usecase/ShotEditUsecase.ts @@ -320,7 +320,7 @@ export class VideoSegmentEditUseCase { projectId: string, videoId: string, targetImageUrl: string - ): Promise { + ) { try { this.loading = true; @@ -371,12 +371,16 @@ export interface BoundingBox { * @description 从图片中识别出的与已知角色匹配的人物信息 */ export interface MatchedPerson { - /** 人物ID */ + /** 人物名 */ person_id: string; - /** 边界框信息 */ - bbox: BoundingBox; - /** 匹配置信度(0-1之间) */ - confidence: number; + /**x坐标 小数百分比形式 */ + x: number; + /**y坐标 小数百分比形式 */ + y: number; + /**宽度 小数百分比形式 */ + width: number; + /**高度 小数百分比形式 */ + height: number; } /** @@ -439,38 +443,38 @@ export interface RoleRecognitionResponse { characters_used: CharacterUsed[]; } -export const roleRecognitionResponse:RoleRecognitionResponse = { - "project_id": "d0df7120-e27b-4f84-875c-e532f1bd318c", - "video_id": "984f3347-c81c-4af8-9145-49ead82becde", - "target_image_url": "https://cdn.qikongjian.com/videos/1754970412744_kqxplx.png", - "known_persons_count": 1, - "recognition_result": { - "code": 200, - "message": "识别完成", - "data": - { - "target_image_url": "https://cdn.qikongjian.com/videos/1754970412744_kqxplx.png", - "total_persons_detected": 1, - "matched_persons": [ - { - "person_id": "CH-01", - "bbox": { - "x": 269, - "y": 23, - "width": 585, - "height": 685 - }, - "confidence": 0.36905956268310547 - } - ] - } - }, - "characters_used": [ - { - "character_name": "CH-01", - "c_id": "无C-ID", - "image_path": "无image_path", - "avatar": "https://cdn.huiying.video/template/Whisk_9afb196368.jpg" - } - ] -} +// export const roleRecognitionResponse:RoleRecognitionResponse = { +// "project_id": "d0df7120-e27b-4f84-875c-e532f1bd318c", +// "video_id": "984f3347-c81c-4af8-9145-49ead82becde", +// "target_image_url": "https://cdn.qikongjian.com/videos/1754970412744_kqxplx.png", +// "known_persons_count": 1, +// "recognition_result": { +// "code": 200, +// "message": "识别完成", +// "data": +// { +// "target_image_url": "https://cdn.qikongjian.com/videos/1754970412744_kqxplx.png", +// "total_persons_detected": 1, +// "matched_persons": [ +// { +// "person_id": "CH-01", +// "bbox": { +// "x": 269, +// "y": 23, +// "width": 585, +// "height": 685 +// }, +// "confidence": 0.36905956268310547 +// } +// ] +// } +// }, +// "characters_used": [ +// { +// "character_name": "CH-01", +// "c_id": "无C-ID", +// "image_path": "无image_path", +// "avatar": "https://cdn.huiying.video/template/Whisk_9afb196368.jpg" +// } +// ] +// }