xueteng/video-flow-b
1
0
Fork 0
forked from 77media/video-flow
Code Pull Requests Activity

修复问题

Browse Source
This commit is contained in:
海龙 2025-08-13 02:08:08 +08:00
parent 63e63a52c0
commit cf388d0bb7
5 changed files with 248 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/video_flow.ts
View File

@ -292,11 +292,14 @@ export const applyRoleToShots = async (request: {
character_replacements: CharacterReplacement[]; character_replacements: CharacterReplacement[];
/** 是否等待完成 */ /** 是否等待完成 */
wait_for_completion?: boolean; wait_for_completion?: boolean;
/** 数据缓存 */
character_draft: string;
}): Promise<ApiResponse<{ }): Promise<ApiResponse<{
/** 应用成功的分镜数量 */ /** 应用成功的分镜数量 */
success_count: number; success_count: number;
/** 应用失败的分镜数量 */ /** 应用失败的分镜数量 */
failed_count: number; failed_count: number;
/** 应用结果详情 */ /** 应用结果详情 */
results: Array<{ results: Array<{
shot_id: string; shot_id: string;

3
app/service/Interaction/RoleShotService.ts
View File

@ -180,7 +180,8 @@ export const useRoleShotServiceHook = (projectId: string,selectRole?:RoleEntity)
project_id: projectId, project_id: projectId,
shot_id: shot.id, // 单个分镜ID shot_id: shot.id, // 单个分镜ID
character_replacements: characterReplacements, character_replacements: characterReplacements,
wait_for_completion: false // 不等待完成，异步处理 wait_for_completion: false, // 不等待完成，异步处理
character_draft: JSON.stringify(selectedRole)
}).then(response => { }).then(response => {
if (response.successful) { if (response.successful) {
console.log(`分镜 ${shot.id} 角色替换成功:`, response.data); console.log(`分镜 ${shot.id} 角色替换成功:`, response.data);

291
app/service/Interaction/ShotService.ts
View File

@ -1,5 +1,9 @@
import { useState, useCallback, useEffect } from "react"; import { useState, useCallback, useEffect } from "react";
import { VideoSegmentEditUseCase } from "../usecase/ShotEditUsecase"; import {
MatchedPerson,
RoleRecognitionResponse,
VideoSegmentEditUseCase,
} from "../usecase/ShotEditUsecase";
import { VideoSegmentEntity } from "../domain/Entities"; import { VideoSegmentEntity } from "../domain/Entities";
import { LensType, SimpleCharacter } from "../domain/valueObject"; import { LensType, SimpleCharacter } from "../domain/valueObject";
import { getUploadToken, uploadToQiniu } from "@/api/common"; import { getUploadToken, uploadToQiniu } from "@/api/common";
@ -16,15 +20,15 @@ export interface UseShotService {
videoSegments: VideoSegmentEntity[]; videoSegments: VideoSegmentEntity[];
/** 当前选中的视频片段 */ /** 当前选中的视频片段 */
selectedSegment: VideoSegmentEntity | null; selectedSegment: VideoSegmentEntity | null;
/** 识别出的人物信息 */
matched_persons: MatchedPerson[];
// 操作方法 // 操作方法
/** 获取视频片段列表 */ /** 获取视频片段列表 */
getVideoSegmentList: (projectId: string) => Promise<void>; getVideoSegmentList: (projectId: string) => Promise<void>;
/** 重新生成视频片段 */ /** 重新生成视频片段 */
regenerateVideoSegment: ( regenerateVideoSegment: () => // roleReplaceParams?: { oldId: string; newId: string }[],
// roleReplaceParams?: { oldId: string; newId: string }[], // sceneReplaceParams?: { oldId: string; newId: string }[]
// sceneReplaceParams?: { oldId: string; newId: string }[] Promise<VideoSegmentEntity>;
) => Promise<VideoSegmentEntity>;
/** AI优化视频内容 */ /** AI优化视频内容 */
optimizeVideoContent: ( optimizeVideoContent: (
shotId: string, shotId: string,
@ -40,9 +44,19 @@ export interface UseShotService {
/** 删除指定镜头 */ /** 删除指定镜头 */
deleteLens: (lensName: string) => void; deleteLens: (lensName: string) => void;
/** 获取视频当前帧并上传到七牛云 */ /** 获取视频当前帧并上传到七牛云 */
filterRole: (video: HTMLVideoElement) => Promise<string>; filterRole: (
video: HTMLVideoElement
) => Promise<RoleRecognitionResponse | undefined>;
/** 设置角色简单数据 */ /** 设置角色简单数据 */
setSimpleCharacter: (characters: SimpleCharacter[]) => void; setSimpleCharacter: (characters: SimpleCharacter[]) => void;
/** 计算识别框 */
calculateRecognitionBoxes: (containerElement: HTMLElement) => Array<{
left: number;
top: number;
width: number;
height: number;
person_id: string;
}>;
} }
/** /**
@ -54,9 +68,11 @@ export const useShotService = (): UseShotService => {
// 响应式状态 // 响应式状态
const [loading, setLoading] = useState<boolean>(false); const [loading, setLoading] = useState<boolean>(false);
const [videoSegments, setVideoSegments] = useState<VideoSegmentEntity[]>([]); const [videoSegments, setVideoSegments] = useState<VideoSegmentEntity[]>([]);
const [selectedSegment, setSelectedSegment] = useState<VideoSegmentEntity | null>(null); const [selectedSegment, setSelectedSegment] =
useState<VideoSegmentEntity | null>(null);
const [projectId, setProjectId] = useState<string>(""); const [projectId, setProjectId] = useState<string>("");
const [simpleCharacter, setSimpleCharacter] = useState<SimpleCharacter[]>([]); const [simpleCharacter, setSimpleCharacter] = useState<SimpleCharacter[]>([]);
const [matched_persons, setMatched_persons] = useState<MatchedPerson[]>([]);
// 轮询任务ID // 轮询任务ID
const [intervalId, setIntervalId] = useState<NodeJS.Timeout | null>(null); const [intervalId, setIntervalId] = useState<NodeJS.Timeout | null>(null);
// UseCase实例 // UseCase实例
@ -87,7 +103,6 @@ export const useShotService = (): UseShotService => {
); );
const setIntervalIdHandler = async (projectId: string): Promise<void> => { const setIntervalIdHandler = async (projectId: string): Promise<void> => {
// 每次执行前先清除之前的定时器，确保只存在一个定时器 // 每次执行前先清除之前的定时器，确保只存在一个定时器
if (intervalId) { if (intervalId) {
clearInterval(intervalId); clearInterval(intervalId);
@ -98,11 +113,15 @@ export const useShotService = (): UseShotService => {
try { try {
const segments = await vidoEditUseCase.getVideoSegmentList(projectId); const segments = await vidoEditUseCase.getVideoSegmentList(projectId);
setVideoSegments(prevSegments => { setVideoSegments((prevSegments) => {
const existingSegmentsMap = new Map(prevSegments.map(segment => [segment.id, segment])); const existingSegmentsMap = new Map(
const segmentsToUpdate = segments.filter(segment => segment.id !== selectedSegment?.id); prevSegments.map((segment) => [segment.id, segment])
);
const segmentsToUpdate = segments.filter(
(segment) => segment.id !== selectedSegment?.id
);
segmentsToUpdate.forEach(newSegment => { segmentsToUpdate.forEach((newSegment) => {
const existingSegment = existingSegmentsMap.get(newSegment.id); const existingSegment = existingSegmentsMap.get(newSegment.id);
if (existingSegment) { if (existingSegment) {
@ -113,7 +132,7 @@ export const useShotService = (): UseShotService => {
sketchUrl: newSegment.sketchUrl, sketchUrl: newSegment.sketchUrl,
lens: newSegment.lens, lens: newSegment.lens,
updatedAt: newSegment.updatedAt, updatedAt: newSegment.updatedAt,
loadingProgress: newSegment.loadingProgress loadingProgress: newSegment.loadingProgress,
}); });
} else { } else {
existingSegmentsMap.set(newSegment.id, newSegment); existingSegmentsMap.set(newSegment.id, newSegment);
@ -123,7 +142,7 @@ export const useShotService = (): UseShotService => {
return Array.from(existingSegmentsMap.values()); return Array.from(existingSegmentsMap.values());
}); });
} catch (error) { } catch (error) {
console.error('定时获取视频片段列表失败:', error); console.error("定时获取视频片段列表失败:", error);
} }
}, 5000); }, 5000);
@ -148,22 +167,21 @@ export const useShotService = (): UseShotService => {
* @param sceneReplaceParams 场景替换参数（可选） * @param sceneReplaceParams 场景替换参数（可选）
* @returns Promise<VideoSegmentEntity> 重新生成的视频片段 * @returns Promise<VideoSegmentEntity> 重新生成的视频片段
*/ */
const regenerateVideoSegment = useCallback( const regenerateVideoSegment =
async ( useCallback(async (): Promise<VideoSegmentEntity> => {
): Promise<VideoSegmentEntity> => {
try { try {
setLoading(true); setLoading(true);
const regeneratedSegment = await vidoEditUseCase.regenerateVideoSegment( const regeneratedSegment = await vidoEditUseCase.regenerateVideoSegment(
projectId, projectId,
selectedSegment!.lens, selectedSegment!.lens,
selectedSegment!.id, selectedSegment!.id
); );
// 如果重新生成的是现有片段，更新列表中的对应项 // 如果重新生成的是现有片段，更新列表中的对应项
if (selectedSegment) { if (selectedSegment) {
setVideoSegments(prev => setVideoSegments((prev) =>
prev.map(segment => prev.map((segment) =>
segment.id === selectedSegment.id ? regeneratedSegment : segment segment.id === selectedSegment.id ? regeneratedSegment : segment
) )
); );
@ -176,9 +194,7 @@ export const useShotService = (): UseShotService => {
} finally { } finally {
setLoading(false); setLoading(false);
} }
}, }, [projectId, selectedSegment, vidoEditUseCase]);
[projectId, selectedSegment, vidoEditUseCase]
);
/** /**
* AI优化视频内容 * AI优化视频内容
@ -226,9 +242,12 @@ export const useShotService = (): UseShotService => {
/** /**
* 设置选中的视频片段 * 设置选中的视频片段
*/ */
const setSelectedSegmentHandler = useCallback((segment: VideoSegmentEntity | null): void => { const setSelectedSegmentHandler = useCallback(
setSelectedSegment(segment); (segment: VideoSegmentEntity | null): void => {
}, []); setSelectedSegment(segment);
},
[]
);
/** /**
* 添加新镜头到选中的视频片段 * 添加新镜头到选中的视频片段
@ -250,13 +269,15 @@ export const useShotService = (): UseShotService => {
// 创建更新后的片段 // 创建更新后的片段
const updatedSegment: VideoSegmentEntity = { const updatedSegment: VideoSegmentEntity = {
...selectedSegment, ...selectedSegment,
lens: [...selectedSegment.lens, newLens] lens: [...selectedSegment.lens, newLens],
}; };
// 批量更新状态，避免多次重渲染 // 批量更新状态，避免多次重渲染
setSelectedSegment(updatedSegment); setSelectedSegment(updatedSegment);
setVideoSegments(prev => { setVideoSegments((prev) => {
const segmentIndex = prev.findIndex(segment => segment.id === selectedSegment.id); const segmentIndex = prev.findIndex(
(segment) => segment.id === selectedSegment.id
);
if (segmentIndex === -1) return prev; if (segmentIndex === -1) return prev;
const newSegments = [...prev]; const newSegments = [...prev];
@ -269,34 +290,42 @@ export const useShotService = (): UseShotService => {
* 删除指定镜头 * 删除指定镜头
* @param lensName 要删除的镜头名称 * @param lensName 要删除的镜头名称
*/ */
const deleteLens = useCallback((lensName: string): void => { const deleteLens = useCallback(
if (!selectedSegment) { (lensName: string): void => {
console.warn("没有选中的视频片段，无法删除镜头"); if (!selectedSegment) {
return; console.warn("没有选中的视频片段，无法删除镜头");
} return;
}
// 过滤掉指定名称的镜头并重新排序 // 过滤掉指定名称的镜头并重新排序
const updatedLens = selectedSegment.lens const updatedLens = selectedSegment.lens
.filter(lens => lens.name !== lensName) .filter((lens) => lens.name !== lensName)
.map((lens, index) => new LensType(`镜头${index + 1}`, lens.script, lens.content)); .map(
(lens, index) =>
new LensType(`镜头${index + 1}`, lens.script, lens.content)
);
// 创建更新后的片段 // 创建更新后的片段
const updatedSegment: VideoSegmentEntity = { const updatedSegment: VideoSegmentEntity = {
...selectedSegment, ...selectedSegment,
lens: updatedLens lens: updatedLens,
}; };
// 批量更新状态，避免多次重渲染 // 批量更新状态，避免多次重渲染
setSelectedSegment(updatedSegment); setSelectedSegment(updatedSegment);
setVideoSegments(prev => { setVideoSegments((prev) => {
const segmentIndex = prev.findIndex(segment => segment.id === selectedSegment.id); const segmentIndex = prev.findIndex(
if (segmentIndex === -1) return prev; (segment) => segment.id === selectedSegment.id
);
if (segmentIndex === -1) return prev;
const newSegments = [...prev]; const newSegments = [...prev];
newSegments[segmentIndex] = updatedSegment; newSegments[segmentIndex] = updatedSegment;
return newSegments; return newSegments;
}); });
}, [selectedSegment]); },
[selectedSegment]
);
/** /**
* 获取视频当前帧的画面，上传到七牛云，并返回七牛云的图片地址,然后调用接口识别出里面的人物信息，返回人物信息 * 获取视频当前帧的画面，上传到七牛云，并返回七牛云的图片地址,然后调用接口识别出里面的人物信息，返回人物信息
@ -305,69 +334,130 @@ export const useShotService = (): UseShotService => {
* @param videoId 视频ID * @param videoId 视频ID
* @returns Promise<string> 七牛云的图片地址 * @returns Promise<string> 七牛云的图片地址
*/ */
const filterRole = useCallback(async ( const filterRole = useCallback(
video: HTMLVideoElement, async (video: HTMLVideoElement) => {
) => { try {
try { // 创建canvas元素来截取视频帧
// 创建canvas元素来截取视频帧 const canvas = document.createElement("canvas");
const canvas = document.createElement('canvas'); const ctx = canvas.getContext("2d");
const ctx = canvas.getContext('2d'); console.log(video);
console.log(video); video.crossOrigin = "anonymous";
video.crossOrigin = 'anonymous'; if (!ctx) {
if (!ctx) { throw new Error("无法获取canvas上下文");
throw new Error('无法获取canvas上下文'); }
}
// 设置canvas尺寸为视频尺寸 // 设置canvas尺寸为视频尺寸
canvas.width = video.videoWidth; canvas.width = video.videoWidth;
canvas.height = video.videoHeight; canvas.height = video.videoHeight;
// 将当前视频帧绘制到canvas上 // 将当前视频帧绘制到canvas上
ctx.drawImage(video, 0, 0, canvas.width, canvas.height); ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
// 将canvas转换为blob // 将canvas转换为blob
const blob = await new Promise<Blob>((resolve, reject) => { const blob = await new Promise<Blob>((resolve, reject) => {
canvas.toBlob((blob) => { canvas.toBlob((blob) => {
if (blob) { if (blob) {
resolve(blob); resolve(blob);
} else { } else {
reject(new Error('无法将canvas转换为blob')); reject(new Error("无法将canvas转换为blob"));
} }
}, 'image/png'); }, "image/png");
}); });
// 创建File对象 // 创建File对象
const file = new File([blob], `frame_${Date.now()}.png`, { type: 'image/png' }); const file = new File([blob], `frame_${Date.now()}.png`, {
type: "image/png",
});
// 获取上传token // 获取上传token
const { token } = await getUploadToken(); const { token } = await getUploadToken();
// 上传到七牛云 // 上传到七牛云
const imageUrl = await uploadToQiniu(file, token); const imageUrl = await uploadToQiniu(file, token);
// 调用用例中的识别角色方法 // 调用用例中的识别角色方法
if (vidoEditUseCase) {
try { try {
const recognitionResult = await vidoEditUseCase.recognizeRoleFromImage( const recognitionResult =
projectId, await vidoEditUseCase.recognizeRoleFromImage(
selectedSegment!.id, projectId,
imageUrl selectedSegment!.id,
imageUrl
);
console.log("角色识别结果:", recognitionResult);
setMatched_persons(
recognitionResult.recognition_result.data.matched_persons
); );
console.log('角色识别结果:', recognitionResult);
return recognitionResult; return recognitionResult;
} catch (recognitionError) { } catch (recognitionError) {
console.warn('角色识别失败，但图片上传成功:', recognitionError); console.warn("角色识别失败，但图片上传成功:", recognitionError);
} }
} catch (error) {
console.error("获取视频帧失败:", error);
throw error;
} }
} catch (error) { },
console.error('获取视频帧失败:', error); [projectId, selectedSegment, vidoEditUseCase]
throw error; );
}
}, [projectId, selectedSegment, vidoEditUseCase]); /**
* 计算识别框的属性
* @description 根据DOM元素尺寸和匹配数据计算识别框的位置和尺寸
* @param containerElement DOM容器元素
* @returns 计算后的识别框属性数组
*/
const calculateRecognitionBoxes = (
containerElement: HTMLElement
): Array<{
/** 横向定位坐标 */
left: number;
/** 纵向定位坐标 */
top: number;
/** 宽度 */
width: number;
/** 高度 */
height: number;
/** 人物ID */
person_id: string;
}> => {
// 获取容器元素的尺寸
const containerRect = containerElement.getBoundingClientRect();
const containerWidth = containerRect.width;
const containerHeight = containerRect.height;
// 计算识别框属性
return matched_persons
.map((person) => {
// 取出bbox信息
const bbox = (person as any).bbox;
if (!bbox) return null;
// 计算绝对坐标和尺寸（百分比转像素）
const left = (bbox.x || 0) * containerWidth;
const top = (bbox.y || 0) * containerHeight;
const width = (bbox.width || 0) * containerWidth;
const height = (bbox.height || 0) * containerHeight;
return {
left,
top,
width,
height,
person_id: person.person_id
};
})
.filter(Boolean) as Array<{
left: number;
top: number;
width: number;
height: number;
person_id: string;
}>;
};
return { return {
// 响应式状态 // 响应式状态
loading, loading,
videoSegments, videoSegments,
selectedSegment, selectedSegment,
matched_persons,
// 操作方法 // 操作方法
getVideoSegmentList, getVideoSegmentList,
regenerateVideoSegment, regenerateVideoSegment,
@ -378,5 +468,6 @@ export const useShotService = (): UseShotService => {
deleteLens, deleteLens,
filterRole, filterRole,
setSimpleCharacter, setSimpleCharacter,
calculateRecognitionBoxes
}; };
}; };

7
app/service/usecase/RoleEditUseCase.ts
View File

@ -24,6 +24,8 @@ export interface RoleResponse {
highlights: string[]; highlights: string[];
/** 角色图片地址 */ /** 角色图片地址 */
image_path: string; image_path: string;
/**缓存 */
character_draft: string;
} }
/** /**
@ -73,6 +75,7 @@ export class RoleEditUseCase {
const characters = newCharacterData.data || []; const characters = newCharacterData.data || [];
return characters.map((char, index) => { return characters.map((char, index) => {
const roleEntity: RoleEntity = { const roleEntity: RoleEntity = {
id: `role_${index + 1}`, id: `role_${index + 1}`,
name: char.character_name || '', name: char.character_name || '',
@ -101,6 +104,10 @@ export class RoleEditUseCase {
} }
return projectRoleData.map((char, index) => { return projectRoleData.map((char, index) => {
if(char.character_draft){
const roleEntity: RoleEntity = JSON.parse(char.character_draft);
return roleEntity;
}
/** 角色实体对象 */ /** 角色实体对象 */
const roleEntity: RoleEntity = { const roleEntity: RoleEntity = {
id: `role_${index + 1}`, id: `role_${index + 1}`,

86
app/service/usecase/ShotEditUsecase.ts
View File

@ -320,7 +320,7 @@ export class VideoSegmentEditUseCase {
projectId: string, projectId: string,
videoId: string, videoId: string,
targetImageUrl: string targetImageUrl: string
): Promise<any> { ) {
try { try {
this.loading = true; this.loading = true;
@ -371,12 +371,16 @@ export interface BoundingBox {
* @description 从图片中识别出的与已知角色匹配的人物信息 * @description 从图片中识别出的与已知角色匹配的人物信息
*/ */
export interface MatchedPerson { export interface MatchedPerson {
/** 人物ID */ /** 人物名 */
person_id: string; person_id: string;
/** 边界框信息 */ /**x坐标 小数百分比形式 */
bbox: BoundingBox; x: number;
/** 匹配置信度（0-1之间） */ /**y坐标 小数百分比形式 */
confidence: number; y: number;
/**宽度 小数百分比形式 */
width: number;
/**高度 小数百分比形式 */
height: number;
} }
/** /**
@ -439,38 +443,38 @@ export interface RoleRecognitionResponse {
characters_used: CharacterUsed[]; characters_used: CharacterUsed[];
} }
export const roleRecognitionResponse:RoleRecognitionResponse = { // export const roleRecognitionResponse:RoleRecognitionResponse = {
"project_id": "d0df7120-e27b-4f84-875c-e532f1bd318c", // "project_id": "d0df7120-e27b-4f84-875c-e532f1bd318c",
"video_id": "984f3347-c81c-4af8-9145-49ead82becde", // "video_id": "984f3347-c81c-4af8-9145-49ead82becde",
"target_image_url": "https://cdn.qikongjian.com/videos/1754970412744_kqxplx.png", // "target_image_url": "https://cdn.qikongjian.com/videos/1754970412744_kqxplx.png",
"known_persons_count": 1, // "known_persons_count": 1,
"recognition_result": { // "recognition_result": {
"code": 200, // "code": 200,
"message": "识别完成", // "message": "识别完成",
"data": // "data":
{ // {
"target_image_url": "https://cdn.qikongjian.com/videos/1754970412744_kqxplx.png", // "target_image_url": "https://cdn.qikongjian.com/videos/1754970412744_kqxplx.png",
"total_persons_detected": 1, // "total_persons_detected": 1,
"matched_persons": [ // "matched_persons": [
{ // {
"person_id": "CH-01", // "person_id": "CH-01",
"bbox": { // "bbox": {
"x": 269, // "x": 269,
"y": 23, // "y": 23,
"width": 585, // "width": 585,
"height": 685 // "height": 685
}, // },
"confidence": 0.36905956268310547 // "confidence": 0.36905956268310547
} // }
] // ]
} // }
}, // },
"characters_used": [ // "characters_used": [
{ // {
"character_name": "CH-01", // "character_name": "CH-01",
"c_id": "无C-ID", // "c_id": "无C-ID",
"image_path": "无image_path", // "image_path": "无image_path",
"avatar": "https://cdn.huiying.video/template/Whisk_9afb196368.jpg" // "avatar": "https://cdn.huiying.video/template/Whisk_9afb196368.jpg"
} // }
] // ]
} // }