From 490698cb92c0c9641d2c84060608bcfe5995d7a1 Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 20 Jan 2026 11:16:36 +0100 Subject: [PATCH] feat: try sync audio to video --- functions/src/upload.js | 33 ++++++-- src/screens/Playback/RecordPlayback.js | 66 ++++++++------- src/screens/Playback/RecordPlayback.web.js | 87 ++++++++++---------- src/screens/Playback/RecordedPlayback.js | 25 +++--- src/screens/Playback/RecordedPlayback.web.js | 30 +++---- 5 files changed, 131 insertions(+), 110 deletions(-) diff --git a/functions/src/upload.js b/functions/src/upload.js index d40af73..bc16069 100644 --- a/functions/src/upload.js +++ b/functions/src/upload.js @@ -31,11 +31,26 @@ async function downloadToFile(url, destPath) { const SCALE_FILTER = "scale='trunc(min(1920,iw)/2)*2':'trunc(min(1920,ih)/2)*2':force_original_aspect_ratio=decrease" -async function muxAudioIntoVideo({ videoPath, audioPath, outPath }) { +async function muxAudioIntoVideo({ videoPath, audioPath, outPath, syncOffset = 0 }) { return new Promise((resolve, reject) => { - ffmpeg() - .input(videoPath) // 0:v - .input(audioPath) // 1:a + const command = ffmpeg() + + const offset = Number(syncOffset) || 0 + + // Si offset > 0 : la vidéo est en avance (ou capturée en retard dans le passé relatif ?), on veut la retarder pour qu'elle commence plus tard + // c-à-d on décale le flux vidéo. + // Si offset < 0 : on décale l'audio. + if (offset > 0) { + command.inputOptions(['-itsoffset', String(offset)]) + } + command.input(videoPath) // 0:v + + if (offset < 0) { + command.inputOptions(['-itsoffset', String(Math.abs(offset))]) + } + command.input(audioPath) // 1:a + + command .outputOptions([ '-map', '0:v:0', // garder la 1re piste vidéo de l'entrée 0 @@ -72,7 +87,7 @@ async function muxAudioIntoVideo({ videoPath, audioPath, outPath }) { }) } -async function uploadPlaybackAsset({ videoUrl, audioUrl, storagePath }) { +async function uploadPlaybackAsset({ videoUrl, audioUrl, storagePath, syncOffset = 0 }) { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'merge-')) const videoPath = path.join(tmpDir, 'video.mp4') const audioPath = path.join(tmpDir, 'audio.mp3') @@ -84,8 +99,8 @@ async function uploadPlaybackAsset({ videoUrl, audioUrl, storagePath }) { await downloadToFile(videoUrl, videoPath) await downloadToFile(audioUrl, audioPath) - logger.info('[merge] transcodage/mux ffmpeg') - await muxAudioIntoVideo({ videoPath, audioPath, outPath }) + logger.info('[merge] transcodage/mux ffmpeg', { syncOffset }) + await muxAudioIntoVideo({ videoPath, audioPath, outPath, syncOffset }) const bucket = admin.storage().bucket() const downloadToken = crypto.randomUUID() @@ -141,7 +156,7 @@ exports.mergeVideoAndAudio = onCall( const uid = auth?.uid if (!uid) throw new HttpsError('unauthenticated', 'Authentification requise') - const { videoUrl, audioUrl, storagePath, projectId } = data || {} + const { videoUrl, audioUrl, storagePath, projectId, syncOffset } = data || {} if (!videoUrl || !audioUrl || !storagePath) { throw new HttpsError('invalid-argument', 'Requis: { videoUrl, audioUrl, storagePath }') @@ -152,7 +167,7 @@ exports.mergeVideoAndAudio = onCall( throw new HttpsError('permission-denied', `storagePath doit commencer par ${expectedPrefix}`) } - const result = await uploadPlaybackAsset({ videoUrl, audioUrl, storagePath }) + const result = await uploadPlaybackAsset({ videoUrl, audioUrl, storagePath, syncOffset }) if (projectId) { await markPlaybackCompatibility({ projectId, initiatorUid: uid }) } diff --git a/src/screens/Playback/RecordPlayback.js b/src/screens/Playback/RecordPlayback.js index a81d3d0..204be4c 100644 --- a/src/screens/Playback/RecordPlayback.js +++ b/src/screens/Playback/RecordPlayback.js @@ -25,7 +25,7 @@ const LOG_PREFIX = '[RecordPlayback]' const log = typeof __DEV__ === 'undefined' || __DEV__ ? (...args) => console.log(LOG_PREFIX, ...args) - : () => {} + : () => { } const CAMERA_FACING_OPTIONS = [ { label: 'Avant', value: 'front' }, @@ -211,7 +211,7 @@ const RecordPlayback = ({ route }) => { }) } progressLogRef.current = { bucket, lastPos: pos, lastDur: dur } - } catch (_) {} + } catch (_) { } }, 250) return () => clearInterval(id) }, [player]) @@ -232,7 +232,7 @@ const RecordPlayback = ({ route }) => { // Permissions au mount + cleanup useEffect(() => { - ;(async () => { + ; (async () => { try { if (!cameraPermission?.granted) { log('Requesting camera permission on mount') @@ -240,7 +240,7 @@ const RecordPlayback = ({ route }) => { } else { log('Camera permission already granted on mount') } - } catch (_) {} + } catch (_) { } })() return () => { try { @@ -251,7 +251,7 @@ const RecordPlayback = ({ route }) => { listenTimerRef.current = null checkSongEndRef.current = null log('Cleanup on unmount, cleared timers') - } catch (_) {} + } catch (_) { } } }, []) // eslint-disable-line react-hooks/exhaustive-deps @@ -298,15 +298,15 @@ const RecordPlayback = ({ route }) => { setCountdown(0) try { await cameraRef.current?.resumePreview?.() - } catch (_) {} + } catch (_) { } if (player) { try { if (player.playing) await player.pause?.() await player.seekTo?.(0) - } catch (_) {} + } catch (_) { } } - } catch (_) {} + } catch (_) { } }, [player] ) @@ -442,7 +442,7 @@ const RecordPlayback = ({ route }) => { isPausedRef.current = false try { await cameraRef.current?.resumePreview?.() - } catch (_) {} + } catch (_) { } playbackStartedRef.current = false progressLogRef.current = { bucket: -1, lastPos: -1, lastDur: -1 } log('startRecordingWithMusic', { @@ -521,6 +521,10 @@ const RecordPlayback = ({ route }) => { message: error?.message || String(error || ''), }) } + + // Add artificial delay to compensate for camera startup latency (Audio usually starts faster than video recording) + await new Promise(r => setTimeout(r, 250)) + try { await player.play?.() log('player.play invoked') @@ -544,10 +548,10 @@ const RecordPlayback = ({ route }) => { try { await projectsRef.doc(project.id).set({ views: increment(1) }, { merge: true }) log('Views incremented') - } catch (_) {} + } catch (_) { } } } - } catch (_) {} + } catch (_) { } }, 500) } @@ -565,7 +569,7 @@ const RecordPlayback = ({ route }) => { try { cameraRef.current?.stopRecording?.() log('stopRecording retried while awaiting stop') - } catch (_) {} + } catch (_) { } } return } @@ -590,9 +594,9 @@ const RecordPlayback = ({ route }) => { try { cameraRef.current?.stopRecording?.() log('stopRecording triggered') - } catch (_) {} + } catch (_) { } } - } catch (_) {} + } catch (_) { } }, 500) } @@ -610,7 +614,7 @@ const RecordPlayback = ({ route }) => { } try { if (player?.playing) await player.pause?.() - } catch (_) {} + } catch (_) { } if (listenTimerRef.current) { clearInterval(listenTimerRef.current) listenTimerRef.current = null @@ -643,7 +647,7 @@ const RecordPlayback = ({ route }) => { await discardRecordingFile(video?.uri, 'too_short') try { goBack() - } catch (_) {} + } catch (_) { } return } } @@ -744,7 +748,7 @@ const RecordPlayback = ({ route }) => { onPress: () => { try { goBack() - } catch (_) {} + } catch (_) { } }, }, ], @@ -771,21 +775,21 @@ const RecordPlayback = ({ route }) => { if (player?.playing) { const maybePromise = player.pause?.() if (maybePromise && typeof maybePromise.catch === 'function') { - maybePromise.catch(() => {}) + maybePromise.catch(() => { }) } } - } catch (_) {} + } catch (_) { } try { if (isRecording) { cameraRef.current?.stopRecording?.() } - } catch (_) {} + } catch (_) { } } else { exitRequestedRef.current = false } try { goBack() - } catch (_) {} + } catch (_) { } return true }, [goBack, isPreparing, isRecording, player]) @@ -808,21 +812,21 @@ const RecordPlayback = ({ route }) => { stopRequestedAtRef.current = 0 try { await player?.pause?.() - } catch (_) {} + } catch (_) { } try { await cameraRef.current?.pausePreview?.() - } catch (_) {} + } catch (_) { } return } setIsPaused(false) isPausedRef.current = false try { await cameraRef.current?.resumePreview?.() - } catch (_) {} + } catch (_) { } try { await player?.play?.() - } catch (_) {} - } catch (_) {} + } catch (_) { } + } catch (_) { } }, [isRecording, player]) const handleRestartRecording = async () => { @@ -837,7 +841,7 @@ const RecordPlayback = ({ route }) => { isPausedRef.current = false try { await cameraRef.current?.resumePreview?.() - } catch (_) {} + } catch (_) { } if (isPreparing) { if (hasRecordingPending) { await startCountdownThenRecord({ @@ -864,16 +868,16 @@ const RecordPlayback = ({ route }) => { stopRequestedAtRef.current = Date.now() try { if (player?.playing) await player.pause?.() - } catch (_) {} + } catch (_) { } try { cameraRef.current?.stopRecording?.() - } catch (_) {} + } catch (_) { } await startCountdownThenRecord({ preserveRestartFlag: true, preserveSongEndWatcher: true, preserveStopRequest: true, }) - } catch (_) {} + } catch (_) { } } const permissionsGranted = !!cameraPermission?.granted @@ -945,7 +949,7 @@ const RecordPlayback = ({ route }) => { onPress={async () => { try { if (!cameraPermission?.granted) await requestCameraPermission() - } catch (_) {} + } catch (_) { } }} /> diff --git a/src/screens/Playback/RecordPlayback.web.js b/src/screens/Playback/RecordPlayback.web.js index f52446a..2df13df 100644 --- a/src/screens/Playback/RecordPlayback.web.js +++ b/src/screens/Playback/RecordPlayback.web.js @@ -44,7 +44,7 @@ const pickRecorderMimeType = () => { for (const mimeType of candidates) { try { if (MediaRecorder.isTypeSupported(mimeType)) return mimeType - } catch {} + } catch { } } return undefined @@ -162,7 +162,7 @@ const RecordPlayback = ({ route }) => { if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') { mediaRecorderRef.current.stop() } - } catch (e) {} + } catch (e) { } recordedChunksRef.current = [] releaseRecordingUrl() @@ -248,7 +248,7 @@ const RecordPlayback = ({ route }) => { } recorder.stop() } - } catch (e) {} + } catch (e) { } if (!waitPromise) { return recordedUrlRef.current || null @@ -272,10 +272,10 @@ const RecordPlayback = ({ route }) => { try { await player?.pause?.() await player?.seekTo?.(0) - } catch (e) {} + } catch (e) { } try { await stopRecorderAndGetUrl() - } catch (e) {} + } catch (e) { } if (!preserveRecordingForNextScreenRef.current) { releaseRecordingUrl() } @@ -315,12 +315,12 @@ const RecordPlayback = ({ route }) => { ) useEffect(() => { - ;(async () => { + ; (async () => { try { if (!cameraPermission?.granted) { await requestCameraPermission() } - } catch (e) {} + } catch (e) { } })() return () => { clearAllTimers() @@ -345,25 +345,25 @@ const RecordPlayback = ({ route }) => { } let cancelled = false - ;(async () => { - try { - const stream = await navigator.mediaDevices.getUserMedia({ - video: { facingMode: 'user' }, - audio: false, - }) - if (cancelled) { - stream.getTracks().forEach((track) => track.stop()) - return + ; (async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ + video: { facingMode: 'user' }, + audio: false, + }) + if (cancelled) { + stream.getTracks().forEach((track) => track.stop()) + return + } + mediaStreamRef.current = stream + setMediaReady(true) + setMediaError(null) + } catch (e) { + if (cancelled) return + setMediaError(e instanceof Error ? e : new Error(String(e || ''))) + setMediaReady(false) } - mediaStreamRef.current = stream - setMediaReady(true) - setMediaError(null) - } catch (e) { - if (cancelled) return - setMediaError(e instanceof Error ? e : new Error(String(e || ''))) - setMediaReady(false) - } - })() + })() return () => { cancelled = true @@ -382,13 +382,13 @@ const RecordPlayback = ({ route }) => { } const playPromise = video.play?.() if (playPromise && typeof playPromise.catch === 'function') { - playPromise.catch(() => {}) + playPromise.catch(() => { }) } - } catch (e) {} + } catch (e) { } } else { try { video.srcObject = null - } catch {} + } catch { } } }, [mediaReady, mediaError]) @@ -398,7 +398,7 @@ const RecordPlayback = ({ route }) => { if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') { mediaRecorderRef.current.stop() } - } catch {} + } catch { } if (mediaStreamRef.current) { mediaStreamRef.current.getTracks().forEach((track) => track.stop()) mediaStreamRef.current = null @@ -414,7 +414,7 @@ const RecordPlayback = ({ route }) => { clearAllTimers() try { await player?.pause?.() - } catch (e) {} + } catch (e) { } let videoUrl = null try { @@ -422,7 +422,7 @@ const RecordPlayback = ({ route }) => { if (!videoUrl) { videoUrl = recordedUrlRef.current || null } - } catch (e) {} + } catch (e) { } setIsRecording(false) setIsPaused(false) @@ -457,7 +457,7 @@ const RecordPlayback = ({ route }) => { await player?.seekTo?.(0) correctedInitialJumpRef.current = true nextPos = 0 - } catch (e) {} + } catch (e) { } } const timeSinceStart = @@ -501,11 +501,11 @@ const RecordPlayback = ({ route }) => { if (project?.id) { try { await projectsRef.doc(project.id).set({ views: increment(1) }, { merge: true }) - } catch (e) {} + } catch (e) { } } } } - } catch (e) {} + } catch (e) { } }, 500) }, [player, project?.id]) @@ -520,6 +520,7 @@ const RecordPlayback = ({ route }) => { const recorderStarted = startRecorder() if (!recorderStarted) { } + await new Promise(r => setTimeout(r, 250)) await player?.play?.() // attend la promesse → l'élément est prêt perfStartRef.current = perfStartRef.current ?? performance.now() correctedInitialJumpRef.current = false // autorise la correction 1-shot @@ -543,7 +544,7 @@ const RecordPlayback = ({ route }) => { onPress: () => { try { goBack() - } catch {} + } catch { } }, }, ], @@ -588,13 +589,13 @@ const RecordPlayback = ({ route }) => { pausedAtRef.current = performance.now() try { await player?.pause?.() - } catch (e) {} + } catch (e) { } try { const recorder = mediaRecorderRef.current if (recorder && recorder.state === 'recording') { recorder.pause?.() } - } catch (e) {} + } catch (e) { } return } const pausedAt = pausedAtRef.current @@ -609,17 +610,17 @@ const RecordPlayback = ({ route }) => { if (recorder && recorder.state === 'paused') { recorder.resume?.() } - } catch (e) {} + } catch (e) { } try { await player?.play?.() - } catch (e) {} - } catch (e) {} + } catch (e) { } + } catch (e) { } }, [isRecording, player]) const handleRestartRecording = useCallback(async () => { try { await startCountdownThenRecord() - } catch (e) {} + } catch (e) { } }, [startCountdownThenRecord]) const progressRatio = dur > 0 ? Math.min(1, pos / dur) : 0 @@ -630,7 +631,7 @@ const RecordPlayback = ({ route }) => { // containerStyle={{ backgroundColor: Palette.gray }} headerType="NONE" backgroundColor={Palette.grayMid} - // backgroundImg={background.playbackBG2} + // backgroundImg={background.playbackBG2} > @@ -848,7 +849,7 @@ const RecordPlayback = ({ route }) => { onPress={async () => { try { if (!cameraPermission?.granted) await requestCameraPermission() - } catch {} + } catch { } }} /> diff --git a/src/screens/Playback/RecordedPlayback.js b/src/screens/Playback/RecordedPlayback.js index 4539c9e..0ab766e 100644 --- a/src/screens/Playback/RecordedPlayback.js +++ b/src/screens/Playback/RecordedPlayback.js @@ -12,6 +12,7 @@ import Page from '../../layouts/Page' import { Routes } from '../../navigation' import { goBack, navigate } from '../../navigation/NavigationService' import { gutters, Palette } from '../../styles' +import { FONT_FAMILY } from '../../styles/Fonts' const RecordedPlayback = ({ route }) => { const { videoUri, project } = route.params || {} const songUrl = project?.songUrl || null @@ -40,10 +41,10 @@ const RecordedPlayback = ({ route }) => { const stopPlayback = useCallback(async () => { try { if (audioPlayer?.playing) await audioPlayer.pause?.() - } catch (e) {} + } catch (e) { } try { if (videoPlayer?.playing) videoPlayer.pause() - } catch (e) {} + } catch (e) { } }, [audioPlayer, videoPlayer]) useEffect(() => { @@ -52,7 +53,7 @@ const RecordedPlayback = ({ route }) => { try { if (audioPlayer && songUrl) await audioPlayer.play?.() if (videoPlayer) videoPlayer.play() - } catch (e) {} + } catch (e) { } } start() return () => { @@ -78,7 +79,7 @@ const RecordedPlayback = ({ route }) => { videoPlayer.currentTime = Math.max(0, (pos || 0) / 1000) } } - } catch (e) {} + } catch (e) { } }, 250) return () => global.clearInterval(id) }, [audioPlayer, videoPlayer]) @@ -89,7 +90,7 @@ const RecordedPlayback = ({ route }) => { const pos = Math.max(0, Math.min(dur, Math.floor(targetMs))) if (audioPlayer && dur > 0) await audioPlayer.seekTo?.(Math.floor(pos / 1000)) if (videoPlayer) videoPlayer.currentTime = Math.max(0, pos / 1000) - } catch (e) {} + } catch (e) { } } const onSeekStart = useCallback(() => { @@ -99,19 +100,19 @@ const RecordedPlayback = ({ route }) => { const pauseDuringSeek = useCallback(async () => { try { if (audioPlayer?.playing) await audioPlayer.pause?.() - } catch (e) {} + } catch (e) { } try { if (videoPlayer?.playing) videoPlayer.pause() - } catch (e) {} + } catch (e) { } }, [audioPlayer, videoPlayer]) const resumeAfterSeek = useCallback(async () => { try { if (audioPlayer) await audioPlayer.play?.() - } catch (e) {} + } catch (e) { } try { if (videoPlayer) videoPlayer.play() - } catch (e) {} + } catch (e) { } }, [audioPlayer, videoPlayer]) const sliderProgress = useMemo(() => { @@ -155,7 +156,7 @@ const RecordedPlayback = ({ route }) => { playbackEndedRef.current = false if (songUrl && audioPlayer) await audioPlayer.play?.() if (videoPlayer) videoPlayer.play() - } catch (e) {} + } catch (e) { } } return ( @@ -225,7 +226,7 @@ const RecordedPlayback = ({ route }) => { onPress={async () => { try { await stopPlayback() - } catch (e) {} + } catch (e) { } try { if (videoUri) { const info = await FileSystem.getInfoAsync(videoUri) @@ -234,7 +235,7 @@ const RecordedPlayback = ({ route }) => { idempotent: true, }) } - } catch (e) {} + } catch (e) { } navigate(Routes.RecordPlayback, { project }) }} /> diff --git a/src/screens/Playback/RecordedPlayback.web.js b/src/screens/Playback/RecordedPlayback.web.js index 9346eff..3649565 100644 --- a/src/screens/Playback/RecordedPlayback.web.js +++ b/src/screens/Playback/RecordedPlayback.web.js @@ -55,12 +55,12 @@ const RecordedPlayback = ({ route }) => { const stopPlayback = useCallback(async () => { try { if (audioPlayer?.playing) await audioPlayer.pause?.() - } catch {} + } catch { } try { if (videoElRef.current && !videoElRef.current.paused) { videoElRef.current.pause() } - } catch {} + } catch { } }, [audioPlayer]) useEffect(() => { @@ -82,9 +82,9 @@ const RecordedPlayback = ({ route }) => { if (videoElRef.current && videoUri) { // Lecture vidéo HTML5 (muet pour éviter les policies) videoElRef.current.muted = true - videoElRef.current.play().catch(() => {}) + videoElRef.current.play().catch(() => { }) } - } catch {} + } catch { } } start() @@ -115,7 +115,7 @@ const RecordedPlayback = ({ route }) => { videoElRef.current.currentTime = Math.max(0, posS) } } - } catch {} + } catch { } }, 250) return () => clearInterval(id) }, [audioPlayer, videoUri]) @@ -141,12 +141,12 @@ const RecordedPlayback = ({ route }) => { await stopPlayback() try { if (audioPlayer) await audioPlayer.seekTo?.(0) - } catch {} + } catch { } try { if (videoElRef.current) { videoElRef.current.currentTime = 0 } - } catch {} + } catch { } })() } }, [progress, stopPlayback, audioPlayer]) @@ -163,7 +163,7 @@ const RecordedPlayback = ({ route }) => { if (videoElRef.current && videoUri) { videoElRef.current.currentTime = Math.max(0, target) } - } catch {} + } catch { } })() pendingSeekRef.current = promise await promise @@ -177,7 +177,7 @@ const RecordedPlayback = ({ route }) => { if (!promise) return try { await promise - } catch {} + } catch { } }, []) const wasPlayingRef = useRef(false) @@ -193,7 +193,7 @@ const RecordedPlayback = ({ route }) => { if (videoElRef.current && !videoElRef.current.paused) { videoElRef.current.pause() } - } catch {} + } catch { } }, [audioPlayer]) const onSeekEnd = useCallback(async () => { try { @@ -202,9 +202,9 @@ const RecordedPlayback = ({ route }) => { isSeekingRef.current = false if (wasPlayingRef.current) { if (audioPlayer) await audioPlayer.resume?.() - if (videoElRef.current && videoUri) videoElRef.current.play().catch(() => {}) + if (videoElRef.current && videoUri) videoElRef.current.play().catch(() => { }) } - } catch {} + } catch { } }, [audioPlayer, videoUri, waitForPendingSeek]) const handleTogglePlayback = useCallback(async () => { @@ -233,9 +233,9 @@ const RecordedPlayback = ({ route }) => { } if (videoElRef.current && videoUri) { videoElRef.current.muted = true - videoElRef.current.play().catch(() => {}) + videoElRef.current.play().catch(() => { }) } - } catch {} + } catch { } }, [audioPlayer, songUrl, stopPlayback, progress, videoUri]) return ( @@ -377,7 +377,7 @@ const RecordedPlayback = ({ route }) => { onPress={async () => { try { await stopPlayback() - } catch {} + } catch { } navigate(Routes.RecordPlayback, { project }) }} />