leonm/musicland
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: try sync audio to video

Browse Source
This commit is contained in:
Leon
2026-01-20 11:16:36 +01:00
parent 4343cee298
commit 490698cb92
5 changed files with 131 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
functions/src/upload.js
+24 -9
View File
@@ -31,11 +31,26 @@ async function downloadToFile(url, destPath) {
const SCALE_FILTER =
"scale='trunc(min(1920,iw)/2)*2':'trunc(min(1920,ih)/2)*2':force_original_aspect_ratio=decrease"
async function muxAudioIntoVideo({ videoPath, audioPath, outPath }) {
async function muxAudioIntoVideo({ videoPath, audioPath, outPath, syncOffset = 0 }) {
return new Promise((resolve, reject) => {
ffmpeg()
.input(videoPath) // 0:v
.input(audioPath) // 1:a
const command = ffmpeg()
const offset = Number(syncOffset) || 0
// Si offset > 0 : la vidéo est en avance (ou capturée en retard dans le passé relatif ?), on veut la retarder pour qu'elle commence plus tard
// c-à-d on décale le flux vidéo.
// Si offset < 0 : on décale l'audio.
if (offset > 0) {
command.inputOptions(['-itsoffset', String(offset)])
}
command.input(videoPath) // 0:v
if (offset < 0) {
command.inputOptions(['-itsoffset', String(Math.abs(offset))])
}
command.input(audioPath) // 1:a
command
.outputOptions([
'-map',
'0:v:0', // garder la 1re piste vidéo de l'entrée 0
@@ -72,7 +87,7 @@ async function muxAudioIntoVideo({ videoPath, audioPath, outPath }) {
})
}
async function uploadPlaybackAsset({ videoUrl, audioUrl, storagePath }) {
async function uploadPlaybackAsset({ videoUrl, audioUrl, storagePath, syncOffset = 0 }) {
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'merge-'))
const videoPath = path.join(tmpDir, 'video.mp4')
const audioPath = path.join(tmpDir, 'audio.mp3')
@@ -84,8 +99,8 @@ async function uploadPlaybackAsset({ videoUrl, audioUrl, storagePath }) {
await downloadToFile(videoUrl, videoPath)
await downloadToFile(audioUrl, audioPath)
logger.info('[merge] transcodage/mux ffmpeg')
await muxAudioIntoVideo({ videoPath, audioPath, outPath })
logger.info('[merge] transcodage/mux ffmpeg', { syncOffset })
await muxAudioIntoVideo({ videoPath, audioPath, outPath, syncOffset })
const bucket = admin.storage().bucket()
const downloadToken = crypto.randomUUID()
@@ -141,7 +156,7 @@ exports.mergeVideoAndAudio = onCall(
const uid = auth?.uid
if (!uid) throw new HttpsError('unauthenticated', 'Authentification requise')
const { videoUrl, audioUrl, storagePath, projectId } = data || {}
const { videoUrl, audioUrl, storagePath, projectId, syncOffset } = data || {}
if (!videoUrl || !audioUrl || !storagePath) {
throw new HttpsError('invalid-argument', 'Requis: { videoUrl, audioUrl, storagePath }')
@@ -152,7 +167,7 @@ exports.mergeVideoAndAudio = onCall(
throw new HttpsError('permission-denied', `storagePath doit commencer par ${expectedPrefix}`)
}
const result = await uploadPlaybackAsset({ videoUrl, audioUrl, storagePath })
const result = await uploadPlaybackAsset({ videoUrl, audioUrl, storagePath, syncOffset })
if (projectId) {
await markPlaybackCompatibility({ projectId, initiatorUid: uid })
}
src/screens/Playback/RecordPlayback.js
+35 -31
View File
@@ -25,7 +25,7 @@ const LOG_PREFIX = '[RecordPlayback]'
const log =
typeof __DEV__ === 'undefined' || __DEV__
? (...args) => console.log(LOG_PREFIX, ...args)
: () => {}
: () => { }
const CAMERA_FACING_OPTIONS = [
{ label: 'Avant', value: 'front' },
@@ -211,7 +211,7 @@ const RecordPlayback = ({ route }) => {
})
}
progressLogRef.current = { bucket, lastPos: pos, lastDur: dur }
} catch (_) {}
} catch (_) { }
}, 250)
return () => clearInterval(id)
}, [player])
@@ -232,7 +232,7 @@ const RecordPlayback = ({ route }) => {
// Permissions au mount + cleanup
useEffect(() => {
;(async () => {
; (async () => {
try {
if (!cameraPermission?.granted) {
log('Requesting camera permission on mount')
@@ -240,7 +240,7 @@ const RecordPlayback = ({ route }) => {
} else {
log('Camera permission already granted on mount')
}
} catch (_) {}
} catch (_) { }
})()
return () => {
try {
@@ -251,7 +251,7 @@ const RecordPlayback = ({ route }) => {
listenTimerRef.current = null
checkSongEndRef.current = null
log('Cleanup on unmount, cleared timers')
} catch (_) {}
} catch (_) { }
}
}, []) // eslint-disable-line react-hooks/exhaustive-deps
@@ -298,15 +298,15 @@ const RecordPlayback = ({ route }) => {
setCountdown(0)
try {
await cameraRef.current?.resumePreview?.()
} catch (_) {}
} catch (_) { }
if (player) {
try {
if (player.playing) await player.pause?.()
await player.seekTo?.(0)
} catch (_) {}
} catch (_) { }
}
} catch (_) {}
} catch (_) { }
},
[player]
)
@@ -442,7 +442,7 @@ const RecordPlayback = ({ route }) => {
isPausedRef.current = false
try {
await cameraRef.current?.resumePreview?.()
} catch (_) {}
} catch (_) { }
playbackStartedRef.current = false
progressLogRef.current = { bucket: -1, lastPos: -1, lastDur: -1 }
log('startRecordingWithMusic', {
@@ -521,6 +521,10 @@ const RecordPlayback = ({ route }) => {
message: error?.message || String(error || ''),
})
}
// Add artificial delay to compensate for camera startup latency (Audio usually starts faster than video recording)
await new Promise(r => setTimeout(r, 250))
try {
await player.play?.()
log('player.play invoked')
@@ -544,10 +548,10 @@ const RecordPlayback = ({ route }) => {
try {
await projectsRef.doc(project.id).set({ views: increment(1) }, { merge: true })
log('Views incremented')
} catch (_) {}
} catch (_) { }
}
}
} catch (_) {}
} catch (_) { }
}, 500)
}
@@ -565,7 +569,7 @@ const RecordPlayback = ({ route }) => {
try {
cameraRef.current?.stopRecording?.()
log('stopRecording retried while awaiting stop')
} catch (_) {}
} catch (_) { }
}
return
}
@@ -590,9 +594,9 @@ const RecordPlayback = ({ route }) => {
try {
cameraRef.current?.stopRecording?.()
log('stopRecording triggered')
} catch (_) {}
} catch (_) { }
}
} catch (_) {}
} catch (_) { }
}, 500)
}
@@ -610,7 +614,7 @@ const RecordPlayback = ({ route }) => {
}
try {
if (player?.playing) await player.pause?.()
} catch (_) {}
} catch (_) { }
if (listenTimerRef.current) {
clearInterval(listenTimerRef.current)
listenTimerRef.current = null
@@ -643,7 +647,7 @@ const RecordPlayback = ({ route }) => {
await discardRecordingFile(video?.uri, 'too_short')
try {
goBack()
} catch (_) {}
} catch (_) { }
return
}
}
@@ -744,7 +748,7 @@ const RecordPlayback = ({ route }) => {
onPress: () => {
try {
goBack()
} catch (_) {}
} catch (_) { }
},
},
],
@@ -771,21 +775,21 @@ const RecordPlayback = ({ route }) => {
if (player?.playing) {
const maybePromise = player.pause?.()
if (maybePromise && typeof maybePromise.catch === 'function') {
maybePromise.catch(() => {})
maybePromise.catch(() => { })
}
}
} catch (_) {}
} catch (_) { }
try {
if (isRecording) {
cameraRef.current?.stopRecording?.()
}
} catch (_) {}
} catch (_) { }
} else {
exitRequestedRef.current = false
}
try {
goBack()
} catch (_) {}
} catch (_) { }
return true
}, [goBack, isPreparing, isRecording, player])
@@ -808,21 +812,21 @@ const RecordPlayback = ({ route }) => {
stopRequestedAtRef.current = 0
try {
await player?.pause?.()
} catch (_) {}
} catch (_) { }
try {
await cameraRef.current?.pausePreview?.()
} catch (_) {}
} catch (_) { }
return
}
setIsPaused(false)
isPausedRef.current = false
try {
await cameraRef.current?.resumePreview?.()
} catch (_) {}
} catch (_) { }
try {
await player?.play?.()
} catch (_) {}
} catch (_) {}
} catch (_) { }
} catch (_) { }
}, [isRecording, player])
const handleRestartRecording = async () => {
@@ -837,7 +841,7 @@ const RecordPlayback = ({ route }) => {
isPausedRef.current = false
try {
await cameraRef.current?.resumePreview?.()
} catch (_) {}
} catch (_) { }
if (isPreparing) {
if (hasRecordingPending) {
await startCountdownThenRecord({
@@ -864,16 +868,16 @@ const RecordPlayback = ({ route }) => {
stopRequestedAtRef.current = Date.now()
try {
if (player?.playing) await player.pause?.()
} catch (_) {}
} catch (_) { }
try {
cameraRef.current?.stopRecording?.()
} catch (_) {}
} catch (_) { }
await startCountdownThenRecord({
preserveRestartFlag: true,
preserveSongEndWatcher: true,
preserveStopRequest: true,
})
} catch (_) {}
} catch (_) { }
}
const permissionsGranted = !!cameraPermission?.granted
@@ -945,7 +949,7 @@ const RecordPlayback = ({ route }) => {
onPress={async () => {
try {
if (!cameraPermission?.granted) await requestCameraPermission()
} catch (_) {}
} catch (_) { }
}}
/>
</View>
src/screens/Playback/RecordPlayback.web.js
+44 -43
View File
@@ -44,7 +44,7 @@ const pickRecorderMimeType = () => {
for (const mimeType of candidates) {
try {
if (MediaRecorder.isTypeSupported(mimeType)) return mimeType
} catch {}
} catch { }
}
return undefined
@@ -162,7 +162,7 @@ const RecordPlayback = ({ route }) => {
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
mediaRecorderRef.current.stop()
}
} catch (e) {}
} catch (e) { }
recordedChunksRef.current = []
releaseRecordingUrl()
@@ -248,7 +248,7 @@ const RecordPlayback = ({ route }) => {
}
recorder.stop()
}
} catch (e) {}
} catch (e) { }
if (!waitPromise) {
return recordedUrlRef.current || null
@@ -272,10 +272,10 @@ const RecordPlayback = ({ route }) => {
try {
await player?.pause?.()
await player?.seekTo?.(0)
} catch (e) {}
} catch (e) { }
try {
await stopRecorderAndGetUrl()
} catch (e) {}
} catch (e) { }
if (!preserveRecordingForNextScreenRef.current) {
releaseRecordingUrl()
}
@@ -315,12 +315,12 @@ const RecordPlayback = ({ route }) => {
)
useEffect(() => {
;(async () => {
; (async () => {
try {
if (!cameraPermission?.granted) {
await requestCameraPermission()
}
} catch (e) {}
} catch (e) { }
})()
return () => {
clearAllTimers()
@@ -345,25 +345,25 @@ const RecordPlayback = ({ route }) => {
}
let cancelled = false
;(async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: 'user' },
audio: false,
})
if (cancelled) {
stream.getTracks().forEach((track) => track.stop())
return
; (async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: 'user' },
audio: false,
})
if (cancelled) {
stream.getTracks().forEach((track) => track.stop())
return
}
mediaStreamRef.current = stream
setMediaReady(true)
setMediaError(null)
} catch (e) {
if (cancelled) return
setMediaError(e instanceof Error ? e : new Error(String(e || '')))
setMediaReady(false)
}
mediaStreamRef.current = stream
setMediaReady(true)
setMediaError(null)
} catch (e) {
if (cancelled) return
setMediaError(e instanceof Error ? e : new Error(String(e || '')))
setMediaReady(false)
}
})()
})()
return () => {
cancelled = true
@@ -382,13 +382,13 @@ const RecordPlayback = ({ route }) => {
}
const playPromise = video.play?.()
if (playPromise && typeof playPromise.catch === 'function') {
playPromise.catch(() => {})
playPromise.catch(() => { })
}
} catch (e) {}
} catch (e) { }
} else {
try {
video.srcObject = null
} catch {}
} catch { }
}
}, [mediaReady, mediaError])
@@ -398,7 +398,7 @@ const RecordPlayback = ({ route }) => {
if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') {
mediaRecorderRef.current.stop()
}
} catch {}
} catch { }
if (mediaStreamRef.current) {
mediaStreamRef.current.getTracks().forEach((track) => track.stop())
mediaStreamRef.current = null
@@ -414,7 +414,7 @@ const RecordPlayback = ({ route }) => {
clearAllTimers()
try {
await player?.pause?.()
} catch (e) {}
} catch (e) { }
let videoUrl = null
try {
@@ -422,7 +422,7 @@ const RecordPlayback = ({ route }) => {
if (!videoUrl) {
videoUrl = recordedUrlRef.current || null
}
} catch (e) {}
} catch (e) { }
setIsRecording(false)
setIsPaused(false)
@@ -457,7 +457,7 @@ const RecordPlayback = ({ route }) => {
await player?.seekTo?.(0)
correctedInitialJumpRef.current = true
nextPos = 0
} catch (e) {}
} catch (e) { }
}
const timeSinceStart =
@@ -501,11 +501,11 @@ const RecordPlayback = ({ route }) => {
if (project?.id) {
try {
await projectsRef.doc(project.id).set({ views: increment(1) }, { merge: true })
} catch (e) {}
} catch (e) { }
}
}
}
} catch (e) {}
} catch (e) { }
}, 500)
}, [player, project?.id])
@@ -520,6 +520,7 @@ const RecordPlayback = ({ route }) => {
const recorderStarted = startRecorder()
if (!recorderStarted) {
}
await new Promise(r => setTimeout(r, 250))
await player?.play?.() // attend la promesse → l'élément est prêt
perfStartRef.current = perfStartRef.current ?? performance.now()
correctedInitialJumpRef.current = false // autorise la correction 1-shot
@@ -543,7 +544,7 @@ const RecordPlayback = ({ route }) => {
onPress: () => {
try {
goBack()
} catch {}
} catch { }
},
},
],
@@ -588,13 +589,13 @@ const RecordPlayback = ({ route }) => {
pausedAtRef.current = performance.now()
try {
await player?.pause?.()
} catch (e) {}
} catch (e) { }
try {
const recorder = mediaRecorderRef.current
if (recorder && recorder.state === 'recording') {
recorder.pause?.()
}
} catch (e) {}
} catch (e) { }
return
}
const pausedAt = pausedAtRef.current
@@ -609,17 +610,17 @@ const RecordPlayback = ({ route }) => {
if (recorder && recorder.state === 'paused') {
recorder.resume?.()
}
} catch (e) {}
} catch (e) { }
try {
await player?.play?.()
} catch (e) {}
} catch (e) {}
} catch (e) { }
} catch (e) { }
}, [isRecording, player])
const handleRestartRecording = useCallback(async () => {
try {
await startCountdownThenRecord()
} catch (e) {}
} catch (e) { }
}, [startCountdownThenRecord])
const progressRatio = dur > 0 ? Math.min(1, pos / dur) : 0
@@ -630,7 +631,7 @@ const RecordPlayback = ({ route }) => {
// containerStyle={{ backgroundColor: Palette.gray }}
headerType="NONE"
backgroundColor={Palette.grayMid}
// backgroundImg={background.playbackBG2}
// backgroundImg={background.playbackBG2}
>
<MusicLandHeader progress={9} onPressBack={goBack} />
@@ -848,7 +849,7 @@ const RecordPlayback = ({ route }) => {
onPress={async () => {
try {
if (!cameraPermission?.granted) await requestCameraPermission()
} catch {}
} catch { }
}}
/>
</View>
src/screens/Playback/RecordedPlayback.js
+13 -12
View File
@@ -12,6 +12,7 @@ import Page from '../../layouts/Page'
import { Routes } from '../../navigation'
import { goBack, navigate } from '../../navigation/NavigationService'
import { gutters, Palette } from '../../styles'
import { FONT_FAMILY } from '../../styles/Fonts'
const RecordedPlayback = ({ route }) => {
const { videoUri, project } = route.params || {}
const songUrl = project?.songUrl || null
@@ -40,10 +41,10 @@ const RecordedPlayback = ({ route }) => {
const stopPlayback = useCallback(async () => {
try {
if (audioPlayer?.playing) await audioPlayer.pause?.()
} catch (e) {}
} catch (e) { }
try {
if (videoPlayer?.playing) videoPlayer.pause()
} catch (e) {}
} catch (e) { }
}, [audioPlayer, videoPlayer])
useEffect(() => {
@@ -52,7 +53,7 @@ const RecordedPlayback = ({ route }) => {
try {
if (audioPlayer && songUrl) await audioPlayer.play?.()
if (videoPlayer) videoPlayer.play()
} catch (e) {}
} catch (e) { }
}
start()
return () => {
@@ -78,7 +79,7 @@ const RecordedPlayback = ({ route }) => {
videoPlayer.currentTime = Math.max(0, (pos || 0) / 1000)
}
}
} catch (e) {}
} catch (e) { }
}, 250)
return () => global.clearInterval(id)
}, [audioPlayer, videoPlayer])
@@ -89,7 +90,7 @@ const RecordedPlayback = ({ route }) => {
const pos = Math.max(0, Math.min(dur, Math.floor(targetMs)))
if (audioPlayer && dur > 0) await audioPlayer.seekTo?.(Math.floor(pos / 1000))
if (videoPlayer) videoPlayer.currentTime = Math.max(0, pos / 1000)
} catch (e) {}
} catch (e) { }
}
const onSeekStart = useCallback(() => {
@@ -99,19 +100,19 @@ const RecordedPlayback = ({ route }) => {
const pauseDuringSeek = useCallback(async () => {
try {
if (audioPlayer?.playing) await audioPlayer.pause?.()
} catch (e) {}
} catch (e) { }
try {
if (videoPlayer?.playing) videoPlayer.pause()
} catch (e) {}
} catch (e) { }
}, [audioPlayer, videoPlayer])
const resumeAfterSeek = useCallback(async () => {
try {
if (audioPlayer) await audioPlayer.play?.()
} catch (e) {}
} catch (e) { }
try {
if (videoPlayer) videoPlayer.play()
} catch (e) {}
} catch (e) { }
}, [audioPlayer, videoPlayer])
const sliderProgress = useMemo(() => {
@@ -155,7 +156,7 @@ const RecordedPlayback = ({ route }) => {
playbackEndedRef.current = false
if (songUrl && audioPlayer) await audioPlayer.play?.()
if (videoPlayer) videoPlayer.play()
} catch (e) {}
} catch (e) { }
}
return (
@@ -225,7 +226,7 @@ const RecordedPlayback = ({ route }) => {
onPress={async () => {
try {
await stopPlayback()
} catch (e) {}
} catch (e) { }
try {
if (videoUri) {
const info = await FileSystem.getInfoAsync(videoUri)
@@ -234,7 +235,7 @@ const RecordedPlayback = ({ route }) => {
idempotent: true,
})
}
} catch (e) {}
} catch (e) { }
navigate(Routes.RecordPlayback, { project })
}}
/>
src/screens/Playback/RecordedPlayback.web.js
+15 -15
View File
@@ -55,12 +55,12 @@ const RecordedPlayback = ({ route }) => {
const stopPlayback = useCallback(async () => {
try {
if (audioPlayer?.playing) await audioPlayer.pause?.()
} catch {}
} catch { }
try {
if (videoElRef.current && !videoElRef.current.paused) {
videoElRef.current.pause()
}
} catch {}
} catch { }
}, [audioPlayer])
useEffect(() => {
@@ -82,9 +82,9 @@ const RecordedPlayback = ({ route }) => {
if (videoElRef.current && videoUri) {
// Lecture vidéo HTML5 (muet pour éviter les policies)
videoElRef.current.muted = true
videoElRef.current.play().catch(() => {})
videoElRef.current.play().catch(() => { })
}
} catch {}
} catch { }
}
start()
@@ -115,7 +115,7 @@ const RecordedPlayback = ({ route }) => {
videoElRef.current.currentTime = Math.max(0, posS)
}
}
} catch {}
} catch { }
}, 250)
return () => clearInterval(id)
}, [audioPlayer, videoUri])
@@ -141,12 +141,12 @@ const RecordedPlayback = ({ route }) => {
await stopPlayback()
try {
if (audioPlayer) await audioPlayer.seekTo?.(0)
} catch {}
} catch { }
try {
if (videoElRef.current) {
videoElRef.current.currentTime = 0
}
} catch {}
} catch { }
})()
}
}, [progress, stopPlayback, audioPlayer])
@@ -163,7 +163,7 @@ const RecordedPlayback = ({ route }) => {
if (videoElRef.current && videoUri) {
videoElRef.current.currentTime = Math.max(0, target)
}
} catch {}
} catch { }
})()
pendingSeekRef.current = promise
await promise
@@ -177,7 +177,7 @@ const RecordedPlayback = ({ route }) => {
if (!promise) return
try {
await promise
} catch {}
} catch { }
}, [])
const wasPlayingRef = useRef(false)
@@ -193,7 +193,7 @@ const RecordedPlayback = ({ route }) => {
if (videoElRef.current && !videoElRef.current.paused) {
videoElRef.current.pause()
}
} catch {}
} catch { }
}, [audioPlayer])
const onSeekEnd = useCallback(async () => {
try {
@@ -202,9 +202,9 @@ const RecordedPlayback = ({ route }) => {
isSeekingRef.current = false
if (wasPlayingRef.current) {
if (audioPlayer) await audioPlayer.resume?.()
if (videoElRef.current && videoUri) videoElRef.current.play().catch(() => {})
if (videoElRef.current && videoUri) videoElRef.current.play().catch(() => { })
}
} catch {}
} catch { }
}, [audioPlayer, videoUri, waitForPendingSeek])
const handleTogglePlayback = useCallback(async () => {
@@ -233,9 +233,9 @@ const RecordedPlayback = ({ route }) => {
}
if (videoElRef.current && videoUri) {
videoElRef.current.muted = true
videoElRef.current.play().catch(() => {})
videoElRef.current.play().catch(() => { })
}
} catch {}
} catch { }
}, [audioPlayer, songUrl, stopPlayback, progress, videoUri])
return (
@@ -377,7 +377,7 @@ const RecordedPlayback = ({ route }) => {
onPress={async () => {
try {
await stopPlayback()
} catch {}
} catch { }
navigate(Routes.RecordPlayback, { project })
}}
/>