From 7521f5939f48515fd5bba1dce5bf05776de84ddf Mon Sep 17 00:00:00 2001 From: Leon Morival Date: Thu, 3 Sep 2026 11:06:08 +0200 Subject: [PATCH] feat: try lipsinck --- functions/src/lyrics.js | 47 +++++- functions/src/music.js | 203 +++++++++++++++--------- functions/src/project.js | 7 +- src/providers/PlayerProvider.js | 2 +- src/screens/Library/MusicDetails.js | 4 +- src/screens/Library/MusicDetails.web.js | 3 +- src/screens/Studio/ComposeSong.js | 7 + src/screens/Studio/ComposeSong.web.js | 7 + src/screens/Studio/SongReady.js | 4 + 9 files changed, 196 insertions(+), 88 deletions(-) diff --git a/functions/src/lyrics.js b/functions/src/lyrics.js index 3f16d19..ba9f9a5 100644 --- a/functions/src/lyrics.js +++ b/functions/src/lyrics.js @@ -259,7 +259,7 @@ ${structureTags || fallbackStructureTags} 0. ${languageInstruction} 1. Génère un **Titre** percutant (moins de 6 mots). 2. Pour chaque section
, écris le contenu adapté : - - Si c'est "Instrumental" ou "Solo" : laisse le champ 'lyrics' vide ou mets une brève indication d'ambiance entre parenthèses ex: "(Solo de guitare déchirant)". + - Si c'est "Instrumental", "Intro", "Pont" ou "Solo" : le champ 'lyrics' doit être strictement vide. N'ajoute aucune indication entre parenthèses. - Si c'est "Couplet/Refrain" : Écris 4 à 12 vers. 3. **IMPORTANT** : Le style est "${style}". Assure-toi que le vocabulaire et le rythme collent parfaitement à ce genre. @@ -428,15 +428,16 @@ async function getSunoTimestamps(projectId) { const docSnap = await refList.projects.doc(projectId).get() if (!docSnap.exists) throw new Error('Projet introuvable') - const { sunoTaskId, songIndex } = docSnap.data() - if (!sunoTaskId) throw new Error('TaskId manquant') + const { songSunoTaskId, sunoAudioId, songIndex } = docSnap.data() + if (!songSunoTaskId) throw new Error('TaskId de la piste manquant') + if (!sunoAudioId) throw new Error('AudioId Suno manquant') if (songIndex === undefined || songIndex < 0) throw new Error('musicIndex invalide') - console.log('🔎 getSunoTimestamps', { sunoTaskId, songIndex }) + console.log('🔎 getSunoTimestamps', { songSunoTaskId, sunoAudioId, songIndex }) const response = await axios.post( `${SUNO_API_BASE}${SUNO_TIMESTAMPED_LYRICS_PATH}`, - { taskId: sunoTaskId, musicIndex: songIndex }, + { taskId: songSunoTaskId, audioId: sunoAudioId }, { headers: { 'Content-Type': 'application/json', @@ -446,11 +447,32 @@ async function getSunoTimestamps(projectId) { } ) - const dataToReturn = response.data?.data || response.data || {} + if (response.data?.code !== 200) { + throw new Error(response.data?.msg || 'Récupération des timestamps refusée par Suno') + } + + const dataToReturn = response.data?.data + if (!Array.isArray(dataToReturn?.alignedWords) || dataToReturn.alignedWords.length === 0) { + throw new Error('Aucune parole synchronisée retournée par Suno') + } + const failedWordCount = dataToReturn.alignedWords.filter( + (word) => word?.success === false + ).length + const hootCer = Number(dataToReturn.hootCer) await refList.projects.doc(projectId).set( { musicTimestamps: { [songIndex]: dataToReturn }, + lyricsSyncStatus: 'READY', + lyricsSyncError: FieldValue.delete(), + lyricsSyncDiagnostics: { + alignedWordCount: dataToReturn.alignedWords.length, + failedWordCount, + successfulWordRatio: + (dataToReturn.alignedWords.length - failedWordCount) / + dataToReturn.alignedWords.length, + hootCer: Number.isFinite(hootCer) ? hootCer : null, + }, updatedAt: FieldValue.serverTimestamp(), }, { merge: true } @@ -459,6 +481,19 @@ async function getSunoTimestamps(projectId) { return { success: true } } catch (error) { console.error('❌ getSunoTimestamps Error:', error.message) + if (projectId) { + await refList.projects + .doc(projectId) + .set( + { + lyricsSyncStatus: 'FAILED', + lyricsSyncError: error.message, + updatedAt: FieldValue.serverTimestamp(), + }, + { merge: true } + ) + .catch(() => {}) + } return { success: false, error: { message: error.message, type: 'INTERNAL_ERROR' }, diff --git a/functions/src/music.js b/functions/src/music.js index 530364d..f40926f 100644 --- a/functions/src/music.js +++ b/functions/src/music.js @@ -19,6 +19,8 @@ const { const MUSIC_GENERATION_CREDIT_COST = 8 const MUSIC_REFUND_SOURCE = 'music_generation_refund' +const SUNO_LYRICS_PROMPT_MAX_LENGTH = 5000 +const SUNO_TITLE_MAX_LENGTH = 100 // ========================================== // 1. DICTIONNAIRES DE TRADUCTION (FRONT -> SUNO) @@ -161,30 +163,54 @@ function parseVoiceTags(voiceSelections = []) { /** * Convertit les phrases de structure custom en Balises Suno */ -function mapStructureToTag(description) { - const d = description.toLowerCase() +const STRUCTURE_TAGS = { + couplet: '[Verse]', + refrain: '[Chorus]', + short_intro: '[Short Instrumental Intro]', + long_intro: '[Long Instrumental Intro]', + pre_refrain_instrumental: '[Instrumental Pre-Chorus]', + pont: '[Instrumental Bridge]', + solo_de_guitare: '[Guitar Solo]', + solo_de_guitare_electrique: '[Electric Guitar Solo]', + solo_de_batterie: '[Drum Solo]', + solo_de_saxophone: '[Saxophone Solo]', + solo_de_violon: '[Violin Solo]', + break: '[Instrumental Break]', + interlude: '[Instrumental Interlude]', + interlude_melodique: '[Melodic Instrumental Interlude]', + final_apogee: '[Big Finish]', + arret_net: '[Sudden End]', + fade_out: '[Fade Out]', + transition_douce: '[Fade to Silence]', +} - if (d.includes('intro') && d.includes('court')) return '[Short Intro]' - if (d.includes('intro')) return '[Intro]' - if (d.includes('pré-refrain')) return '[Pre-Chorus]' - if (d.includes('solo guitare électrique')) return '[Electric Guitar Solo]' - if (d.includes('solo guitare')) return '[Guitar Solo]' - if (d.includes('solo batterie')) return '[Drum Solo]' - if (d.includes('solo saxo')) return '[Saxophone Solo]' - if (d.includes('solo violon')) return '[Violin Solo]' - if (d.includes('interlude')) return '[Instrumental Interlude]' - if (d.includes('apogée') || d.includes('finition')) return '[Big Finish]' - if (d.includes('arrêt net')) return '[Sudden End]' - if (d.includes('baissant le volume') || d.includes('fade')) return '[Fade Out]' - if (d.includes('silence')) return '[Fade to Silence]' - if (d.includes('break') || d.includes('pause')) return '[Break]' +const INSTRUMENTAL_STRUCTURE_TYPES = new Set([ + 'short_intro', + 'long_intro', + 'pre_refrain_instrumental', + 'pont', + 'solo_de_guitare', + 'solo_de_guitare_electrique', + 'solo_de_batterie', + 'solo_de_saxophone', + 'solo_de_violon', + 'break', + 'interlude', + 'interlude_melodique', + 'final_apogee', + 'arret_net', + 'fade_out', + 'transition_douce', +]) - // Mapping standard des types lyrics - if (d === 'couplet' || d === 'verse') return '[Verse]' - if (d === 'refrain' || d === 'chorus') return '[Chorus]' - if (d === 'pont' || d === 'bridge') return '[Bridge]' - - return null // Pas de tag trouvé +function mapStructureToTag(type) { + return ( + STRUCTURE_TAGS[ + String(type || '') + .trim() + .toLowerCase() + ] || null + ) } // ========================================== @@ -244,44 +270,24 @@ function buildFormattedLyrics(lyricsArray) { if (!Array.isArray(lyricsArray) || lyricsArray.length === 0) return '' const formattedLines = lyricsArray.map((section) => { - // Le frontend peut envoyer soit { type: "...", lyrics: "..." } soit juste une string description pour les instrumentaux - const typeOrDescription = section.type || section.description || '' - const textContent = section.lyrics || '' + const type = String(section?.type || '') + .trim() + .toLowerCase() + const textContent = typeof section?.lyrics === 'string' ? section.lyrics.trim() : '' + const tag = mapStructureToTag(type) - // Essayer de trouver un tag spécial (ex: "Introduction instrumentale") - let tag = mapStructureToTag(typeOrDescription) - - // Fallback si pas de tag spécial trouvé mais type standard if (!tag) { - if (typeOrDescription.toLowerCase().includes('couplet')) tag = '[Verse]' - else if (typeOrDescription.toLowerCase().includes('refrain')) tag = '[Chorus]' - else tag = `[${typeOrDescription}]` // Fallback générique + throw new HttpsError('invalid-argument', `Structure musicale non reconnue : ${type}.`) } - // Si c'est une section instrumentale (pas de lyrics) - if (!textContent.trim()) { + if (INSTRUMENTAL_STRUCTURE_TYPES.has(type) || !textContent) { return `\n${tag}\n` } - return `\n${tag}\n${textContent.trim()}` + return `\n${tag}\n${textContent}` }) - // Sécurité: Ajouter Intro et Outro si absents (Suno best practice) - const fullText = formattedLines.join('\n') - let finalPrompt = fullText - - if (!fullText.includes('[Intro]')) { - finalPrompt = '[Intro]\n' + finalPrompt - } - if ( - !fullText.includes('[Outro]') && - !fullText.includes('[Fade Out]') && - !fullText.includes('[Sudden End]') - ) { - finalPrompt = finalPrompt + '\n\n[Outro]' - } - - return finalPrompt.trim() + return formattedLines.join('\n').trim() } // ========================================== @@ -324,10 +330,14 @@ const parseSunoCallbackPayload = (rawBody = {}) => { return { code, status, taskId, tracks } } -const extractAudioUrlsFromTracks = (tracks = []) => +const extractGeneratedTracks = (tracks = []) => tracks - .map((t) => t.audio_url || t.audioUrl || t.stream_audio_url || t.streamAudioUrl) - .filter(Boolean) + .map((track) => ({ + audioId: typeof track?.id === 'string' ? track.id.trim() : '', + audioUrl: typeof track?.audio_url === 'string' ? track.audio_url.trim() : '', + durationS: Number(track?.duration), + })) + .filter((track) => track.audioId && track.audioUrl) .slice(0, 2) const fetchProjectByTaskId = async (taskId) => { @@ -437,31 +447,62 @@ const downloadTrackToStorage = async (url, { userId, projectId, taskId, bucket, } } -const saveTracksToStorage = async (audioUrls, meta) => { - if (!audioUrls?.length) return [] +const saveTracksToStorage = async (tracks, meta) => { + if (!tracks?.length) return [] const bucket = admin.storage().bucket() const results = await Promise.all( - audioUrls.map((url, index) => downloadTrackToStorage(url, { ...meta, bucket, index })) + tracks.map(async (track, index) => { + const stored = await downloadTrackToStorage(track.audioUrl, { ...meta, bucket, index }) + if (!stored) return null + return { + url: stored.url, + audioId: track.audioId, + durationS: Number.isFinite(track.durationS) && track.durationS > 0 ? track.durationS : null, + } + }) ) - return results.filter(Boolean).map((entry) => entry.url) + return results.filter(Boolean) } -const mergeMusicUrls = async (projectRef, newUrls) => { - const sanitized = sanitizeMusicUrls(newUrls) - let existing = [] - try { - existing = sanitizeMusicUrls((await projectRef.get())?.data()?.musicUrls) - } catch (e) { } - const final = [...new Set([...existing, ...sanitized])] +const saveGeneratedTracks = async (projectRef, tracks, taskId) => { + const projectData = (await projectRef.get())?.data() || {} + const existingUrls = sanitizeMusicUrls(projectData.musicUrls) + const existingAudioIds = Array.isArray(projectData.musicAudioIds) ? projectData.musicAudioIds : [] + const existingDurations = Array.isArray(projectData.musicDurations) + ? projectData.musicDurations + : [] + const existingTaskIds = Array.isArray(projectData.musicTaskIds) ? projectData.musicTaskIds : [] + + const mergedTracks = existingUrls.map((url, index) => ({ + url, + audioId: existingAudioIds[index] || null, + durationS: existingDurations[index] || null, + taskId: existingTaskIds[index] || null, + })) + const knownAudioIds = new Set(mergedTracks.map((track) => track.audioId).filter(Boolean)) + + tracks.forEach((track) => { + if (knownAudioIds.has(track.audioId)) return + mergedTracks.push({ ...track, taskId }) + knownAudioIds.add(track.audioId) + }) + + const musicUrls = mergedTracks.map((track) => track.url) + const musicAudioIds = mergedTracks.map((track) => track.audioId) + const musicDurations = mergedTracks.map((track) => track.durationS) + const musicTaskIds = mergedTracks.map((track) => track.taskId) await projectRef.set( { musicStatus: 'GENERATED', - musicUrls: final, + musicUrls, + musicAudioIds, + musicDurations, + musicTaskIds, musicError: FieldValue.delete(), }, { merge: true } ) - return final + return musicUrls } // ========================================== @@ -478,7 +519,7 @@ exports.generateMusic = onCall({ secrets: [SUNO_API_KEY] }, async ({ data = {} } songStyle = '', // "Upbeat" voice = [], // ["Une voix féminine...", "Un cri brut..."] (array ou objet) instruments = [], // ["Piano", "Violon"] - rhythm = '', // "Très rapide" + tempo = '', // "Très rapide" // Les champs suivants (step 1) ne sont plus utilisés dans le style Suno direct pour éviter le bruit, // mais ont déjà servi à générer les paroles (lyrics). // audience, context, objective, etc. @@ -506,18 +547,27 @@ exports.generateMusic = onCall({ secrets: [SUNO_API_KEY] }, async ({ data = {} } songStyle, voiceData, instruments, - rhythm, + rhythm: tempo, }) // 2. Construction du PROMPT (Paroles + Structure) const formattedPrompt = buildFormattedLyrics(lyrics) - const safeTitle = clampLen(title, 80) + if (!formattedPrompt) { + throw new HttpsError('invalid-argument', 'Les paroles à mettre en musique sont vides.') + } + if (formattedPrompt.length > SUNO_LYRICS_PROMPT_MAX_LENGTH) { + throw new HttpsError( + 'invalid-argument', + `Les paroles dépassent la limite de ${SUNO_LYRICS_PROMPT_MAX_LENGTH} caractères du modèle ${SUNO_MODEL}.` + ) + } + const safeTitle = clampLen(title, SUNO_TITLE_MAX_LENGTH) // 3. Détection du genre vocal pour le param vocalGender (optimisation v3.5) // On regarde si on trouve 'male' ou 'female' dans les tags générés let vGender = null - if (optimizedStyle.includes('Female')) vGender = 'female' - else if (optimizedStyle.includes('Male')) vGender = 'male' + if (optimizedStyle.includes('Female')) vGender = 'f' + else if (optimizedStyle.includes('Male')) vGender = 'm' console.log('🎵 [Suno Optimisation] Result:', { style: optimizedStyle, @@ -531,7 +581,7 @@ exports.generateMusic = onCall({ secrets: [SUNO_API_KEY] }, async ({ data = {} } customMode: true, instrumental: false, model: SUNO_MODEL || 'chirp-v3-5', // Toujours viser le dernier modèle - prompt: clampLen(formattedPrompt, 3000), + prompt: formattedPrompt, title: safeTitle, style: optimizedStyle, callBackUrl: SUNO_CALLBACK_URL || '', @@ -562,7 +612,7 @@ exports.generateMusic = onCall({ secrets: [SUNO_API_KEY] }, async ({ data = {} } console.error('❌ Erreur generateMusic:', error) try { await markProjectMusicFailure(data?.projectId, error) - } catch (e) { } + } catch (e) {} const status = error?.response?.status || 'INTERNAL_ERROR' const message = error?.response?.data?.msg || error?.message || 'Erreur Suno' @@ -607,12 +657,13 @@ exports.sunoCallback = onRequest({ methods: ['POST'], memory: '1GiB' }, async (r const { projectId, projectData, projectRef } = await fetchProjectByTaskId(taskId) const { userId, projectTitle } = formatProjectMeta(projectData) - const storedUrls = await saveTracksToStorage(extractAudioUrlsFromTracks(tracks), { + const storedTracks = await saveTracksToStorage(extractGeneratedTracks(tracks), { userId, projectId, taskId, }) - const musicUrls = await mergeMusicUrls(projectRef, storedUrls) + if (!storedTracks.length) throw new Error('SUNO_TRACK_DOWNLOAD_FAILED') + const musicUrls = await saveGeneratedTracks(projectRef, storedTracks, taskId) if (userId) { try { diff --git a/functions/src/project.js b/functions/src/project.js index 7f439a0..8faecce 100644 --- a/functions/src/project.js +++ b/functions/src/project.js @@ -39,7 +39,12 @@ exports.onProjectWritten = onDocumentWritten(PROJECT_FUNCTION_OPTIONS, async (pr } else if (!beforeData) { //song create } else { - if (!beforeData?.songUrl && afterData?.songUrl) { + const selectedSongChanged = + !!afterData?.songUrl && + (beforeData?.songUrl !== afterData.songUrl || + beforeData?.sunoAudioId !== afterData?.sunoAudioId) + + if (selectedSongChanged) { await getSunoTimestamps(projectId) await refList.projects.doc(projectId).update({ hasSong: true }) } diff --git a/src/providers/PlayerProvider.js b/src/providers/PlayerProvider.js index 8253f10..0c69351 100644 --- a/src/providers/PlayerProvider.js +++ b/src/providers/PlayerProvider.js @@ -374,7 +374,7 @@ const PlayerProvider = ({ children }) => { return currentTrack.source }, [currentTrack?.source]) - const player = useAudioPlayer(source || null, 200) + const player = useAudioPlayer(source || null, 100) const status = useAudioPlayerStatus(player) const shouldHideOnRoute = activeRouteName ? HIDDEN_ROUTE_NAMES.has(activeRouteName) : false const isStatusLoaded = useMemo(() => { diff --git a/src/screens/Library/MusicDetails.js b/src/screens/Library/MusicDetails.js index b5cd4b1..5ca9de6 100644 --- a/src/screens/Library/MusicDetails.js +++ b/src/screens/Library/MusicDetails.js @@ -138,7 +138,7 @@ const MusicDetails = ({ route }) => { }, [project?.musicTimestamps, project?.songIndex]) const flatLines = useMemo(() => sections.flatMap((s) => s.lines), [sections]) - const visibleTimeS = (positionMs || 0) / 1000 + 0.3 // Offset de 300ms pour fluidité visuelle + const visibleTimeS = (positionMs || 0) / 1000 const currentLineIdx = useMemo(() => { if (!flatLines.length) return -1 @@ -414,4 +414,4 @@ const styles = StyleSheet.create({ textTransform: 'uppercase', marginBottom: 6, }, -}) \ No newline at end of file +}) diff --git a/src/screens/Library/MusicDetails.web.js b/src/screens/Library/MusicDetails.web.js index b6c9be1..fa3897b 100644 --- a/src/screens/Library/MusicDetails.web.js +++ b/src/screens/Library/MusicDetails.web.js @@ -548,8 +548,7 @@ const MusicDetails = ({ route }) => { // Current time in seconds for highlighting const currentTimeS = useMemo(() => Math.max(0, (positionMs || 0) / 1000), [positionMs]) - // Preview lead: show words 0.5s earlier - const visibleTimeS = useMemo(() => Math.max(0, currentTimeS + 0.5), [currentTimeS]) + const visibleTimeS = currentTimeS // Helpers to group aligned words into sections and timed lines const SECTION_TAG_REGEX = /^\s*\[([^\]]+)\]\s*$/i const SECTION_TAG_LEADING_REGEX = /^\s*\[([^\]]+)\]\s*/i diff --git a/src/screens/Studio/ComposeSong.js b/src/screens/Studio/ComposeSong.js index 465d7f7..abad6da 100644 --- a/src/screens/Studio/ComposeSong.js +++ b/src/screens/Studio/ComposeSong.js @@ -148,6 +148,13 @@ const ComposeSong = () => { if (!isRegenerationFlow) { payload.musicUrls = firebase.firestore.FieldValue.delete() + payload.musicAudioIds = firebase.firestore.FieldValue.delete() + payload.musicDurations = firebase.firestore.FieldValue.delete() + payload.musicTaskIds = firebase.firestore.FieldValue.delete() + payload.musicTimestamps = firebase.firestore.FieldValue.delete() + payload.lyricsSyncStatus = firebase.firestore.FieldValue.delete() + payload.lyricsSyncError = firebase.firestore.FieldValue.delete() + payload.lyricsSyncDiagnostics = firebase.firestore.FieldValue.delete() } await updateProjectData(payload) diff --git a/src/screens/Studio/ComposeSong.web.js b/src/screens/Studio/ComposeSong.web.js index 5e9f906..f5fd816 100644 --- a/src/screens/Studio/ComposeSong.web.js +++ b/src/screens/Studio/ComposeSong.web.js @@ -204,6 +204,13 @@ const ComposeSong = () => { if (!isRegenerationFlow) { payload.musicUrls = firebase.firestore.FieldValue.delete() + payload.musicAudioIds = firebase.firestore.FieldValue.delete() + payload.musicDurations = firebase.firestore.FieldValue.delete() + payload.musicTaskIds = firebase.firestore.FieldValue.delete() + payload.musicTimestamps = firebase.firestore.FieldValue.delete() + payload.lyricsSyncStatus = firebase.firestore.FieldValue.delete() + payload.lyricsSyncError = firebase.firestore.FieldValue.delete() + payload.lyricsSyncDiagnostics = firebase.firestore.FieldValue.delete() } await updateProjectData(payload) diff --git a/src/screens/Studio/SongReady.js b/src/screens/Studio/SongReady.js index a3ea88b..1c89587 100644 --- a/src/screens/Studio/SongReady.js +++ b/src/screens/Studio/SongReady.js @@ -118,6 +118,10 @@ const SongReady = () => { await updateProjectData({ songIndex: selectedIndex, songUrl: url, + sunoAudioId: selectedProject?.musicAudioIds?.[selectedIndex] || null, + songSunoTaskId: selectedProject?.musicTaskIds?.[selectedIndex] || null, + lyricsSyncStatus: 'PENDING', + lyricsSyncError: null, cover: null, coverUrl: null, coverStatus: null,