Add support FLAC codec to MP4/MSE

pkg/mp4/consumer.go

@@ -6,6 +6,7 @@ import (
 	"github.com/AlexxIT/go2rtc/pkg/core"
 	"github.com/AlexxIT/go2rtc/pkg/h264"
 	"github.com/AlexxIT/go2rtc/pkg/h265"
+	"github.com/AlexxIT/go2rtc/pkg/pcm"
 	"github.com/pion/rtp"
 	"sync"
 )
@@ -54,7 +55,8 @@ func (c *Consumer) GetMedias() []*core.Media {
 func (c *Consumer) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiver) error {
 	trackID := byte(len(c.senders))
 
-	handler := core.NewSender(media, track.Codec)
+	codec := track.Codec.Clone()
+	handler := core.NewSender(media, codec)
 
 	switch track.Codec.Name {
 	case core.CodecH264:
@@ -112,38 +114,33 @@ func (c *Consumer) AddTrack(media *core.Media, _ *core.Codec, track *core.Receiv
 			handler.Handler = h265.RTPDepay(track.Codec, handler.Handler)
 		}
 
-	case core.CodecAAC:
-		handler.Handler = func(packet *rtp.Packet) {
-			if c.wait != waitNone {
-				return
-			}
-
-			c.mu.Lock()
-			buf := c.muxer.Marshal(trackID, packet)
-			c.Fire(buf)
-			c.send += len(buf)
-			c.mu.Unlock()
-		}
-
-		if track.Codec.IsRTP() {
-			handler.Handler = aac.RTPDepay(handler.Handler)
-		}
-
-	case core.CodecOpus, core.CodecMP3, core.CodecPCMU, core.CodecPCMA:
-		handler.Handler = func(packet *rtp.Packet) {
-			if c.wait != waitNone {
-				return
-			}
-
-			c.mu.Lock()
-			buf := c.muxer.Marshal(trackID, packet)
-			c.Fire(buf)
-			c.send += len(buf)
-			c.mu.Unlock()
-		}
-
 	default:
-		panic("unsupported codec")
+		handler.Handler = func(packet *rtp.Packet) {
+			if c.wait != waitNone {
+				return
+			}
+
+			c.mu.Lock()
+			buf := c.muxer.Marshal(trackID, packet)
+			c.Fire(buf)
+			c.send += len(buf)
+			c.mu.Unlock()
+		}
+
+		switch track.Codec.Name {
+		case core.CodecAAC:
+			if track.Codec.IsRTP() {
+				handler.Handler = aac.RTPDepay(handler.Handler)
+			}
+		case core.CodecOpus, core.CodecMP3: // no changes
+		case core.CodecPCMA, core.CodecPCMU, core.CodecPCM:
+			handler.Handler = pcm.FLACEncoder(track.Codec, handler.Handler)
+			codec.Name = core.CodecFLAC
+
+		default:
+			println("ERROR: MP4 unsupported codec: " + track.Codec.Name)
+			return nil
+		}
 	}
 
 	handler.HandleRTP(track)

pkg/mp4/muxer.go

@@ -15,12 +15,14 @@ type Muxer struct {
 	fragIndex uint32
 	dts       []uint64
 	pts       []uint32
+	codecs    []*core.Codec
 }
 
 const (
 	MimeH264 = "avc1.640029"
 	MimeH265 = "hvc1.1.6.L153.B0"
 	MimeAAC  = "mp4a.40.2"
+	MimeFlac = "flac"
 	MimeOpus = "opus"
 )
 
@@ -43,6 +45,8 @@ func (m *Muxer) MimeCodecs(codecs []*core.Codec) string {
 			s += MimeAAC
 		case core.CodecOpus:
 			s += MimeOpus
+		case core.CodecFLAC:
+			s += MimeFlac
 		}
 	}
 
@@ -108,14 +112,15 @@ func (m *Muxer) GetInit(codecs []*core.Codec) ([]byte, error) {
 				uint32(i+1), codec.Name, codec.ClockRate, codec.Channels, b,
 			)
 
-		case core.CodecOpus, core.CodecMP3, core.CodecPCMU, core.CodecPCMA:
+		case core.CodecOpus, core.CodecMP3, core.CodecPCMA, core.CodecPCMU, core.CodecPCM, core.CodecFLAC:
 			mv.WriteAudioTrack(
 				uint32(i+1), codec.Name, codec.ClockRate, codec.Channels, nil,
 			)
 		}
 
-		m.pts = append(m.pts, 0)
 		m.dts = append(m.dts, 0)
+		m.pts = append(m.pts, 0)
+		m.codecs = append(m.codecs, codec)
 	}
 
 	mv.StartAtom(iso.MoovMvex)
@@ -138,28 +143,49 @@ func (m *Muxer) Reset() {
 }
 
 func (m *Muxer) Marshal(trackID byte, packet *rtp.Packet) []byte {
-	// important before increment
-	time := m.dts[trackID]
+	codec := m.codecs[trackID]
+
+	duration := packet.Timestamp - m.pts[trackID]
+	m.pts[trackID] = packet.Timestamp
+
+	// minumum duration important for MSE in Apple Safari
+	if duration == 0 || duration > codec.ClockRate {
+		duration = codec.ClockRate/1000 + 1
+		m.pts[trackID] += duration
+	}
+
+	size := len(packet.Payload)
+
+	// flags important for Apple Finder video preview
+	var flags uint32
+	switch codec.Name {
+	case core.CodecH264:
+		if h264.IsKeyframe(packet.Payload) {
+			flags = iso.SampleVideoIFrame
+		} else {
+			flags = iso.SampleVideoNonIFrame
+		}
+	case core.CodecH265:
+		if h265.IsKeyframe(packet.Payload) {
+			flags = iso.SampleVideoIFrame
+		} else {
+			flags = iso.SampleVideoNonIFrame
+		}
+	default:
+		flags = iso.SampleAudio // not important
+	}
 
 	m.fragIndex++
 
-	var duration uint32
-	newTime := packet.Timestamp
-	if m.pts[trackID] > 0 {
-		duration = newTime - m.pts[trackID]
-		m.dts[trackID] += uint64(duration)
-	} else {
-		// important, or Safari will fail with first frame
-		duration = 1
-	}
-	m.pts[trackID] = newTime
-
-	mv := iso.NewMovie(1024 + len(packet.Payload))
+	mv := iso.NewMovie(1024 + size)
 	mv.WriteMovieFragment(
-		m.fragIndex, uint32(trackID+1), duration,
-		uint32(len(packet.Payload)), time,
+		m.fragIndex, uint32(trackID+1), duration, uint32(size), flags, m.dts[trackID],
 	)
 	mv.WriteData(packet.Payload)
 
+	//log.Printf("[MP4] track=%d ts=%6d dur=%5d idx=%3d len=%d", trackID+1, m.dts[trackID], duration, m.fragIndex, len(packet.Payload))
+
+	m.dts[trackID] += uint64(duration)
+
 	return mv.Bytes()
 }