Fix OPUS for HomeKit server #667 #843

pkg/opus/README.md

@@ -0,0 +1,5 @@
+## Useful links
+
+- [RFC 3550: RTP: A Transport Protocol for Real-Time Applications](https://datatracker.ietf.org/doc/html/rfc3550)
+- [RFC 6716: Definition of the Opus Audio Codec](https://datatracker.ietf.org/doc/html/rfc6716)
+- [RFC 7587: RTP Payload Format for the Opus Speech and Audio Codec](https://datatracker.ietf.org/doc/html/rfc7587)

pkg/opus/homekit.go

@@ -0,0 +1,96 @@
+package opus
+
+import (
+	"github.com/AlexxIT/go2rtc/pkg/core"
+	"github.com/pion/rtp"
+)
+
+// Some info about this magic:
+// - Apple has no respect for RFC 7587 standard and using RFC 3550 for RTP timestamps
+// - Apple can request packets with 20ms duration over LAN connection and 60ms over LTE
+// - FFmpeg produce packets with 20ms duration by default and only one frame per packet
+// - FFmpeg should use "-min_comp 0" option, so every packet will be same duration
+// - Apple doesn't care about real sample rate of track
+// - Apple only cares about proper timestamp based on REQUESTED sample rate
+
+// RepackToHAP - convert standart RTP packet with OPUS to HAP packet
+// We expect that:
+// - incoming packet will be 20ms duration and only one frame per packet
+// - outgouing packet will be 20ms or 60ms duration
+// - incoming sample rate will be any (but not very big if we needs 60ms packets for output)
+// - outgouing sample rate will be 16000
+// https://github.com/AlexxIT/go2rtc/issues/667
+func RepackToHAP(rtpTime byte, handler core.HandlerFunc) core.HandlerFunc {
+	switch rtpTime {
+	case 20:
+		return repackToHAP20(handler)
+	case 60:
+		return repackToHAP60(handler)
+	}
+	return handler
+}
+
+// we using only one sample rate in the pkg/hap/camera/accessory.go
+const (
+	timestamp20 = 16000 * 0.020
+	timestamp60 = 16000 * 0.060
+)
+
+// repackToHAP20 - just fix RTP timestamp from RFC 7587 to RFC 3550
+func repackToHAP20(handler core.HandlerFunc) core.HandlerFunc {
+	var timestamp uint32
+
+	return func(pkt *rtp.Packet) {
+		timestamp += timestamp20
+
+		clone := *pkt
+		clone.Timestamp = timestamp
+		handler(&clone)
+	}
+}
+
+// repackToHAP60 - collect 20ms frames to single 60ms packet
+// thanks to @civita idea https://github.com/AlexxIT/go2rtc/pull/843
+func repackToHAP60(handler core.HandlerFunc) core.HandlerFunc {
+	var sequence uint16
+	var timestamp uint32
+
+	var framesCount byte
+	var framesSize []byte
+	var framesData []byte
+
+	return func(pkt *rtp.Packet) {
+		framesData = append(framesData, pkt.Payload[1:]...)
+
+		if framesCount++; framesCount < 3 {
+			if frameSize := len(pkt.Payload) - 1; frameSize >= 252 {
+				b0 := 252 + byte(frameSize)&0b11
+				framesSize = append(framesSize, b0, byte(frameSize/4)-b0)
+			} else {
+				framesSize = append(framesSize, byte(frameSize))
+			}
+			return
+		}
+
+		toc := pkt.Payload[0]
+
+		payload := make([]byte, 2, len(framesSize)+len(framesData))
+		payload[0] = toc | 0b11  // code 3 (multiple frames per packet)
+		payload[1] = 0b1000_0011 // VBR, no padding, 3 frames
+		payload = append(payload, framesSize...)
+		payload = append(payload, framesData...)
+
+		sequence++
+		timestamp += timestamp60
+
+		clone := *pkt
+		clone.Payload = payload
+		clone.SequenceNumber = sequence
+		clone.Timestamp = timestamp
+		handler(&clone)
+
+		framesCount = 0
+		framesSize = framesSize[:0]
+		framesData = framesData[:0]
+	}
+}

pkg/opus/opus.go

@@ -0,0 +1,69 @@
+package opus
+
+import (
+	"time"
+
+	"github.com/AlexxIT/go2rtc/pkg/core"
+	"github.com/pion/rtp"
+	"github.com/rs/zerolog/log"
+)
+
+func Log(handler core.HandlerFunc) core.HandlerFunc {
+	var ts uint32
+
+	return func(pkt *rtp.Packet) {
+		if ts == 0 {
+			ts = pkt.Timestamp
+		}
+
+		toc := pkt.Payload[0]
+		//config := toc >> 3
+		code := toc & 0b11
+
+		frame := parseFrameSize(toc)
+		rate := parseSampleRate(toc)
+
+		log.Printf(
+			"[RTP/OPUS] frame=%s rate=%5d code=%d size=%6d ts=%10d dt=%5d pt=%2d ssrc=%d seq=%d mark=%t",
+			frame, rate, code, len(pkt.Payload), pkt.Timestamp, pkt.Timestamp-ts, pkt.PayloadType, pkt.SSRC, pkt.SequenceNumber, pkt.Marker,
+		)
+
+		ts = pkt.Timestamp
+
+		handler(pkt)
+	}
+}
+
+func parseFrameSize(toc byte) time.Duration {
+	switch toc >> 3 {
+	case 0, 4, 8, 12, 14, 18, 22, 26, 30:
+		return 10_000_000
+	case 1, 5, 9, 13, 15, 19, 23, 27, 31:
+		return 20_000_000
+	case 2, 6, 10:
+		return 40_000_000
+	case 3, 7, 11:
+		return 60_000_000
+	case 16, 20, 24, 28:
+		return 2_500_000
+	case 17, 21, 25, 29:
+		return 5_000_000
+	}
+	return 0
+}
+
+func parseSampleRate(toc byte) uint16 {
+	switch toc >> 3 {
+	case 0, 1, 2, 3, 16, 17, 18, 19:
+		return 8000
+	case 4, 5, 6, 7:
+		return 12000
+	case 8, 9, 10, 11, 20, 21, 22, 23:
+		return 16000
+	case 12, 13, 24, 25, 26, 27:
+		return 24000
+	case 14, 15, 28, 29, 30, 31:
+		return 48000
+	}
+	return 0
+}