Total rewrite mov/mp4 encoder

This commit is contained in:
Alexey Khit
2023-01-26 22:29:12 +03:00
parent e295bc4eaf
commit d05ab79f88
10 changed files with 1157 additions and 164 deletions
+100
View File
@@ -0,0 +1,100 @@
package mp4
import (
"encoding/binary"
"github.com/deepch/vdk/format/mp4/mp4io"
"github.com/deepch/vdk/format/mp4f"
"github.com/deepch/vdk/format/mp4f/mp4fio"
"time"
)
var matrix = [9]int32{0x10000, 0, 0, 0, 0x10000, 0, 0, 0, 0x40000000}
var time0 = time.Date(1904, time.January, 1, 0, 0, 0, 0, time.UTC)
func FTYP() []byte {
b := make([]byte, 0x18)
binary.BigEndian.PutUint32(b, 0x18)
copy(b[0x04:], "ftyp")
copy(b[0x08:], "iso5")
copy(b[0x10:], "iso5")
copy(b[0x14:], "avc1")
return b
}
func MOOV() *mp4io.Movie {
return &mp4io.Movie{
Header: &mp4io.MovieHeader{
PreferredRate: 1,
PreferredVolume: 1,
Matrix: matrix,
NextTrackId: -1,
Duration: 0,
TimeScale: 1000,
CreateTime: time0,
ModifyTime: time0,
PreviewTime: time0,
PreviewDuration: time0,
PosterTime: time0,
SelectionTime: time0,
SelectionDuration: time0,
CurrentTime: time0,
},
MovieExtend: &mp4io.MovieExtend{},
}
}
func TRAK(id int) *mp4io.Track {
return &mp4io.Track{
// trak > tkhd
Header: &mp4io.TrackHeader{
TrackId: int32(id),
Flags: 0x0007, // 7 ENABLED IN-MOVIE IN-PREVIEW
Duration: 0, // OK
Matrix: matrix,
CreateTime: time0,
ModifyTime: time0,
},
// trak > mdia
Media: &mp4io.Media{
// trak > mdia > mdhd
Header: &mp4io.MediaHeader{
TimeScale: 1000,
Duration: 0,
Language: 0x55C4,
CreateTime: time0,
ModifyTime: time0,
},
// trak > mdia > minf
Info: &mp4io.MediaInfo{
// trak > mdia > minf > dinf
Data: &mp4io.DataInfo{
Refer: &mp4io.DataRefer{
Url: &mp4io.DataReferUrl{
Flags: 0x000001, // self reference
},
},
},
// trak > mdia > minf > stbl
Sample: &mp4io.SampleTable{
SampleDesc: &mp4io.SampleDesc{},
TimeToSample: &mp4io.TimeToSample{},
SampleToChunk: &mp4io.SampleToChunk{},
SampleSize: &mp4io.SampleSize{},
ChunkOffset: &mp4io.ChunkOffset{},
},
},
},
}
}
func ESDS(conf []byte) *mp4f.FDummy {
esds := &mp4fio.ElemStreamDesc{DecConfig: conf}
b := make([]byte, esds.Len())
esds.Marshal(b)
return &mp4f.FDummy{
Data: b,
Tag_: mp4io.Tag(uint32(mp4io.ESDS)),
}
}
+174
View File
@@ -0,0 +1,174 @@
package mp4
import (
"encoding/json"
"github.com/AlexxIT/go2rtc/pkg/aac"
"github.com/AlexxIT/go2rtc/pkg/h264"
"github.com/AlexxIT/go2rtc/pkg/h265"
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/pion/rtp"
"sync/atomic"
)
type Consumer struct {
streamer.Element
Medias []*streamer.Media
UserAgent string
RemoteAddr string
muxer *Muxer
codecs []*streamer.Codec
wait byte
send uint32
}
const (
waitNone byte = iota
waitKeyframe
waitInit
)
func (c *Consumer) GetMedias() []*streamer.Media {
if c.Medias != nil {
return c.Medias
}
// default medias
return []*streamer.Media{
{
Kind: streamer.KindVideo,
Direction: streamer.DirectionRecvonly,
Codecs: []*streamer.Codec{
{Name: streamer.CodecH264},
{Name: streamer.CodecH265},
},
},
{
Kind: streamer.KindAudio,
Direction: streamer.DirectionRecvonly,
Codecs: []*streamer.Codec{
{Name: streamer.CodecAAC},
},
},
}
}
func (c *Consumer) AddTrack(media *streamer.Media, track *streamer.Track) *streamer.Track {
trackID := byte(len(c.codecs))
c.codecs = append(c.codecs, track.Codec)
codec := track.Codec
switch codec.Name {
case streamer.CodecH264:
c.wait = waitInit
push := func(packet *rtp.Packet) error {
if packet.Version != h264.RTPPacketVersionAVC {
return nil
}
if c.wait != waitNone {
if c.wait == waitInit || !h264.IsKeyframe(packet.Payload) {
return nil
}
c.wait = waitNone
}
buf := c.muxer.Marshal(trackID, packet)
atomic.AddUint32(&c.send, uint32(len(buf)))
c.Fire(buf)
return nil
}
var wrapper streamer.WrapperFunc
if codec.IsRTP() {
wrapper = h264.RTPDepay(track)
} else {
wrapper = h264.RepairAVC(track)
}
push = wrapper(push)
return track.Bind(push)
case streamer.CodecH265:
c.wait = waitInit
push := func(packet *rtp.Packet) error {
if packet.Version != h264.RTPPacketVersionAVC {
return nil
}
if c.wait != waitNone {
if c.wait == waitInit || !h265.IsKeyframe(packet.Payload) {
return nil
}
c.wait = waitNone
}
buf := c.muxer.Marshal(trackID, packet)
atomic.AddUint32(&c.send, uint32(len(buf)))
c.Fire(buf)
return nil
}
if codec.IsRTP() {
wrapper := h265.RTPDepay(track)
push = wrapper(push)
}
return track.Bind(push)
case streamer.CodecAAC:
push := func(packet *rtp.Packet) error {
if c.wait != waitNone {
return nil
}
buf := c.muxer.Marshal(trackID, packet)
atomic.AddUint32(&c.send, uint32(len(buf)))
c.Fire(buf)
return nil
}
if codec.IsRTP() {
wrapper := aac.RTPDepay(track)
push = wrapper(push)
}
return track.Bind(push)
}
panic("unsupported codec")
}
func (c *Consumer) MimeType() string {
return c.muxer.MimeType(c.codecs)
}
func (c *Consumer) Init() ([]byte, error) {
c.muxer = &Muxer{}
return c.muxer.GetInit(c.codecs)
}
func (c *Consumer) Start() {
if c.wait == waitInit {
c.wait = waitKeyframe
}
}
//
func (c *Consumer) MarshalJSON() ([]byte, error) {
info := &streamer.Info{
Type: "MP4 client",
RemoteAddr: c.RemoteAddr,
UserAgent: c.UserAgent,
Send: atomic.LoadUint32(&c.send),
}
return json.Marshal(info)
}
+256
View File
@@ -0,0 +1,256 @@
package mp4
import (
"encoding/binary"
"encoding/hex"
"github.com/AlexxIT/go2rtc/pkg/h264"
"github.com/AlexxIT/go2rtc/pkg/h265"
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/deepch/vdk/av"
"github.com/deepch/vdk/codec/h264parser"
"github.com/deepch/vdk/codec/h265parser"
"github.com/deepch/vdk/format/fmp4/fmp4io"
"github.com/deepch/vdk/format/mp4/mp4io"
"github.com/deepch/vdk/format/mp4f/mp4fio"
"github.com/pion/rtp"
)
type Muxer struct {
fragIndex uint32
dts []uint64
pts []uint32
}
func (m *Muxer) MimeType(codecs []*streamer.Codec) string {
s := `video/mp4; codecs="`
for i, codec := range codecs {
if i > 0 {
s += ","
}
switch codec.Name {
case streamer.CodecH264:
s += "avc1." + h264.GetProfileLevelID(codec.FmtpLine)
case streamer.CodecH265:
// H.265 profile=main level=5.1
// hvc1 - supported in Safari, hev1 - doesn't, both supported in Chrome
s += "hvc1.1.6.L153.B0"
case streamer.CodecAAC:
s += "mp4a.40.2"
}
}
return s + `"`
}
func (m *Muxer) GetInit(codecs []*streamer.Codec) ([]byte, error) {
moov := MOOV()
for i, codec := range codecs {
switch codec.Name {
case streamer.CodecH264:
sps, pps := h264.GetParameterSet(codec.FmtpLine)
if sps == nil {
// some dummy SPS and PPS not a problem
sps = []byte{0x67, 0x42, 0x00, 0x0a, 0xf8, 0x41, 0xa2}
pps = []byte{0x68, 0xce, 0x38, 0x80}
}
codecData, err := h264parser.NewCodecDataFromSPSAndPPS(sps, pps)
if err != nil {
return nil, err
}
width := codecData.Width()
height := codecData.Height()
trak := TRAK(i + 1)
trak.Header.TrackWidth = float64(width)
trak.Header.TrackHeight = float64(height)
trak.Media.Header.TimeScale = int32(codec.ClockRate)
trak.Media.Handler = &mp4io.HandlerRefer{
SubType: [4]byte{'v', 'i', 'd', 'e'},
Name: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'm', 'a', 'i', 'n', 0},
}
trak.Media.Info.Video = &mp4io.VideoMediaInfo{
Flags: 0x000001,
}
trak.Media.Info.Sample.SampleDesc.AVC1Desc = &mp4io.AVC1Desc{
DataRefIdx: 1,
HorizontalResolution: 72,
VorizontalResolution: 72,
Width: int16(width),
Height: int16(height),
FrameCount: 1,
Depth: 24,
ColorTableId: -1,
Conf: &mp4io.AVC1Conf{
Data: codecData.AVCDecoderConfRecordBytes(),
},
}
moov.Tracks = append(moov.Tracks, trak)
case streamer.CodecH265:
vps, sps, pps := h265.GetParameterSet(codec.FmtpLine)
if sps == nil {
// some dummy SPS and PPS not a problem
vps = []byte{0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x01, 0x40, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x99, 0xac, 0x09}
sps = []byte{0x42, 0x01, 0x01, 0x01, 0x40, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x99, 0xa0, 0x01, 0x40, 0x20, 0x05, 0xa1, 0xfe, 0x5a, 0xee, 0x46, 0xc1, 0xae, 0x55, 0x04}
pps = []byte{0x44, 0x01, 0xc0, 0x73, 0xc0, 0x4c, 0x90}
}
codecData, err := h265parser.NewCodecDataFromVPSAndSPSAndPPS(vps, sps, pps)
if err != nil {
return nil, err
}
width := codecData.Width()
height := codecData.Height()
trak := TRAK(i + 1)
trak.Header.TrackWidth = float64(width)
trak.Header.TrackHeight = float64(height)
trak.Media.Header.TimeScale = int32(codec.ClockRate)
trak.Media.Handler = &mp4io.HandlerRefer{
SubType: [4]byte{'v', 'i', 'd', 'e'},
Name: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'm', 'a', 'i', 'n', 0},
}
trak.Media.Info.Video = &mp4io.VideoMediaInfo{
Flags: 0x000001,
}
trak.Media.Info.Sample.SampleDesc.HV1Desc = &mp4io.HV1Desc{
DataRefIdx: 1,
HorizontalResolution: 72,
VorizontalResolution: 72,
Width: int16(width),
Height: int16(height),
FrameCount: 1,
Depth: 24,
ColorTableId: -1,
Conf: &mp4io.HV1Conf{
Data: codecData.AVCDecoderConfRecordBytes(),
},
}
moov.Tracks = append(moov.Tracks, trak)
case streamer.CodecAAC:
s := streamer.Between(codec.FmtpLine, "config=", ";")
b, err := hex.DecodeString(s)
if err != nil {
return nil, err
}
trak := TRAK(i + 1)
trak.Header.AlternateGroup = 1
trak.Header.Duration = 0
trak.Header.Volume = 1
trak.Media.Header.TimeScale = int32(codec.ClockRate)
trak.Media.Handler = &mp4io.HandlerRefer{
SubType: [4]byte{'s', 'o', 'u', 'n'},
Name: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'm', 'a', 'i', 'n', 0},
}
trak.Media.Info.Sound = &mp4io.SoundMediaInfo{}
trak.Media.Info.Sample.SampleDesc.MP4ADesc = &mp4io.MP4ADesc{
DataRefIdx: 1,
NumberOfChannels: int16(codec.Channels),
SampleSize: int16(av.FLTP.BytesPerSample() * 4),
SampleRate: float64(codec.ClockRate),
Unknowns: []mp4io.Atom{ESDS(b)},
}
moov.Tracks = append(moov.Tracks, trak)
}
trex := &mp4io.TrackExtend{
TrackId: uint32(i + 1),
DefaultSampleDescIdx: 1,
DefaultSampleDuration: 0,
}
moov.MovieExtend.Tracks = append(moov.MovieExtend.Tracks, trex)
m.pts = append(m.pts, 0)
m.dts = append(m.dts, 0)
}
data := make([]byte, moov.Len())
moov.Marshal(data)
return append(FTYP(), data...), nil
}
func (m *Muxer) Reset() {
m.fragIndex = 0
for i := range m.dts {
m.dts[i] = 0
m.pts[i] = 0
}
}
func (m *Muxer) Marshal(trackID byte, packet *rtp.Packet) []byte {
run := &mp4fio.TrackFragRun{
Flags: 0x000b05,
FirstSampleFlags: uint32(fmp4io.SampleNoDependencies),
DataOffset: 0,
Entries: []mp4io.TrackFragRunEntry{},
}
moof := &mp4fio.MovieFrag{
Header: &mp4fio.MovieFragHeader{
Seqnum: m.fragIndex + 1,
},
Tracks: []*mp4fio.TrackFrag{
{
Header: &mp4fio.TrackFragHeader{
Data: []byte{0x00, 0x02, 0x00, 0x20, 0x00, 0x00, 0x00, trackID + 1, 0x01, 0x01, 0x00, 0x00},
},
DecodeTime: &mp4fio.TrackFragDecodeTime{
Version: 1,
Flags: 0,
Time: m.dts[trackID],
},
Run: run,
},
},
}
entry := mp4io.TrackFragRunEntry{
Size: uint32(len(packet.Payload)),
}
newTime := packet.Timestamp
if m.pts[trackID] > 0 {
entry.Duration = newTime - m.pts[trackID]
m.dts[trackID] += uint64(entry.Duration)
} else {
// important, or Safari will fail with first frame
entry.Duration = 1
}
m.pts[trackID] = newTime
// important before moof.Len()
run.Entries = append(run.Entries, entry)
moofLen := moof.Len()
mdatLen := 8 + len(packet.Payload)
// important after moof.Len()
run.DataOffset = uint32(moofLen + 8)
buf := make([]byte, moofLen+mdatLen)
moof.Marshal(buf)
binary.BigEndian.PutUint32(buf[moofLen:], uint32(mdatLen))
copy(buf[moofLen+4:], "mdat")
copy(buf[moofLen+8:], packet.Payload)
m.fragIndex++
//m.total += moofLen + mdatLen
return buf
}
+143
View File
@@ -0,0 +1,143 @@
package mp4
import (
"encoding/json"
"github.com/AlexxIT/go2rtc/pkg/h264"
"github.com/AlexxIT/go2rtc/pkg/h265"
"github.com/AlexxIT/go2rtc/pkg/streamer"
"github.com/pion/rtp"
"sync/atomic"
)
type Segment struct {
streamer.Element
Medias []*streamer.Media
UserAgent string
RemoteAddr string
MimeType string
OnlyKeyframe bool
send uint32
}
func (c *Segment) GetMedias() []*streamer.Media {
if c.Medias != nil {
return c.Medias
}
// default medias
return []*streamer.Media{
{
Kind: streamer.KindVideo,
Direction: streamer.DirectionRecvonly,
Codecs: []*streamer.Codec{
{Name: streamer.CodecH264},
{Name: streamer.CodecH265},
},
},
}
}
func (c *Segment) AddTrack(media *streamer.Media, track *streamer.Track) *streamer.Track {
muxer := &Muxer{}
codecs := []*streamer.Codec{track.Codec}
init, err := muxer.GetInit(codecs)
if err != nil {
return nil
}
c.MimeType = muxer.MimeType(codecs)
switch track.Codec.Name {
case streamer.CodecH264:
var push streamer.WriterFunc
if c.OnlyKeyframe {
push = func(packet *rtp.Packet) error {
if !h264.IsKeyframe(packet.Payload) {
return nil
}
buf := muxer.Marshal(0, packet)
atomic.AddUint32(&c.send, uint32(len(buf)))
c.Fire(append(init, buf...))
return nil
}
} else {
var buf []byte
push = func(packet *rtp.Packet) error {
if h264.IsKeyframe(packet.Payload) {
// fist frame - send only IFrame
// other frames - send IFrame and all PFrames
if buf == nil {
buf = append(buf, init...)
b := muxer.Marshal(0, packet)
buf = append(buf, b...)
}
atomic.AddUint32(&c.send, uint32(len(buf)))
c.Fire(buf)
buf = buf[:0]
buf = append(buf, init...)
muxer.Reset()
}
if buf != nil {
b := muxer.Marshal(0, packet)
buf = append(buf, b...)
}
return nil
}
}
var wrapper streamer.WrapperFunc
if track.Codec.IsRTP() {
wrapper = h264.RTPDepay(track)
} else {
wrapper = h264.RepairAVC(track)
}
push = wrapper(push)
return track.Bind(push)
case streamer.CodecH265:
push := func(packet *rtp.Packet) error {
if !h265.IsKeyframe(packet.Payload) {
return nil
}
buf := muxer.Marshal(0, packet)
atomic.AddUint32(&c.send, uint32(len(buf)))
c.Fire(append(init, buf...))
return nil
}
if track.Codec.IsRTP() {
wrapper := h265.RTPDepay(track)
push = wrapper(push)
}
return track.Bind(push)
}
panic("unsupported codec")
}
func (c *Segment) MarshalJSON() ([]byte, error) {
info := &streamer.Info{
Type: "WS/MP4 client",
RemoteAddr: c.RemoteAddr,
UserAgent: c.UserAgent,
Send: atomic.LoadUint32(&c.send),
}
return json.Marshal(info)
}