Merge pull request #61 from Dash-Industry-Forum/thumbnails

feat: DASH-IF thumbnail support in all segment template modes

CHANGELOG.md

@@ -9,9 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- support DASH-IF thumbnails including multi-period. New test sequence added
 - new URL parameter `timesubswvtt` provides generated timing wvtt subtitles
 - `timesubsstpp` and `timesubswvtt` now work with SegmentTimeline
-- `continous_1` URL parameter to signal multiperiod continuity
+- `continuous_1` URL parameter to signal multiperiod continuity
 - Automatic Let's Encrypt certificates for HTTPS for one or more domains via `domains` parameter
 
 ## [0.6.0] - 2023-06-10

README.md

@@ -63,8 +63,8 @@ Major values to configure are:
 
 * the top directory `vodroot` for searching for VoD assets to be used
 * the HTTPS `domains` if Let's Encrypt automatic certificates are used
-    - `certpath` and `keypath` if HTTPS is used with manually downloaded certificates
-    - the HTTP/HTTPS `port` if `domains` is not being used (default: 8888)
+  * `certpath` and `keypath` if HTTPS is used with manually downloaded certificates
+  * the HTTP/HTTPS `port` if `domains` is not being used (default: 8888)
 
 Once the server is started, it will scan the file tree starting from
 `vodroot` and gather metadata about all DASH VoD assets it finds.

cmd/livesim2/app/asset.go

@@ -120,7 +120,7 @@ func (am *assetMgr) loadAsset(mpdPath string) error {
 				return fmt.Errorf("getRep: %w", err)
 			}
 			if len(r.segments) == 0 {
-				return fmt.Errorf("rep %s has no segments", rep.Id)
+				return fmt.Errorf("rep %s of type %s has no segments", rep.Id, r.ContentType)
 			}
 			asset.Reps[r.ID] = r
 			avgSegDurMS := (r.duration() * 1000) / (r.MediaTimescale * len(r.segments))
@@ -179,27 +179,12 @@ func (am *assetMgr) loadRep(assetPath string, mpd *m.MPD, as *m.AdaptationSetTyp
 		rp.MpdTimescale = int(*st.Timescale)
 	}
 
-	data, err := fs.ReadFile(am.vodFS, path.Join(assetPath, rp.initURI))
-	if err != nil {
-		return nil, err
-	}
-	sr := bits.NewFixedSliceReader(data)
-	initFile, err := mp4.DecodeFileSR(sr)
-	if err != nil {
-		return nil, fmt.Errorf("decode init: %w", err)
-	}
-	rp.initSeg = initFile.Init
-	b := make([]byte, 0, rp.initSeg.Size())
-	buf := bytes.NewBuffer(b)
-	err = rp.initSeg.Encode(buf)
-	if err != nil {
-		return nil, fmt.Errorf("encode init seg: %w", err)
+	if rp.ContentType != "image" {
+		err := rp.readInit(am.vodFS, assetPath)
+		if err != nil {
+			return nil, err
+		}
 	}
-	rp.initBytes = buf.Bytes()
-
-	rp.MediaTimescale = int(rp.initSeg.Moov.Trak.Mdia.Mdhd.Timescale)
-	trex := rp.initSeg.Moov.Mvex.Trex
-	defaultSampleDuration := trex.DefaultSampleDuration
 
 	switch {
 	case st.SegmentTimeline != nil && rp.typeURI == timeURI:
@@ -227,45 +212,41 @@ func (am *assetMgr) loadRep(assetPath string, mpd *m.MPD, as *m.AdaptationSetTyp
 		if st.StartNumber != nil {
 			startNr = *st.StartNumber
 		}
-		endNr := startNr
+		endNr := startNr - 1
 		if st.EndNumber != nil {
 			endNr = *st.EndNumber
 		}
 		nr := startNr
-		var seg *mp4.File
+		var seg segment
+		var err error
+		var segDur uint64
+		if rp.ContentType == "image" && as.SegmentTemplate.Duration != nil {
+			segDur = uint64(*as.SegmentTemplate.Duration)
+			rp.MediaTimescale = int(as.SegmentTemplate.GetTimescale())
+		}
 		for {
-			uri := replaceTimeAndNr(rp.mediaURI, 0, nr)
-			repPath := path.Join(assetPath, uri)
-			data, err := fs.ReadFile(am.vodFS, repPath)
-			if err != nil {
-				break // No more files
+			if rp.ContentType != "image" {
+				seg, err = rp.readMP4Segment(am.vodFS, assetPath, nr)
+			} else {
+				seg, err = rp.readThumbSegment(am.vodFS, assetPath, nr, startNr, segDur)
 			}
-			sr := bits.NewFixedSliceReader(data)
-			seg, err = mp4.DecodeFileSR(sr)
 			if err != nil {
-				return nil, fmt.Errorf("decode %s: %w", repPath, err)
+				endNr = nr - 1
+				break
 			}
-			t := seg.Segments[0].Fragments[0].Moof.Traf.Tfdt.BaseMediaDecodeTime()
 			if nr > startNr {
-				rp.segments[len(rp.segments)-1].endTime = t
+				rp.segments[len(rp.segments)-1].endTime = seg.startTime
 			}
-			rp.segments = append(rp.segments, segment{uri, t, 0, nr})
-			nr++
+			rp.segments = append(rp.segments, seg)
 
 			if nr == endNr { // This only happens if endNumber is set
 				break
 			}
+			nr++
 		}
-		if nr == startNr {
+		if endNr < startNr {
 			return nil, fmt.Errorf("no segments read for rep %s", path.Join(assetPath, rp.mediaURI))
 		}
-		nf := len(seg.Segments[0].Fragments)
-		lastFragTraf := seg.Segments[0].Fragments[nf-1].Moof.Traf
-		if lastFragTraf.Tfhd.HasDefaultSampleDuration() {
-			defaultSampleDuration = lastFragTraf.Tfhd.DefaultSampleDuration
-		}
-		endTime := lastFragTraf.Tfdt.BaseMediaDecodeTime() + lastFragTraf.Trun.Duration(defaultSampleDuration)
-		rp.segments[len(rp.segments)-1].endTime = endTime
 	default:
 		return nil, fmt.Errorf("unknown type of representation")
 	}
@@ -408,18 +389,19 @@ const (
 
 // RepData provides information about a representation
 type RepData struct {
-	ID             string           `json:"id"`
-	ContentType    string           `json:"contentType"`
-	Codecs         string           `json:"codecs"`
-	MpdTimescale   int              `json:"mpdTimescale"`
-	MediaTimescale int              `json:"mediaTimescale"` // Used in the segments
-	initURI        string           `json:"-"`
-	mediaURI       string           `json:"-"`
-	typeURI        mediaURIType     `json:"-"`
-	mediaRegexp    *regexp.Regexp   `json:"-"`
-	initSeg        *mp4.InitSegment `json:"-"`
-	initBytes      []byte           `json:"-"`
-	segments       []segment        `json:"-"`
+	ID                    string           `json:"id"`
+	ContentType           string           `json:"contentType"`
+	Codecs                string           `json:"codecs"`
+	MpdTimescale          int              `json:"mpdTimescale"`
+	MediaTimescale        int              `json:"mediaTimescale"` // Used in the segments
+	initURI               string           `json:"-"`
+	mediaURI              string           `json:"-"`
+	typeURI               mediaURIType     `json:"-"`
+	mediaRegexp           *regexp.Regexp   `json:"-"`
+	initSeg               *mp4.InitSegment `json:"-"`
+	initBytes             []byte           `json:"-"`
+	defaultSampleDuration uint32           `json:"-"`
+	segments              []segment        `json:"-"`
 }
 
 func (r RepData) duration() int {
@@ -435,20 +417,89 @@ func (r RepData) findSegmentIndexFromTime(t uint64) int {
 	})
 }
 
-// SegmentTYpe returns MIME type for MP4 segment.
+// SegmentType returns MIME type for MP4 segment.
 func (r RepData) SegmentType() string {
 	var segType string
 	switch r.ContentType {
 	case "audio":
 		segType = "audio/mp4"
 	case "subtitle":
 		segType = "application/mp4"
-	default:
+	case "video":
 		segType = "video/mp4"
+	case "image":
+		segType = "image/jpeg"
+	default:
+		segType = "unknown_content_type"
 	}
 	return segType
 }
 
+func (r *RepData) readInit(vodFS fs.FS, assetPath string) error {
+	data, err := fs.ReadFile(vodFS, path.Join(assetPath, r.initURI))
+	if err != nil {
+		return fmt.Errorf("read initURI %q: %w", r.initURI, err)
+	}
+	sr := bits.NewFixedSliceReader(data)
+	initFile, err := mp4.DecodeFileSR(sr)
+	if err != nil {
+		return fmt.Errorf("decode init: %w", err)
+	}
+	r.initSeg = initFile.Init
+	b := make([]byte, 0, r.initSeg.Size())
+	buf := bytes.NewBuffer(b)
+	err = r.initSeg.Encode(buf)
+	if err != nil {
+		return fmt.Errorf("encode init seg: %w", err)
+	}
+	r.initBytes = buf.Bytes()
+
+	r.MediaTimescale = int(r.initSeg.Moov.Trak.Mdia.Mdhd.Timescale)
+	trex := r.initSeg.Moov.Mvex.Trex
+	r.defaultSampleDuration = trex.DefaultSampleDuration
+	return nil
+}
+
+func (r *RepData) readMP4Segment(vodFS fs.FS, assetPath string, nr uint32) (segment, error) {
+	var seg segment
+	uri := replaceTimeAndNr(r.mediaURI, 0, nr)
+	repPath := path.Join(assetPath, uri)
+
+	data, err := fs.ReadFile(vodFS, repPath)
+	if err != nil {
+		return seg, err
+	}
+	sr := bits.NewFixedSliceReader(data)
+	mp4Seg, err := mp4.DecodeFileSR(sr)
+	if err != nil {
+		return seg, fmt.Errorf("decode %s: %w", repPath, err)
+	}
+
+	t := mp4Seg.Segments[0].Fragments[0].Moof.Traf.Tfdt.BaseMediaDecodeTime()
+	nf := len(mp4Seg.Segments[0].Fragments)
+	lastFragTraf := mp4Seg.Segments[0].Fragments[nf-1].Moof.Traf
+	if lastFragTraf.Tfhd.HasDefaultSampleDuration() {
+		r.defaultSampleDuration = lastFragTraf.Tfhd.DefaultSampleDuration
+	}
+	endTime := lastFragTraf.Tfdt.BaseMediaDecodeTime() + lastFragTraf.Trun.Duration(r.defaultSampleDuration)
+	return segment{uri, t, endTime, nr}, nil
+}
+
+func (r *RepData) readThumbSegment(vodFS fs.FS, assetPath string, nr, startNr uint32, dur uint64) (segment, error) {
+	var seg segment
+	uri := replaceTimeAndNr(r.mediaURI, 0, nr)
+	repPath := path.Join(assetPath, uri)
+
+	info, err := fs.Stat(vodFS, repPath)
+	if err != nil {
+		fmt.Printf("%v\n", info)
+		return seg, err
+	}
+	deltaNr := nr - startNr
+	startTime := uint64(deltaNr) * dur
+	return segment{uri, startTime, startTime + dur, nr}, nil
+}
+
 func replaceIdentifiers(r *m.RepresentationType, str string) string {
 	str = strings.ReplaceAll(str, "$RepresentationID$", r.Id)
 	str = strings.ReplaceAll(str, "$Bandwidth$", strconv.Itoa(int(r.Bandwidth)))

cmd/livesim2/app/asset_test.go

@@ -19,7 +19,7 @@ func TestLoadAsset(t *testing.T) {
 	require.NoError(t, err)
 	asset, ok := am.findAsset("assets/testpic_2s")
 	require.True(t, ok)
-	require.Equal(t, 2, len(asset.Reps))
+	require.Equal(t, 3, len(asset.Reps))
 	rep := asset.Reps["V300"]
 	assert.Equal(t, "V300/init.mp4", rep.initURI)
 	assert.Equal(t, 4, len(rep.segments))

cmd/livesim2/app/configurl.go

@@ -106,14 +106,23 @@ func (rc *ResponseConfig) liveMPDType() liveMPDType {
 	}
 }
 
+// getRepType returns the live representation type depending on MPD and segment name (type).
+// Normally follows the MPD type, but for image (thumbnails), always returns segmentNumber.
+func (rc *ResponseConfig) getRepType(segName string) liveMPDType {
+	if isImage(segName) {
+		return segmentNumber
+	}
+	return rc.liveMPDType()
+}
+
 // getAvailabilityTimeOffset returns the availabilityTimeOffsetS. Note that it can be infinite.
 func (rc *ResponseConfig) getAvailabilityTimeOffsetS() float64 {
 	return rc.AvailabilityTimeOffsetS
 }
 
 // getStartNr for MPD. Default value if not set is 1.
 func (rc *ResponseConfig) getStartNr() int {
-	// Default startNr is 1, but can be overridden by actual value set in cfg.
+	// Default startNr is 1 according to spec, but can be overridden by actual value set in cfg.
 	if rc.StartNr != nil {
 		return *rc.StartNr
 	}

cmd/livesim2/app/livempd.go

@@ -113,7 +113,11 @@ func LiveMPD(a *asset, mpdName string, cfg *ResponseConfig, nowMS int) (*m.MPD,
 		if err != nil {
 			return nil, err
 		}
-		switch cfg.liveMPDType() {
+		templateType := cfg.liveMPDType()
+		if as.ContentType == "image" {
+			templateType = segmentNumber
+		}
+		switch templateType {
 		case timeLineTime:
 			err := adjustAdaptationSetForTimelineTime(cfg, se, as)
 			if err != nil {
@@ -209,7 +213,11 @@ func splitPeriod(mpd *m.MPD, a *asset, cfg *ResponseConfig, wTimes wrapTimes) er
 			inAS := inPeriod.AdaptationSets[aNr]
 			timeScale := int(as.SegmentTemplate.GetTimescale())
 			pto := Ptr(uint64(pNr * periodDur * timeScale))
-			switch cfg.liveMPDType() {
+			templateType := cfg.liveMPDType()
+			if as.ContentType == "image" {
+				templateType = segmentNumber
+			}
+			switch templateType {
 			case segmentNumber:
 				as.SegmentTemplate.PresentationTimeOffset = pto
 				segDur := int(*as.SegmentTemplate.Duration)

cmd/livesim2/app/livempd_test.go

@@ -8,6 +8,7 @@ import (
 	"math"
 	"os"
 	"path"
+	"strings"
 	"testing"
 	"time"
 
@@ -30,6 +31,13 @@ func TestLiveMPD(t *testing.T) {
 		timeMedia string
 		timescale int
 	}{
+		{
+			asset:     "testpic_2s",
+			mpdName:   "Manifest_thumbs.mpd",
+			nrMedia:   "$RepresentationID$/$Number$.m4s",
+			timeMedia: "$RepresentationID$/$Time$.m4s",
+			timescale: 1,
+		},
 		{
 			asset:     "WAVE/vectors/cfhd_sets/12.5_25_50/t3/2022-10-17",
 			mpdName:   "stream.mpd",
@@ -61,7 +69,13 @@ func TestLiveMPD(t *testing.T) {
 			stl := as.SegmentTemplate
 			assert.Nil(t, stl.SegmentTimeline)
 			assert.Equal(t, uint32(0), *stl.StartNumber)
-			assert.Equal(t, tc.nrMedia, stl.Media)
+			if as.ContentType != "image" {
+				assert.Equal(t, tc.nrMedia, stl.Media)
+			} else {
+				tcMedia := strings.Replace(tc.nrMedia, ".m4s", ".jpg", 1)
+				assert.Equal(t, tcMedia, stl.Media)
+			}
+
 			require.NotNil(t, stl.Duration)
 			require.Equal(t, tc.timescale, int(stl.GetTimescale()))
 			assert.Equal(t, 2, int(*stl.Duration)/int(stl.GetTimescale()))
@@ -77,8 +91,13 @@ func TestLiveMPD(t *testing.T) {
 			if as.ContentType == "video" {
 				require.Greater(t, stl.SegmentTimeline.S[0].R, 0)
 			}
-			assert.Nil(t, stl.StartNumber)
-			assert.Equal(t, tc.timeMedia, stl.Media)
+			if as.ContentType != "image" {
+				assert.Nil(t, stl.StartNumber)
+				assert.Equal(t, tc.timeMedia, stl.Media)
+			} else {
+				tcMedia := strings.Replace(tc.nrMedia, ".m4s", ".jpg", 1)
+				assert.Equal(t, tcMedia, stl.Media)
+			}
 		}
 		assert.Equal(t, 1, len(liveMPD.UTCTimings))
 	}
@@ -737,3 +756,7 @@ func TestRelStartStopTimeIntoLocation(t *testing.T) {
 		require.Equal(t, c.wantedLocation, string(liveMPD.Location[0]), "the right location element is not inserted")
 	}
 }
+
+func TestThumbnailAS(t *testing.T) {
+
+}