Force audio rate to 90khz to comply with rtsp spec

Makefile

@@ -76,6 +76,8 @@ THIRDPARTY_INC_DIR = ./3rdparty/install/include
 
 STRIP_FLAG := $(if $(filter 0,$(DEBUG_STRIP)),,"-s")
 
+STRIP_FLAG := $(if $(filter 0,$(DEBUG_STRIP)),,"-s")
+
 $(VERSION_FILE): $(SRC_DIR)/version.tpl.hpp
 	@if ! grep -q "$(commit_tag)" version.h > /dev/null 2>&1; then \
 		echo "Updating version.h to $(commit_tag)"; \

src/AudioReframer.cpp

@@ -1,14 +1,18 @@
 #include "AudioReframer.hpp"
 #include <algorithm>
 #include <stdexcept>
+#include <chrono>
 
 AudioReframer::AudioReframer(unsigned int inputSampleRate, unsigned int inputSamplesPerFrame, unsigned int outputSamplesPerFrame)
     : inputSampleRate(inputSampleRate),
       inputSamplesPerFrame(inputSamplesPerFrame),
       outputSamplesPerFrame(outputSamplesPerFrame),
       currentTimestamp(0),
       samplesAccumulated(0),
-      buffer(2 * std::max(inputSamplesPerFrame, outputSamplesPerFrame) * sizeof(uint16_t))
+      buffer(2 * std::max(inputSamplesPerFrame, outputSamplesPerFrame) * sizeof(uint16_t)),
+      base_timestamp(0),
+      timestamp_initialized(false),
+      resetCounter(0)
 {
     if (inputSamplesPerFrame == 0 || outputSamplesPerFrame == 0)
     {
@@ -23,14 +27,24 @@ void AudioReframer::addFrame(const uint8_t* frameData, int64_t timestamp)
         throw std::invalid_argument("Frame data cannot be null.");
     }
 
+    // Reset the timestamp base periodically to prevent long-term drift
+    // This helps maintain synchronization with video over time
+    if (resetCounter >= 300) { // Reset roughly every 300 frames
+        timestamp_initialized = false;
+        resetCounter = 0;
+    }
+
     size_t inputFrameSize = inputSamplesPerFrame * sizeof(uint16_t);
     buffer.push(frameData, inputFrameSize);
 
-    if (samplesAccumulated == 0)
-    {
-        currentTimestamp = timestamp; // Initialize timestamp with the first frame
+    // For the first frame or after a reset, initialize timestamps to start at zero
+    if (!timestamp_initialized) {
+        currentTimestamp = 0; // Start at zero
+        base_timestamp = timestamp;
+        timestamp_initialized = true;
     }
 
+    resetCounter++;
     samplesAccumulated += inputSamplesPerFrame;
 }
 
@@ -49,8 +63,16 @@ void AudioReframer::getReframedFrame(uint8_t* frameData, int64_t& timestamp)
     buffer.fetch(frameData, outputFrameSize);
     samplesAccumulated -= outputSamplesPerFrame;
 
+    // Return a clean timestamp that is a multiple of frame duration
+    // This ensures consistent intervals between audio frames
     timestamp = currentTimestamp;
-    currentTimestamp += (outputSamplesPerFrame * 1000) / inputSampleRate;
+
+    // Calculate frame duration with high precision in microseconds
+    // This is critical for maintaining audio/video sync
+    int64_t frameDuration = (int64_t)((double)outputSamplesPerFrame * 1000000.0 / (double)inputSampleRate);
+
+    // Update the timestamp for the next frame
+    currentTimestamp += frameDuration;
 }
 
 bool AudioReframer::hasMoreFrames() const

src/AudioReframer.hpp

@@ -23,6 +23,11 @@ class AudioReframer
     int64_t currentTimestamp;
     size_t samplesAccumulated;
 
+    // For timestamp normalization and drift prevention
+    int64_t base_timestamp;
+    bool timestamp_initialized;
+    unsigned int resetCounter;
+
     RingBuffer buffer;
 };
 

src/IMPAudioServerMediaSubsession.cpp

@@ -57,7 +57,7 @@ RTPSink* IMPAudioServerMediaSubsession::createNewRTPSink(
     FramedSource* inputSource)
 {
     unsigned rtpPayloadFormat = rtpPayloadTypeIfDynamic;
-    unsigned rtpTimestampFrequency = global_audio[audioChn]->imp_audio->sample_rate;
+    unsigned rtpTimestampFrequency = 90000;
     const char* rtpPayloadFormatName = "L16";
     bool allowMultipleFramesPerPacket = true;
     int outChnCnt = cfg->audio.force_stereo ? 2 : 1;

src/IMPDeviceSource.cpp

@@ -1,6 +1,8 @@
 #include "IMPDeviceSource.hpp"
 #include <iostream>
 #include "GroupsockHelper.hh"
+#include <thread>
+#include <chrono>
 
 // explicit instantiation
 template class IMPDeviceSource<H264NALUnit, video_stream>;
@@ -14,7 +16,8 @@ IMPDeviceSource<FrameType, Stream> *IMPDeviceSource<FrameType, Stream>::createNe
 
 template<typename FrameType, typename Stream>
 IMPDeviceSource<FrameType, Stream>::IMPDeviceSource(UsageEnvironment &env, int encChn, std::shared_ptr<Stream> stream, const char *name)
-    : FramedSource(env), encChn(encChn), stream{stream}, name{name}, eventTriggerId(0)     
+    : FramedSource(env), encChn(encChn), stream{stream}, name{name}, eventTriggerId(0), 
+      firstFrame(true), base_timestamp(0), timestamp_initialized(false), droppedFrames(0)
 {
     std::lock_guard lock_stream {mutex_main};
     std::lock_guard lock_callback {stream->onDataCallbackLock};
@@ -65,20 +68,94 @@ void IMPDeviceSource<FrameType, Stream>::deliverFrame()
     FrameType nal;
     if (stream->msgChannel->read(&nal))
     {
+        // Add frame pacing delays to prevent overwhelming the network stack
+        // This is critical for proper stream delivery, especially for I-frames
+        bool isIFrame = false;
+        if constexpr (std::is_same_v<FrameType, H264NALUnit>) {
+            // Check if this is an I-frame (typically much larger)
+            if (nal.data.size() > 0 && nal.data.size() > 10000) {
+                // I-frames are typically much larger, use a longer delay
+                isIFrame = true;
+                std::this_thread::sleep_for(std::chrono::milliseconds(3));
+            } else {
+                // Regular inter frames get a smaller delay
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            }
+        }
+
+        // Check if the frame is too large for the buffer
         if (nal.data.size() > fMaxSize)
         {
-            fFrameSize = fMaxSize;
-            fNumTruncatedBytes = nal.data.size() - fMaxSize;
+            // Track dropped frames for diagnostics
+            droppedFrames++;
+
+            // If we drop too many frames in succession, log a warning
+            if (droppedFrames % 10 == 1) {
+                LOG_ERROR("Frame size " << nal.data.size() << " exceeds buffer size " << fMaxSize << 
+                          ". Dropped " << droppedFrames << " frames to avoid corruption.");
+            }
+
+            fFrameSize = 0;
+            FramedSource::afterGetting(this);
+            return;
         }
         else
         {
             fFrameSize = nal.data.size();
+            // Reset the dropped frames counter when we successfully process a frame
+            if (droppedFrames > 0) {
+                droppedFrames = 0;
+            }
         }
 
-        /* timestamp fix, can be removed if solved
-        fPresentationTime = nal.time;
-        */
-        gettimeofday(&fPresentationTime, NULL);
+        // Timestamp normalization: ensure the first frame has zero timestamp
+        // and all subsequent frames use normalized timestamps
+        if (firstFrame) {
+            // First frame ALWAYS has timestamp zero for clean synchronization
+            struct timeval zero_time;
+            zero_time.tv_sec = 0;
+            zero_time.tv_usec = 0;  // Exactly zero for first frame
+            fPresentationTime = zero_time;
+            firstFrame = false;
+
+            // Initialize our base timestamp for future normalization
+            if constexpr (std::is_same_v<FrameType, H264NALUnit>) {
+                base_timestamp = nal.imp_ts;
+            } else {
+                // For audio frames, convert the timeval to microseconds
+                base_timestamp = (nal.time.tv_sec * 1000000LL) + nal.time.tv_usec;
+            }
+            timestamp_initialized = true;
+        } else {
+            // For subsequent frames, use normalized timestamps that start from zero
+            struct timeval normalized_time;
+
+            if (timestamp_initialized) {
+                int64_t frame_ts;
+
+                if constexpr (std::is_same_v<FrameType, H264NALUnit>) {
+                    frame_ts = nal.imp_ts;
+                } else {
+                    // For audio frames, convert the timeval to microseconds
+                    frame_ts = (nal.time.tv_sec * 1000000LL) + nal.time.tv_usec;
+                }
+
+                // Calculate normalized timestamp (relative to base timestamp)
+                int64_t normalized_ts = frame_ts - base_timestamp;
+
+                // Ensure we never use negative timestamps
+                if (normalized_ts < 0) normalized_ts = 0;
+
+                // Convert back to timeval format
+                normalized_time.tv_sec = normalized_ts / 1000000;
+                normalized_time.tv_usec = normalized_ts % 1000000;
+
+                fPresentationTime = normalized_time;
+            } else {
+                // Fallback if timestamp wasn't initialized (shouldn't happen)
+                fPresentationTime = nal.time;
+            }
+        }
 
         memcpy(fTo, &nal.data[0], fFrameSize);
 

src/IMPDeviceSource.hpp

@@ -34,6 +34,12 @@ class IMPDeviceSource : public FramedSource
     std::shared_ptr<Stream> stream;
     std::string name;   // for printing
     EventTriggerId eventTriggerId;
+    // For synchronization tracking
+    bool firstFrame;
+    int64_t base_timestamp;
+    bool timestamp_initialized;
+    // For tracking dropped frames
+    unsigned int droppedFrames;
 };
 
 #endif

src/globals.hpp

@@ -28,10 +28,8 @@ struct AudioFrame
 struct H264NALUnit
 {
 	std::vector<uint8_t> data;
-    /* timestamp fix, can be removed if solved
 	struct timeval time;
 	int64_t imp_ts;
-    */
 };
 
 struct jpeg_stream
@@ -84,13 +82,17 @@ struct audio_stream
     std::mutex onDataCallbackLock; // protects onDataCallback from deallocation
     std::condition_variable should_grab_frames;
     std::binary_semaphore is_activated{0};
+
+    // Base timestamp for synchronizing with video
+    int64_t base_timestamp{0};
+    bool timestamp_initialized{false};
 
     StreamReplicator *streamReplicator = nullptr;
 
     audio_stream(int devId, int aiChn, int aeChn)
         : devId(devId), aiChn(aiChn), aeChn(aeChn), running(false), imp_audio(nullptr),
           msgChannel(std::make_shared<MsgChannel<AudioFrame>>(30)),
-          onDataCallback{nullptr}, hasDataCallback{false} {}
+          onDataCallback{nullptr}, hasDataCallback{false}, base_timestamp(0), timestamp_initialized(false) {}
 };
 
 struct video_stream
@@ -112,11 +114,15 @@ struct video_stream
     std::mutex onDataCallbackLock;     // protects onDataCallback from deallocation
     std::condition_variable should_grab_frames;
     std::binary_semaphore is_activated{0};
+
+    // Base timestamp for synchronizing streams - zero point reference
+    int64_t base_timestamp{0};
+    bool timestamp_initialized{false};
 
     video_stream(int encChn, _stream *stream, const char *name)
         : encChn(encChn), stream(stream), name(name), running(false), idr(false), idr_fix(0), imp_encoder(nullptr), imp_framesource(nullptr),
           msgChannel(std::make_shared<MsgChannel<H264NALUnit>>(MSG_CHANNEL_SIZE)), onDataCallback(nullptr),  run_for_jpeg{false},
-          hasDataCallback{false} {}
+          hasDataCallback{false}, base_timestamp(0), timestamp_initialized(false) {}
 };
 
 extern std::condition_variable global_cv_worker_restart;

src/worker.cpp

@@ -355,10 +355,45 @@ void *Worker::stream_grabber(void *arg)
 #endif
                         H264NALUnit nalu;
 
-                        /* timestamp fix, can be removed if solved
+                        // We need to normalize timestamps to start from zero for players
+                        // But also ensure video PTS values are always valid
+
+                        // Make sure we always have a valid timestamp reference
+                        if (!global_video[encChn]->timestamp_initialized) {
+                            // If stream has no timestamps, use the current system time as reference
+                            if (stream.pack[i].timestamp == 0) {
+                                struct timeval current_time;
+                                gettimeofday(&current_time, NULL);
+                                // Convert to microseconds
+                                int64_t current_micros = (current_time.tv_sec * 1000000LL) + current_time.tv_usec;
+                                // Use a non-zero value but close to zero
+                                global_video[encChn]->base_timestamp = current_micros - 1000; // Offset by 1ms
+                                LOG_INFO("Video stream with zero timestamps, using system time reference");
+                            } else {
+                                global_video[encChn]->base_timestamp = stream.pack[i].timestamp;
+                            }
+                            global_video[encChn]->timestamp_initialized = true;
+                        }
+
+                        // Set the timestamp in microseconds
                         nalu.imp_ts = stream.pack[i].timestamp;
-                        nalu.time = encoder_time;
-                        */
+
+                        // Calculate normalized timestamp by subtracting the base timestamp
+                        // This makes timestamps start from close to zero
+                        int64_t normalized_ts = nalu.imp_ts - global_video[encChn]->base_timestamp;
+
+                        // Ensure we never use negative or zero timestamps
+                        if (normalized_ts <= 0) normalized_ts = 1000; // Use 1ms minimum
+
+                        // Add an increment based on the frame position in this batch
+                        // This ensures timestamps are unique even within a batch of frames
+                        normalized_ts += (i+1) * 100; // Add 100μs per frame in the batch
+
+                        // Convert to timeval structure for the decoder
+                        struct timeval normalized_time;
+                        normalized_time.tv_sec = normalized_ts / 1000000;
+                        normalized_time.tv_usec = normalized_ts % 1000000;
+                        nalu.time = normalized_time;
 
                         // We use start+4 because the encoder inserts 4-byte MPEG
                         //'startcodes' at the beginning of each NAL. Live555 complains
@@ -521,13 +556,39 @@ void *Worker::stream_grabber(void *arg)
 #if defined(AUDIO_SUPPORT)
 static void process_audio_frame(int encChn, IMPAudioFrame &frame)
 {
+    // Initialize the audio timestamp base if not already set
+    if (!global_audio[encChn]->timestamp_initialized) {
+        // Handle case with zero timestamps from encoder
+        if (frame.timeStamp == 0) {
+            struct timeval current_time;
+            gettimeofday(&current_time, NULL);
+            // Convert to microseconds
+            int64_t current_micros = (current_time.tv_sec * 1000000LL) + current_time.tv_usec;
+            // Use a non-zero value but close to zero
+            global_audio[encChn]->base_timestamp = current_micros - 1000; // Offset by 1ms
+            LOG_INFO("Audio stream with zero timestamps, using system time reference");
+        } else {
+            global_audio[encChn]->base_timestamp = frame.timeStamp;
+        }
+        global_audio[encChn]->timestamp_initialized = true;
+    }
+
+    // Get the raw timestamp from the encoder
     int64_t audio_ts = frame.timeStamp;
-    struct timeval encoder_time;
-    encoder_time.tv_sec = audio_ts / 1000000;
-    encoder_time.tv_usec = audio_ts % 1000000;
+
+    // Calculate normalized timestamp by subtracting the base timestamp
+    int64_t normalized_ts = audio_ts - global_audio[encChn]->base_timestamp;
+
+    // Ensure we never use negative or zero timestamps
+    if (normalized_ts <= 0) normalized_ts = 1000; // Use 1ms minimum
+
+    // Convert to timeval structure for the audio frame
+    struct timeval normalized_time;
+    normalized_time.tv_sec = normalized_ts / 1000000;
+    normalized_time.tv_usec = normalized_ts % 1000000;
 
     AudioFrame af;
-    af.time = encoder_time;
+    af.time = normalized_time;
 
     uint8_t *start = (uint8_t *)frame.virAddr;
     uint8_t *end = start + frame.len;