ui/qt/rtp_audio_stream.cpp

   1 /* rtp_audio_frame.cpp
   2  *
   3  * Wireshark - Network traffic analyzer
   4  * By Gerald Combs <gerald@wireshark.org>
   5  * Copyright 1998 Gerald Combs
   6  *
   7  * SPDX-License-Identifier: GPL-2.0-or-later
   8  */
   9
  10 #include "rtp_audio_stream.h"
  11
  12 #ifdef QT_MULTIMEDIA_LIB
  13
  14 #include <speex/speex_resampler.h>
  15
  16 #include <epan/rtp_pt.h>
  17 #include <epan/to_str.h>
  18
  19 #include <epan/dissectors/packet-rtp.h>
  20
  21 #include <ui/rtp_media.h>
  22 #include <ui/rtp_stream.h>
  23 #include <ui/tap-rtp-common.h>
  24
  25 #include <wsutil/nstime.h>
  26
  27 #include <ui/qt/utils/rtp_audio_routing_filter.h>
  28 #include <ui/qt/utils/rtp_audio_file.h>
  29
  30 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
  31 #include <QAudioDevice>
  32 #include <QAudioSink>
  33 #endif
  34 #include <QAudioFormat>
  35 #include <QAudioOutput>
  36 #include <QVariant>
  37 #include <QTimer>
  38
  39 // To do:
  40 // - Only allow one rtpstream_info_t per RtpAudioStream?
  41
  42 static const spx_int16_t visual_sample_rate_ = 1000;
  43
  44 RtpAudioStream::RtpAudioStream(QObject *parent, rtpstream_id_t *id, bool stereo_required) :
  45     QObject(parent)
  46     , first_packet_(true)
  47     , decoders_hash_(rtp_decoder_hash_table_new())
  48     , global_start_rel_time_(0.0)
  49     , start_abs_offset_(0.0)
  50     , start_rel_time_(0.0)
  51     , stop_rel_time_(0.0)
  52     , stereo_required_(stereo_required)
  53     , first_sample_rate_(0)
  54     , audio_out_rate_(0)
  55     , audio_requested_out_rate_(0)
  56     , max_sample_val_(1)
  57     , max_sample_val_used_(1)
  58     , color_(0)
  59     , jitter_buffer_size_(50)
  60     , timing_mode_(RtpAudioStream::JitterBuffer)
  61     , start_play_time_(0)
  62     , audio_output_(NULL)
  63 {
  64     rtpstream_id_copy(id, &id_);
  65     memset(&rtpstream_, 0, sizeof(rtpstream_));
  66     rtpstream_id_copy(&id_, &rtpstream_.id);
  67
  68     // Rates will be set later, we just init visual resampler
  69     visual_resampler_ = speex_resampler_init(1, visual_sample_rate_,
  70                                                 visual_sample_rate_, SPEEX_RESAMPLER_QUALITY_MIN, NULL);
  71
  72     try {
  73         // RtpAudioFile is ready for writing Frames
  74         audio_file_ = new RtpAudioFile(prefs.gui_rtp_player_use_disk1, prefs.gui_rtp_player_use_disk2);
  75     } catch (...) {
  76         speex_resampler_destroy(visual_resampler_);
  77         rtpstream_info_free_data(&rtpstream_);
  78         rtpstream_id_free(&id_);
  79         throw -1;
  80     }
  81
  82     // RTP_STREAM_DEBUG("Writing to %s", tempname.toUtf8().constData());
  83 }
  84
  85 RtpAudioStream::~RtpAudioStream()
  86 {
  87     for (int i = 0; i < rtp_packets_.size(); i++) {
  88         rtp_packet_t *rtp_packet = rtp_packets_[i];
  89         g_free(rtp_packet->info);
  90         g_free(rtp_packet->payload_data);
  91         g_free(rtp_packet);
  92     }
  93     g_hash_table_destroy(decoders_hash_);
  94     speex_resampler_destroy(visual_resampler_);
  95     rtpstream_info_free_data(&rtpstream_);
  96     rtpstream_id_free(&id_);
  97     if (audio_file_) delete audio_file_;
  98     // temp_file_ is released by audio_output_
  99     if (audio_output_) delete audio_output_;
 100 }
 101
 102 bool RtpAudioStream::isMatch(const rtpstream_id_t *id) const
 103 {
 104     if (id
 105         && rtpstream_id_equal(&id_, id, RTPSTREAM_ID_EQUAL_SSRC))
 106         return true;
 107     return false;
 108 }
 109
 110 bool RtpAudioStream::isMatch(const _packet_info *pinfo, const _rtp_info *rtp_info) const
 111 {
 112     if (pinfo && rtp_info
 113         && rtpstream_id_equal_pinfo_rtp_info(&id_, pinfo, rtp_info))
 114         return true;
 115     return false;
 116 }
 117
 118 void RtpAudioStream::addRtpPacket(const struct _packet_info *pinfo, const struct _rtp_info *rtp_info)
 119 {
 120     if (!rtp_info) return;
 121
 122     if (first_packet_) {
 123         rtpstream_info_analyse_init(&rtpstream_, pinfo, rtp_info);
 124         first_packet_ = false;
 125     }
 126     rtpstream_info_analyse_process(&rtpstream_, pinfo, rtp_info);
 127
 128     rtp_packet_t *rtp_packet = g_new0(rtp_packet_t, 1);
 129     rtp_packet->info = (struct _rtp_info *) g_memdup2(rtp_info, sizeof(struct _rtp_info));
 130     if (rtp_info->info_all_data_present && (rtp_info->info_payload_len != 0)) {
 131         rtp_packet->payload_data = (uint8_t *) g_memdup2(&(rtp_info->info_data[rtp_info->info_payload_offset]),
 132           rtp_info->info_payload_len);
 133     }
 134
 135     if (rtp_packets_.size() < 1) { // First packet
 136         start_abs_offset_ = nstime_to_sec(&pinfo->abs_ts) - start_rel_time_;
 137         start_rel_time_ = stop_rel_time_ = nstime_to_sec(&pinfo->rel_ts);
 138     }
 139     rtp_packet->frame_num = pinfo->num;
 140     rtp_packet->arrive_offset = nstime_to_sec(&pinfo->rel_ts) - start_rel_time_;
 141
 142     rtp_packets_ << rtp_packet;
 143 }
 144
 145 void RtpAudioStream::clearPackets()
 146 {
 147     for (int i = 0; i < rtp_packets_.size(); i++) {
 148         rtp_packet_t *rtp_packet = rtp_packets_[i];
 149         g_free(rtp_packet->info);
 150         g_free(rtp_packet->payload_data);
 151         g_free(rtp_packet);
 152     }
 153     rtp_packets_.clear();
 154     rtpstream_info_free_data(&rtpstream_);
 155     memset(&rtpstream_, 0, sizeof(rtpstream_));
 156     rtpstream_id_copy(&id_, &rtpstream_.id);
 157     first_packet_ = true;
 158 }
 159
 160 void RtpAudioStream::reset(double global_start_time)
 161 {
 162     global_start_rel_time_ = global_start_time;
 163     stop_rel_time_ = start_rel_time_;
 164     audio_out_rate_ = 0;
 165     max_sample_val_ = 1;
 166     packet_timestamps_.clear();
 167     visual_samples_.clear();
 168     out_of_seq_timestamps_.clear();
 169     jitter_drop_timestamps_.clear();
 170 }
 171
 172 AudioRouting RtpAudioStream::getAudioRouting()
 173 {
 174     return audio_routing_;
 175 }
 176
 177 void RtpAudioStream::setAudioRouting(AudioRouting audio_routing)
 178 {
 179     audio_routing_ = audio_routing;
 180 }
 181
 182 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
 183 void RtpAudioStream::decode(QAudioDevice out_device)
 184 #else
 185 void RtpAudioStream::decode(QAudioDeviceInfo out_device)
 186 #endif
 187 {
 188     if (rtp_packets_.size() < 1) return;
 189
 190     audio_file_->setFrameWriteStage();
 191     decodeAudio(out_device);
 192
 193     // Skip silence at begin of the stream
 194     audio_file_->setFrameReadStage(prepend_samples_);
 195
 196     speex_resampler_reset_mem(visual_resampler_);
 197     decodeVisual();
 198     audio_file_->setDataReadStage();
 199 }
 200
 201 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
 202 quint32 RtpAudioStream::calculateAudioOutRate(QAudioDevice out_device, unsigned int sample_rate, unsigned int requested_out_rate)
 203 #else
 204 quint32 RtpAudioStream::calculateAudioOutRate(QAudioDeviceInfo out_device, unsigned int sample_rate, unsigned int requested_out_rate)
 205 #endif
 206 {
 207     quint32 out_rate;
 208
 209     // Use the first non-zero rate we find. Adjust it to match
 210     // our audio hardware.
 211     QAudioFormat format;
 212     format.setSampleRate(sample_rate);
 213 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
 214     // Must match rtp_media.h.
 215     format.setSampleFormat(QAudioFormat::Int16);
 216 #else
 217     format.setSampleSize(SAMPLE_BYTES * 8); // bits
 218     format.setSampleType(QAudioFormat::SignedInt);
 219 #endif
 220     if (stereo_required_) {
 221         format.setChannelCount(2);
 222     } else {
 223         format.setChannelCount(1);
 224     }
 225 #if (QT_VERSION < QT_VERSION_CHECK(6, 0, 0))
 226     format.setCodec("audio/pcm");
 227 #endif
 228
 229     if (!out_device.isNull() &&
 230         !out_device.isFormatSupported(format) &&
 231         (requested_out_rate == 0)
 232        ) {
 233 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
 234         out_rate = out_device.preferredFormat().sampleRate();
 235 #else
 236         out_rate = out_device.nearestFormat(format).sampleRate();
 237 #endif
 238     } else {
 239         if ((requested_out_rate != 0) &&
 240             (requested_out_rate != sample_rate)
 241            ) {
 242             out_rate = requested_out_rate;
 243         } else {
 244             out_rate = sample_rate;
 245         }
 246     }
 247
 248     RTP_STREAM_DEBUG("Audio sample rate is %u", out_rate);
 249
 250     return out_rate;
 251 }
 252
 253 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
 254 void RtpAudioStream::decodeAudio(QAudioDevice out_device)
 255 #else
 256 void RtpAudioStream::decodeAudio(QAudioDeviceInfo out_device)
 257 #endif
 258 {
 259     // XXX This is more messy than it should be.
 260
 261     int32_t resample_buff_bytes = 0x1000;
 262     SAMPLE *resample_buff = (SAMPLE *) g_malloc(resample_buff_bytes);
 263     char *write_buff = NULL;
 264     qint64 write_bytes = 0;
 265     unsigned int channels = 0;
 266     unsigned int sample_rate = 0;
 267     uint32_t last_sequence = 0;
 268     uint32_t last_sequence_w = 0;  // Last sequence number we wrote data
 269
 270     double rtp_time_prev = 0.0;
 271     double arrive_time_prev = 0.0;
 272     double pack_period = 0.0;
 273     double start_time = 0.0;
 274     double start_rtp_time = 0.0;
 275     uint64_t start_timestamp = 0;
 276
 277     size_t decoded_bytes_prev = 0;
 278     unsigned int audio_resampler_input_rate = 0;
 279     struct SpeexResamplerState_ *audio_resampler = NULL;
 280
 281     for (int cur_packet = 0; cur_packet < rtp_packets_.size(); cur_packet++) {
 282         SAMPLE *decode_buff = NULL;
 283         // TODO: Update a progress bar here.
 284         rtp_packet_t *rtp_packet = rtp_packets_[cur_packet];
 285
 286         stop_rel_time_ = start_rel_time_ + rtp_packet->arrive_offset;
 287
 288         QString payload_name;
 289         if (rtp_packet->info->info_payload_type_str) {
 290             payload_name = rtp_packet->info->info_payload_type_str;
 291         } else {
 292             payload_name = try_val_to_str_ext(rtp_packet->info->info_payload_type, &rtp_payload_type_short_vals_ext);
 293         }
 294         if (!payload_name.isEmpty()) {
 295             payload_names_ << payload_name;
 296         }
 297
 298         if (cur_packet < 1) { // First packet
 299             start_timestamp = rtp_packet->info->info_extended_timestamp;
 300             start_rtp_time = 0;
 301             rtp_time_prev = 0;
 302             last_sequence = rtp_packet->info->info_extended_seq_num - 1;
 303         }
 304
 305         size_t decoded_bytes = decode_rtp_packet(rtp_packet, &decode_buff, decoders_hash_, &channels, &sample_rate);
 306         // XXX: We don't actually *do* anything with channels, and just treat
 307         // everything as if it were mono
 308
 309         unsigned rtp_clock_rate = sample_rate;
 310         if (rtp_packet->info->info_payload_type == PT_G722) {
 311             // G.722 sample rate is 16kHz, but RTP clock rate is 8kHz
 312             // for historic reasons.
 313             rtp_clock_rate = 8000;
 314         } else if (rtp_packet->info->info_is_iuup) {
 315             /* IuUP uses a fixed RTP clock rate of 16kHz, regardless of the payload's codec sample rate.
 316              * See: 3GPP TS 25.414 section 5.1.3.3.1.8
 317              *      "A clock frequency of 16000 Hz shall be used."
 318              * See: https://www.iana.org/assignments/media-types/audio/vnd.3gpp.iufp
 319              *      "A fixed RTP clock rate of 16000 is used.""
 320              */
 321             rtp_clock_rate = 16000;
 322         }
 323
 324         // Length 2 for PT_PCM mean silence packet probably, ignore
 325         if (decoded_bytes == 0 || sample_rate == 0 ||
 326             ((rtp_packet->info->info_payload_type == PT_PCMU ||
 327               rtp_packet->info->info_payload_type == PT_PCMA
 328              ) && (decoded_bytes == 2)
 329             )
 330            ) {
 331             // We didn't decode anything. Clean up and prep for
 332             // the next packet.
 333             last_sequence = rtp_packet->info->info_extended_seq_num;
 334             g_free(decode_buff);
 335             continue;
 336         }
 337
 338         if (audio_out_rate_ == 0) {
 339             first_sample_rate_ = sample_rate;
 340
 341             // We calculate audio_out_rate just for first sample_rate.
 342             // All later are just resampled to it.
 343             // Side effect: it creates and initiates resampler if needed
 344             audio_out_rate_ = calculateAudioOutRate(out_device, sample_rate, audio_requested_out_rate_);
 345
 346             // Calculate count of prepend samples for the stream
 347             // The earliest stream starts at 0.
 348             // Note: Order of operations and separation to two formulas is
 349             // important.
 350             // When joined, calculated incorrectly - probably caused by
 351             // conversions between int/double
 352             prepend_samples_ = (start_rel_time_ - global_start_rel_time_) * sample_rate;
 353             prepend_samples_ = prepend_samples_ * audio_out_rate_ / sample_rate;
 354
 355             // Prepend silence to match our sibling streams.
 356             if ((prepend_samples_ > 0) && (audio_out_rate_ != 0)) {
 357                 audio_file_->frameWriteSilence(rtp_packet->frame_num, prepend_samples_);
 358             }
 359         }
 360
 361         if (rtp_packet->info->info_extended_seq_num != last_sequence+1) {
 362             out_of_seq_timestamps_.append(stop_rel_time_);
 363         }
 364         last_sequence = rtp_packet->info->info_extended_seq_num;
 365
 366         double rtp_time = (double)(rtp_packet->info->info_extended_timestamp-start_timestamp)/rtp_clock_rate - start_rtp_time;
 367         double arrive_time;
 368         if (timing_mode_ == RtpTimestamp) {
 369             arrive_time = rtp_time;
 370         } else {
 371             arrive_time = rtp_packet->arrive_offset - start_time;
 372         }
 373
 374         double diff = qAbs(arrive_time - rtp_time);
 375         if (diff*1000 > jitter_buffer_size_ && timing_mode_ != Uninterrupted) {
 376             // rtp_player.c:628
 377
 378             jitter_drop_timestamps_.append(stop_rel_time_);
 379             RTP_STREAM_DEBUG("Packet drop by jitter buffer exceeded %f > %d", diff*1000, jitter_buffer_size_);
 380
 381             /* if there was a silence period (more than two packetization
 382              * period) resync the source */
 383             if ((rtp_time - rtp_time_prev) > pack_period*2) {
 384                 qint64 silence_samples;
 385                 RTP_STREAM_DEBUG("Resync...");
 386
 387                 silence_samples = (qint64)((arrive_time - arrive_time_prev)*sample_rate - decoded_bytes_prev / SAMPLE_BYTES);
 388                 silence_samples = silence_samples * audio_out_rate_ / sample_rate;
 389                 silence_timestamps_.append(stop_rel_time_);
 390                 // Timestamp shift can make silence calculation negative
 391                 if ((silence_samples > 0) && (audio_out_rate_ != 0)) {
 392                     audio_file_->frameWriteSilence(rtp_packet->frame_num, silence_samples);
 393                 }
 394
 395                 decoded_bytes_prev = 0;
 396                 start_timestamp = rtp_packet->info->info_extended_timestamp;
 397                 start_rtp_time = 0;
 398                 start_time = rtp_packet->arrive_offset;
 399                 rtp_time_prev = 0;
 400             }
 401
 402         } else {
 403             // rtp_player.c:664
 404             /* Add silence if it is necessary */
 405             qint64 silence_samples;
 406
 407             if (timing_mode_ == Uninterrupted) {
 408                 silence_samples = 0;
 409             } else {
 410                 silence_samples = (qint64)((rtp_time - rtp_time_prev)*sample_rate - decoded_bytes_prev / SAMPLE_BYTES);
 411                 silence_samples = silence_samples * audio_out_rate_ / sample_rate;
 412             }
 413
 414             if (silence_samples != 0) {
 415                 wrong_timestamp_timestamps_.append(stop_rel_time_);
 416             }
 417
 418             if (silence_samples > 0) {
 419                 silence_timestamps_.append(stop_rel_time_);
 420                 if ((silence_samples > 0) && (audio_out_rate_ != 0)) {
 421                     audio_file_->frameWriteSilence(rtp_packet->frame_num, silence_samples);
 422                 }
 423             }
 424
 425             // XXX rtp_player.c:696 adds audio here.
 426             rtp_time_prev = rtp_time;
 427             pack_period = (double) decoded_bytes / SAMPLE_BYTES / sample_rate;
 428             decoded_bytes_prev = decoded_bytes;
 429             arrive_time_prev = arrive_time;
 430         }
 431
 432         // Prepare samples to write
 433         write_buff = (char *) decode_buff;
 434         write_bytes = decoded_bytes;
 435
 436         if (audio_out_rate_ != sample_rate) {
 437             // Resample the audio to match output rate.
 438             // Buffer is in SAMPLEs
 439             spx_uint32_t in_len = (spx_uint32_t) (write_bytes / SAMPLE_BYTES);
 440             // Output is audio_out_rate_/sample_rate bigger than input
 441             spx_uint32_t out_len = (spx_uint32_t) ((uint64_t)in_len * audio_out_rate_ / sample_rate);
 442             resample_buff = resizeBufferIfNeeded(resample_buff, &resample_buff_bytes, out_len * SAMPLE_BYTES);
 443
 444             if (audio_resampler &&
 445                 sample_rate != audio_resampler_input_rate
 446                ) {
 447               // Clear old resampler because input rate changed
 448               speex_resampler_destroy(audio_resampler);
 449               audio_resampler_input_rate = 0;
 450               audio_resampler = NULL;
 451             }
 452             if (!audio_resampler) {
 453                 audio_resampler_input_rate = sample_rate;
 454                 audio_resampler = speex_resampler_init(1, sample_rate, audio_out_rate_, 10, NULL);
 455                 RTP_STREAM_DEBUG("Started resampling from %u to (out) %u Hz.", sample_rate, audio_out_rate_);
 456             }
 457             speex_resampler_process_int(audio_resampler, 0, decode_buff, &in_len, resample_buff, &out_len);
 458
 459             write_buff = (char *) resample_buff;
 460             write_bytes = out_len * SAMPLE_BYTES;
 461         }
 462
 463         // We should write only newer data to avoid duplicates in replay
 464         if (last_sequence_w < last_sequence) {
 465             // Write the decoded, possibly-resampled audio to our temp file.
 466             audio_file_->frameWriteSamples(rtp_packet->frame_num, write_buff, write_bytes);
 467             last_sequence_w = last_sequence;
 468         }
 469
 470         g_free(decode_buff);
 471     }
 472     g_free(resample_buff);
 473
 474     if (audio_resampler) speex_resampler_destroy(audio_resampler);
 475 }
 476
 477 // We preallocate buffer, 320 samples is enough for most scenarios
 478 #define VISUAL_BUFF_LEN (320)
 479 #define VISUAL_BUFF_BYTES (SAMPLE_BYTES * VISUAL_BUFF_LEN)
 480 void RtpAudioStream::decodeVisual()
 481 {
 482     spx_uint32_t read_len = 0;
 483     int32_t read_buff_bytes = VISUAL_BUFF_BYTES;
 484     SAMPLE *read_buff = (SAMPLE *) g_malloc(read_buff_bytes);
 485     int32_t resample_buff_bytes = VISUAL_BUFF_BYTES;
 486     SAMPLE *resample_buff = (SAMPLE *) g_malloc(resample_buff_bytes);
 487     unsigned int sample_no = 0;
 488     spx_uint32_t out_len;
 489     uint32_t frame_num;
 490     rtp_frame_type type;
 491
 492     speex_resampler_set_rate(visual_resampler_, audio_out_rate_, visual_sample_rate_);
 493
 494     // Loop over every frame record
 495     // readFrameSamples() maintains size of buffer for us
 496     while (audio_file_->readFrameSamples(&read_buff_bytes, &read_buff, &read_len, &frame_num, &type)) {
 497         out_len = (spx_uint32_t)(((uint64_t)read_len * visual_sample_rate_ ) / audio_out_rate_);
 498
 499         if (type == RTP_FRAME_AUDIO) {
 500             // We resample only audio samples
 501             resample_buff = resizeBufferIfNeeded(resample_buff, &resample_buff_bytes, out_len * SAMPLE_BYTES);
 502
 503             // Resample
 504             speex_resampler_process_int(visual_resampler_, 0, read_buff, &read_len, resample_buff, &out_len);
 505
 506             // Create timestamp and visual sample
 507             for (unsigned i = 0; i < out_len; i++) {
 508                 double time = start_rel_time_ + (double) sample_no / visual_sample_rate_;
 509                 packet_timestamps_[time] = frame_num;
 510                 if (qAbs(resample_buff[i]) > max_sample_val_) max_sample_val_ = qAbs(resample_buff[i]);
 511                 visual_samples_.append(resample_buff[i]);
 512                 sample_no++;
 513             }
 514         } else {
 515             // Insert end of line mark
 516             double time = start_rel_time_ + (double) sample_no / visual_sample_rate_;
 517             packet_timestamps_[time] = frame_num;
 518             visual_samples_.append(SAMPLE_NaN);
 519             sample_no += out_len;
 520         }
 521     }
 522
 523     max_sample_val_used_ = max_sample_val_;
 524     g_free(resample_buff);
 525     g_free(read_buff);
 526 }
 527
 528 const QStringList RtpAudioStream::payloadNames() const
 529 {
 530     QStringList payload_names = payload_names_.values();
 531     payload_names.sort();
 532     return payload_names;
 533 }
 534
 535 const QVector<double> RtpAudioStream::visualTimestamps(bool relative)
 536 {
 537     QVector<double> ts_keys = packet_timestamps_.keys().toVector();
 538     if (relative) return ts_keys;
 539
 540     QVector<double> adj_timestamps;
 541     for (int i = 0; i < ts_keys.size(); i++) {
 542         adj_timestamps.append(ts_keys[i] + start_abs_offset_ - start_rel_time_);
 543     }
 544     return adj_timestamps;
 545 }
 546
 547 // Scale the height of the waveform to global scale (max_sample_val_used_)
 548 // and adjust its Y offset so that they overlap slightly (stack_offset_).
 549 static const double stack_offset_ = INT16_MAX / 3;
 550 const QVector<double> RtpAudioStream::visualSamples(int y_offset)
 551 {
 552     QVector<double> adj_samples;
 553     double scaled_offset = y_offset * stack_offset_;
 554     for (int i = 0; i < visual_samples_.size(); i++) {
 555         if (SAMPLE_NaN != visual_samples_[i]) {
 556             adj_samples.append(((double)visual_samples_[i] * INT16_MAX / max_sample_val_used_) + scaled_offset);
 557         } else {
 558             // Convert to break in graph line
 559             adj_samples.append(qQNaN());
 560         }
 561     }
 562     return adj_samples;
 563 }
 564
 565 const QVector<double> RtpAudioStream::outOfSequenceTimestamps(bool relative)
 566 {
 567     if (relative) return out_of_seq_timestamps_;
 568
 569     QVector<double> adj_timestamps;
 570     for (int i = 0; i < out_of_seq_timestamps_.size(); i++) {
 571         adj_timestamps.append(out_of_seq_timestamps_[i] + start_abs_offset_ - start_rel_time_);
 572     }
 573     return adj_timestamps;
 574 }
 575
 576 const QVector<double> RtpAudioStream::outOfSequenceSamples(int y_offset)
 577 {
 578     QVector<double> adj_samples;
 579     double scaled_offset = y_offset * stack_offset_;  // XXX Should be different for seq, jitter, wrong & silence
 580     for (int i = 0; i < out_of_seq_timestamps_.size(); i++) {
 581         adj_samples.append(scaled_offset);
 582     }
 583     return adj_samples;
 584 }
 585
 586 const QVector<double> RtpAudioStream::jitterDroppedTimestamps(bool relative)
 587 {
 588     if (relative) return jitter_drop_timestamps_;
 589
 590     QVector<double> adj_timestamps;
 591     for (int i = 0; i < jitter_drop_timestamps_.size(); i++) {
 592         adj_timestamps.append(jitter_drop_timestamps_[i] + start_abs_offset_ - start_rel_time_);
 593     }
 594     return adj_timestamps;
 595 }
 596
 597 const QVector<double> RtpAudioStream::jitterDroppedSamples(int y_offset)
 598 {
 599     QVector<double> adj_samples;
 600     double scaled_offset = y_offset * stack_offset_; // XXX Should be different for seq, jitter, wrong & silence
 601     for (int i = 0; i < jitter_drop_timestamps_.size(); i++) {
 602         adj_samples.append(scaled_offset);
 603     }
 604     return adj_samples;
 605 }
 606
 607 const QVector<double> RtpAudioStream::wrongTimestampTimestamps(bool relative)
 608 {
 609     if (relative) return wrong_timestamp_timestamps_;
 610
 611     QVector<double> adj_timestamps;
 612     for (int i = 0; i < wrong_timestamp_timestamps_.size(); i++) {
 613         adj_timestamps.append(wrong_timestamp_timestamps_[i] + start_abs_offset_ - start_rel_time_);
 614     }
 615     return adj_timestamps;
 616 }
 617
 618 const QVector<double> RtpAudioStream::wrongTimestampSamples(int y_offset)
 619 {
 620     QVector<double> adj_samples;
 621     double scaled_offset = y_offset * stack_offset_; // XXX Should be different for seq, jitter, wrong & silence
 622     for (int i = 0; i < wrong_timestamp_timestamps_.size(); i++) {
 623         adj_samples.append(scaled_offset);
 624     }
 625     return adj_samples;
 626 }
 627
 628 const QVector<double> RtpAudioStream::insertedSilenceTimestamps(bool relative)
 629 {
 630     if (relative) return silence_timestamps_;
 631
 632     QVector<double> adj_timestamps;
 633     for (int i = 0; i < silence_timestamps_.size(); i++) {
 634         adj_timestamps.append(silence_timestamps_[i] + start_abs_offset_ - start_rel_time_);
 635     }
 636     return adj_timestamps;
 637 }
 638
 639 const QVector<double> RtpAudioStream::insertedSilenceSamples(int y_offset)
 640 {
 641     QVector<double> adj_samples;
 642     double scaled_offset = y_offset * stack_offset_;  // XXX Should be different for seq, jitter, wrong & silence
 643     for (int i = 0; i < silence_timestamps_.size(); i++) {
 644         adj_samples.append(scaled_offset);
 645     }
 646     return adj_samples;
 647 }
 648
 649 quint32 RtpAudioStream::nearestPacket(double timestamp, bool is_relative)
 650 {
 651     if (packet_timestamps_.size() < 1) return 0;
 652
 653     if (!is_relative) timestamp -= start_abs_offset_;
 654     QMap<double, quint32>::iterator it = packet_timestamps_.lowerBound(timestamp);
 655     if (it == packet_timestamps_.end()) return 0;
 656     return it.value();
 657 }
 658
 659 QAudio::State RtpAudioStream::outputState() const
 660 {
 661     if (!audio_output_) return QAudio::IdleState;
 662     return audio_output_->state();
 663 }
 664
 665 const QString RtpAudioStream::formatDescription(const QAudioFormat &format)
 666 {
 667     QString fmt_descr = QStringLiteral("%1 Hz, ").arg(format.sampleRate());
 668 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
 669     switch (format.sampleFormat()) {
 670     case QAudioFormat::UInt8:
 671         fmt_descr += "UInt8";
 672         break;
 673     case QAudioFormat::Int16:
 674         fmt_descr += "Int16";
 675         break;
 676     case QAudioFormat::Int32:
 677         fmt_descr += "Int32";
 678         break;
 679     case QAudioFormat::Float:
 680         fmt_descr += "Float";
 681         break;
 682     default:
 683         fmt_descr += "Unknown";
 684         break;
 685     }
 686 #else
 687     switch (format.sampleType()) {
 688     case QAudioFormat::SignedInt:
 689         fmt_descr += "Int";
 690         fmt_descr += QString::number(format.sampleSize());
 691         fmt_descr += format.byteOrder() == QAudioFormat::BigEndian ? "BE" : "LE";
 692         break;
 693     case QAudioFormat::UnSignedInt:
 694         fmt_descr += "UInt";
 695         fmt_descr += QString::number(format.sampleSize());
 696         fmt_descr += format.byteOrder() == QAudioFormat::BigEndian ? "BE" : "LE";
 697         break;
 698     case QAudioFormat::Float:
 699         fmt_descr += "Float";
 700         break;
 701     default:
 702         fmt_descr += "Unknown";
 703         break;
 704     }
 705 #endif
 706
 707     return fmt_descr;
 708 }
 709
 710 QString RtpAudioStream::getIDAsQString()
 711 {
 712     char *src_addr_str = address_to_display(NULL, &id_.src_addr);
 713     char *dst_addr_str = address_to_display(NULL, &id_.dst_addr);
 714     QString str = QStringLiteral("%1:%2 - %3:%4 %5")
 715         .arg(src_addr_str)
 716         .arg(id_.src_port)
 717         .arg(dst_addr_str)
 718         .arg(id_.dst_port)
 719         .arg(QStringLiteral("0x%1").arg(id_.ssrc, 0, 16));
 720     wmem_free(NULL, src_addr_str);
 721     wmem_free(NULL, dst_addr_str);
 722
 723     return str;
 724 }
 725
 726 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
 727 bool RtpAudioStream::prepareForPlay(QAudioDevice out_device)
 728 #else
 729 bool RtpAudioStream::prepareForPlay(QAudioDeviceInfo out_device)
 730 #endif
 731 {
 732     qint64 start_pos;
 733     qint64 size;
 734
 735     if (audio_routing_.isMuted())
 736         return false;
 737
 738     if (audio_output_)
 739         return false;
 740
 741     if (audio_out_rate_ == 0) {
 742         /* It is observed, but is not an error
 743         QString error = tr("RTP stream (%1) is empty or codec is unsupported.")
 744             .arg(getIDAsQString());
 745
 746         emit playbackError(error);
 747         */
 748         return false;
 749     }
 750
 751     QAudioFormat format;
 752     format.setSampleRate(audio_out_rate_);
 753 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
 754     // Must match rtp_media.h.
 755     format.setSampleFormat(QAudioFormat::Int16);
 756 #else
 757     format.setSampleSize(SAMPLE_BYTES * 8); // bits
 758     format.setSampleType(QAudioFormat::SignedInt);
 759 #endif
 760     if (stereo_required_) {
 761         format.setChannelCount(2);
 762     } else {
 763         format.setChannelCount(1);
 764     }
 765 #if (QT_VERSION < QT_VERSION_CHECK(6, 0, 0))
 766     format.setCodec("audio/pcm");
 767 #endif
 768
 769     // RTP_STREAM_DEBUG("playing %s %d samples @ %u Hz",
 770     //                 sample_file_->fileName().toUtf8().constData(),
 771     //                 (int) sample_file_->size(), audio_out_rate_);
 772
 773     if (!out_device.isFormatSupported(format)) {
 774 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
 775         QString playback_error = tr("%1 does not support PCM at %2. Preferred format is %3")
 776                 .arg(out_device.description(), formatDescription(format), formatDescription(out_device.preferredFormat()));
 777 #else
 778         QString playback_error = tr("%1 does not support PCM at %2. Preferred format is %3")
 779                 .arg(out_device.deviceName())
 780                 .arg(formatDescription(format))
 781                 .arg(formatDescription(out_device.nearestFormat(format)));
 782 #endif
 783         emit playbackError(playback_error);
 784     }
 785
 786     start_pos = (qint64)(start_play_time_ * SAMPLE_BYTES * audio_out_rate_);
 787     // Round to SAMPLE_BYTES boundary
 788     start_pos = (start_pos / SAMPLE_BYTES) * SAMPLE_BYTES;
 789     size = audio_file_->sampleFileSize();
 790     if (stereo_required_) {
 791         // There is 2x more samples for stereo
 792         start_pos *= 2;
 793         size *= 2;
 794     }
 795     if (start_pos < size) {
 796         audio_file_->setDataReadStage();
 797         temp_file_ = new AudioRoutingFilter(audio_file_, stereo_required_, audio_routing_);
 798         temp_file_->seek(start_pos);
 799         if (audio_output_) delete audio_output_;
 800 #if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0))
 801         audio_output_ = new QAudioSink(out_device, format, this);
 802         connect(audio_output_, &QAudioSink::stateChanged, this, &RtpAudioStream::outputStateChanged);
 803 #else
 804         audio_output_ = new QAudioOutput(out_device, format, this);
 805         connect(audio_output_, &QAudioOutput::stateChanged, this, &RtpAudioStream::outputStateChanged);
 806 #endif
 807         return true;
 808     } else {
 809         // Report stopped audio if start position is later than stream ends
 810         outputStateChanged(QAudio::StoppedState);
 811         return false;
 812     }
 813
 814     return false;
 815 }
 816
 817 void RtpAudioStream::startPlaying()
 818 {
 819    // On Win32/Qt 6.x start() returns, but state() is QAudio::StoppedState even
 820    // everything is OK
 821    audio_output_->start(temp_file_);
 822 #if (QT_VERSION < QT_VERSION_CHECK(6, 0, 0))
 823    // Bug is related to Qt 4.x and probably for 5.x, but not for 6.x
 824    // QTBUG-6548 StoppedState is not always emitted on error, force a cleanup
 825    // in case playback fails immediately.
 826    if (audio_output_ && audio_output_->state() == QAudio::StoppedState) {
 827        outputStateChanged(QAudio::StoppedState);
 828    }
 829 #endif
 830 }
 831
 832 void RtpAudioStream::pausePlaying()
 833 {
 834     if (audio_routing_.isMuted())
 835         return;
 836
 837     if (audio_output_) {
 838         if (QAudio::ActiveState == audio_output_->state()) {
 839             audio_output_->suspend();
 840         } else if (QAudio::SuspendedState == audio_output_->state()) {
 841             audio_output_->resume();
 842         }
 843     }
 844 }
 845
 846 void RtpAudioStream::stopPlaying()
 847 {
 848     if (audio_routing_.isMuted())
 849         return;
 850
 851     if (audio_output_) {
 852         if (audio_output_->state() == QAudio::StoppedState) {
 853             // Looks like "delayed" QTBUG-6548
 854             // It may happen that stream is stopped, but no signal emitted
 855             // Probably triggered by some issue in sound system which is not
 856             // handled by Qt correctly
 857             outputStateChanged(QAudio::StoppedState);
 858         } else {
 859             audio_output_->stop();
 860         }
 861     }
 862 }
 863
 864 void RtpAudioStream::seekPlaying(qint64 samples _U_)
 865 {
 866     if (audio_routing_.isMuted())
 867         return;
 868
 869     if (audio_output_) {
 870         audio_output_->suspend();
 871         audio_file_->seekSample(samples);
 872         audio_output_->resume();
 873     }
 874 }
 875
 876 void RtpAudioStream::outputStateChanged(QAudio::State new_state)
 877 {
 878     if (!audio_output_) return;
 879
 880     // On some platforms including macOS and Windows, the stateChanged signal
 881     // is emitted while a QMutexLocker is active. As a result we shouldn't
 882     // delete audio_output_ here.
 883     switch (new_state) {
 884     case QAudio::StoppedState:
 885         {
 886             // RTP_STREAM_DEBUG("stopped %f", audio_output_->processedUSecs() / 100000.0);
 887             // Detach from parent (RtpAudioStream) to prevent deleteLater
 888             // from being run during destruction of this class.
 889             QAudio::Error error = audio_output_->error();
 890
 891             audio_output_->setParent(0);
 892             audio_output_->disconnect();
 893             audio_output_->deleteLater();
 894             audio_output_ = NULL;
 895             emit finishedPlaying(this, error);
 896             break;
 897         }
 898     case QAudio::IdleState:
 899         // Workaround for Qt behaving on some platforms with some soundcards:
 900         // When ->stop() is called from outputStateChanged(),
 901         // internalQMutexLock is locked and application hangs.
 902         // We can stop the stream later.
 903         QTimer::singleShot(0, this, SLOT(delayedStopStream()));
 904
 905         break;
 906     default:
 907         break;
 908     }
 909 }
 910
 911 void RtpAudioStream::delayedStopStream()
 912 {
 913     audio_output_->stop();
 914 }
 915
 916 SAMPLE *RtpAudioStream::resizeBufferIfNeeded(SAMPLE *buff, int32_t *buff_bytes, qint64 requested_size)
 917 {
 918     if (requested_size > *buff_bytes) {
 919         while ((requested_size > *buff_bytes))
 920             *buff_bytes *= 2;
 921         buff = (SAMPLE *) g_realloc(buff, *buff_bytes);
 922     }
 923
 924     return buff;
 925 }
 926
 927 void RtpAudioStream::seekSample(qint64 samples)
 928 {
 929     audio_file_->seekSample(samples);
 930 }
 931
 932 qint64 RtpAudioStream::readSample(SAMPLE *sample)
 933 {
 934     return audio_file_->readSample(sample);
 935 }
 936
 937 bool RtpAudioStream::savePayload(QIODevice *file)
 938 {
 939     for (int cur_packet = 0; cur_packet < rtp_packets_.size(); cur_packet++) {
 940         // TODO: Update a progress bar here.
 941         rtp_packet_t *rtp_packet = rtp_packets_[cur_packet];
 942
 943         if ((rtp_packet->info->info_payload_type != PT_CN) &&
 944             (rtp_packet->info->info_payload_type != PT_CN_OLD)) {
 945             // All other payloads
 946             int64_t nchars;
 947
 948             if (rtp_packet->payload_data && (rtp_packet->info->info_payload_len > 0)) {
 949                 nchars = file->write((char *)rtp_packet->payload_data, rtp_packet->info->info_payload_len);
 950                 if (nchars != rtp_packet->info->info_payload_len) {
 951                     return false;
 952                 }
 953             }
 954         }
 955     }
 956
 957     return true;
 958 }
 959
 960
 961 #endif // QT_MULTIMEDIA_LIB