Skip to content

Compensate for resampler latency where it is utilized #531

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mdsitton wants to merge 2 commits into
base: master
Choose a base branch
from
+35 −15
Open

Compensate for resampler latency where it is utilized #531

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
97bf554
Add resampler latency to latency and position calculations
mdsitton Jun 7, 2020
b758e0d
Make sure we scale the returned number of samples to what the user st…
mdsitton Jun 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions src/cubeb_audiounit.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ struct cubeb_stream {
atomic<uint32_t> current_latency_frames{ 0 };
atomic<uint32_t> total_output_latency_frames { 0 };
unique_ptr<cubeb_resampler, decltype(&cubeb_resampler_destroy)> resampler;
float resample_ratio;
/* This is true if a device change callback is currently running. */
atomic<bool> switching_device{ false };
atomic<bool> buffer_size_change_state{ false };
Expand Down Expand Up @@ -2681,6 +2682,8 @@ audiounit_setup_stream(cubeb_stream * stm)
input_unconverted_params.rate = stm->input_hw_rate;
}

stm->resample_ratio = static_cast<float>(stm->output_stream_params.rate) / target_sample_rate;

/* Create resampler. Output params are unchanged
* because we do not need conversion on the output. */
stm->resampler.reset(cubeb_resampler_create(stm,
Expand Down Expand Up @@ -2969,10 +2972,13 @@ static int
audiounit_stream_get_position(cubeb_stream * stm, uint64_t * position)
{
assert(stm);
if (stm->current_latency_frames > stm->frames_played) {
uint32_t resample_latency = static_cast<uint32_t>(ceil(cubeb_resampler_latency(stm->resampler.get()) * stm->resample_ratio));
uint32_t latency_frames = stm->total_output_latency_frames + resample_latency;

if (latency_frames > stm->frames_played) {
*position = 0;
} else {
*position = stm->frames_played - stm->current_latency_frames;
*position = stm->frames_played - latency_frames;
}
return CUBEB_OK;
}
Expand All @@ -2984,7 +2990,9 @@ audiounit_stream_get_latency(cubeb_stream * stm, uint32_t * latency)
//TODO
return CUBEB_ERROR_NOT_SUPPORTED;
#else
*latency = stm->total_output_latency_frames;
uint32_t resample_latency = static_cast<uint32_t>(ceil(cubeb_resampler_latency(stm->resampler.get()) * stm->resample_ratio));
Copy link
Collaborator

@padenot padenot Sep 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is already in logical stream rate, no need to adjust.


*latency = stm->total_output_latency_frames + resample_latency;
return CUBEB_OK;
#endif
}
Expand Down
6 changes: 3 additions & 3 deletions src/cubeb_jack.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -658,9 +658,9 @@ cbjack_get_max_channel_count(cubeb * /*ctx*/, uint32_t * max_channels)
}

static int
cbjack_get_latency(cubeb_stream * stm, unsigned int * latency_ms)
cbjack_get_latency(cubeb_stream * stm, unsigned int * latency)
{
*latency_ms = stm->context->jack_latency;
*latency = stm->context->jack_latency + ceil(cubeb_resampler_latency(stm->resampler) * stm->ratio);
return CUBEB_OK;
}

Expand Down Expand Up @@ -955,7 +955,7 @@ cbjack_stream_stop(cubeb_stream * stream)
static int
cbjack_stream_get_position(cubeb_stream * stream, uint64_t * position)
{
*position = stream->position;
*position = stream->position - ceil(cubeb_resampler_latency(stream->resampler) * stream->ratio);
Copy link
Collaborator

@padenot padenot Sep 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will underflow on stream start, you want to branch here and return 0 if this would be negative: if the stream has just started, it's still filling the buffers in the underlying layer, and nothing is being output. Because stream_get_position is the time of what is being heard, then 0 is the correct answer in this situation.

return CUBEB_OK;
}

Expand Down
6 changes: 3 additions & 3 deletions src/cubeb_opensl.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1669,7 +1669,7 @@ opensl_stream_get_position(cubeb_stream * stm, uint64_t * position)
}

uint64_t samplerate = stm->user_output_rate;
uint32_t output_latency = stm->output_latency_ms;
uint32_t output_latency = stm->output_latency_ms + ((double)cubeb_resampler_latency(stm->resampler) / stm->output_configured_rate);

pthread_mutex_lock(&stm->mutex);
int64_t maximum_position = stm->written * (int64_t)stm->user_output_rate / stm->output_configured_rate;
Expand Down Expand Up @@ -1703,8 +1703,8 @@ opensl_stream_get_latency(cubeb_stream * stm, uint32_t * latency)

uint32_t stream_latency_frames =
stm->user_output_rate * (stm->output_latency_ms / 1000);

return stream_latency_frames + cubeb_resampler_latency(stm->resampler);
uint32_t resampler_latency_frames = (uint32_t)ceil(cubeb_resampler_latency(stm->resampler) * ((double)stm->user_output_rate) / stm->output_configured_rate);
return stream_latency_frames + resampler_latency_frames;
}

int
Expand Down
24 changes: 18 additions & 6 deletions src/cubeb_wasapi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2596,12 +2596,15 @@ int wasapi_stream_get_position(cubeb_stream * stm, uint64_t * position)
return CUBEB_ERROR;
}

double resample_ratio = stream_to_mix_samplerate_ratio(stm->output_stream_params, stm->output_mix_params);

/* Calculate how far behind the current stream head the playback cursor is. */
uint64_t stream_delay = static_cast<uint64_t>(current_stream_delay(stm) * stm->output_stream_params.rate);
uint64_t resampler_delay = ceil(cubeb_resampler_latency(stm->resampler.get()) * resample_ratio);
Copy link
Collaborator

@padenot padenot Sep 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is tricky. cubeb_resampler_latency returns the output latency, i.e., the number of frames of latency, expressed in output rate, so we don't need this, because stream_get_position returns its value in frames at the output (= stream) rate.

uint64_t stream_delay = static_cast<uint64_t>((current_stream_delay(stm) * stm->output_stream_params.rate)) + resampler_delay;

/* Calculate the logical stream head in frames at the stream sample rate. */
uint64_t max_pos = stm->total_frames_written +
static_cast<uint64_t>(round(stm->frames_written * stream_to_mix_samplerate_ratio(stm->output_stream_params, stm->output_mix_params)));
static_cast<uint64_t>(round(stm->frames_written * resample_ratio));

*position = max_pos;
if (stream_delay <= *position) {
Expand Down Expand Up @@ -2637,13 +2640,18 @@ int wasapi_stream_get_latency(cubeb_stream * stm, uint32_t * latency)
if (FAILED(hr)) {
return CUBEB_ERROR;
}


double resample_ratio = stream_to_mix_samplerate_ratio(stm->output_stream_params, stm->output_mix_params);
uint32_t resample_latency = ceil(cubeb_resampler_latency(stm->resampler.get()) * resample_ratio);
Copy link
Collaborator

@padenot padenot Sep 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same logic applies here.


// This happens on windows 10: no error, but always 0 for latency.
if (latency_hns == 0) {
double delay_s = current_stream_delay(stm);
// convert to sample-frames
*latency = delay_s * stm->output_stream_params.rate;
// convert to output(mix) rate sample-frames
*latency = (delay_s * stm->output_mix_params.rate) + resample_latency;
} else {
*latency = hns_to_frames(stm, latency_hns);
*latency = hns_to_frames(stm, latency_hns) + resample_latency;
}

return CUBEB_OK;
Expand All @@ -2664,7 +2672,11 @@ int wasapi_stream_get_input_latency(cubeb_stream * stm, uint32_t * latency)
return CUBEB_ERROR;
}

*latency = hns_to_frames(stm, stm->input_latency_hns);

double resample_ratio = stream_to_mix_samplerate_ratio(stm->output_stream_params, stm->output_mix_params);
Copy link
Collaborator

@padenot padenot Sep 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is input latency, and the input mix rate is frequently different from the output rate (for example because the microphone and the output device are two separate devices, that have been configured with different rates: this is sub-optimal, but happens).

uint32_t resample_latency = ceil(cubeb_resampler_latency(stm->resampler.get()) * resample_ratio);
Copy link
Collaborator

@padenot padenot Sep 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It appears that you can't write this function with the current API: cubeb_resampler is a bidirectional resampler, that can simultaneously resample an input stream at rate A (microphone) and an output stream at rate B (speakers), both to a rate C (that can be equal to either A or B). There is nothing in the API that surfaces the input latency, I'll add it.

In general there is some work needed on the resampler, that became very clear when working on https://github.com/kinetiknz/cubeb/pull/598


*latency = hns_to_frames(stm, stm->input_latency_hns) + resample_latency;

return CUBEB_OK;
}
Expand Down