From e867c88ce91c018c298129e81380f6b87a299476 Mon Sep 17 00:00:00 2001 From: Aeterrno Amora <13880894553@163.com> Date: Mon, 4 Mar 2024 18:31:37 +0800 Subject: [PATCH 1/2] AudioSlicer: simplify an internal util function interface --- src/apps/AudioSlicer/slicer/slicer.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/apps/AudioSlicer/slicer/slicer.cpp b/src/apps/AudioSlicer/slicer/slicer.cpp index a72c1d7d..e89adf53 100644 --- a/src/apps/AudioSlicer/slicer/slicer.cpp +++ b/src/apps/AudioSlicer/slicer/slicer.cpp @@ -221,7 +221,7 @@ MarkerList Slicer::slice() // Need slicing. Record the range of silent frames to be removed. if ((i - silence_start) <= m_maxSilKept) { - pos = argmin_range_view(rms_list, silence_start, i + 1) + silence_start; + pos = argmin_range_view(rms_list, silence_start, i + 1); if (silence_start == 0) { sil_tags.emplace_back(0, pos); @@ -235,9 +235,8 @@ MarkerList Slicer::slice() else if ((i - silence_start) <= (m_maxSilKept * 2)) { pos = argmin_range_view(rms_list, i - m_maxSilKept, silence_start + m_maxSilKept + 1); - pos += i - m_maxSilKept; - pos_l = argmin_range_view(rms_list, silence_start, silence_start + m_maxSilKept + 1) + silence_start; - pos_r = argmin_range_view(rms_list, i - m_maxSilKept, i + 1) + i - m_maxSilKept; + pos_l = argmin_range_view(rms_list, silence_start, silence_start + m_maxSilKept + 1); + pos_r = argmin_range_view(rms_list, i - m_maxSilKept, i + 1); if (silence_start == 0) { clip_start = pos_r; @@ -250,8 +249,8 @@ MarkerList Slicer::slice() } } else { - pos_l = argmin_range_view(rms_list, silence_start, silence_start + m_maxSilKept + 1) + silence_start; - pos_r = argmin_range_view(rms_list, i - m_maxSilKept, i + 1) + i - m_maxSilKept; + pos_l = argmin_range_view(rms_list, silence_start, silence_start + m_maxSilKept + 1); + pos_r = argmin_range_view(rms_list, i - m_maxSilKept, i + 1); if (silence_start == 0) { sil_tags.emplace_back(0, pos_r); } @@ -266,7 +265,7 @@ MarkerList Slicer::slice() qint64 total_frames = rms_list.size(); if (has_silence_start && ((total_frames - silence_start) >= m_minInterval)) { qint64 silence_end = std::min(total_frames - 1, silence_start + m_maxSilKept); - pos = argmin_range_view(rms_list, silence_start, silence_end + 1) + silence_start; + pos = argmin_range_view(rms_list, silence_start, silence_end + 1); sil_tags.emplace_back(pos, total_frames + 1); } // Apply and return slices. @@ -378,14 +377,15 @@ inline std::vector get_rms(const std::vector& arr, qint64 frame_lengt template inline qint64 argmin_range_view(const std::vector& v, qint64 begin, qint64 end) { + // Return the absolute index in v, which is in range [begin, end) if possible // Ensure vector access is not out of bound auto size = v.size(); - if (begin > size) begin = size; + if (begin < 0) begin = 0; if (end > size) end = size; - if (begin >= end) return 0; + if (begin >= end) return begin; auto min_it = std::min_element(v.begin() + begin, v.begin() + end); - return std::distance(v.begin() + begin, min_it); + return std::distance(v.begin(), min_it); } template From 9e038c137ac5699d07227d223d03bdcbd2726765 Mon Sep 17 00:00:00 2001 From: Aeterrno Amora <13880894553@163.com> Date: Mon, 4 Mar 2024 18:39:16 +0800 Subject: [PATCH 2/2] AudioSlicer: better slicing algorithm --- src/apps/AudioSlicer/slicer/mainwindow.cpp | 6 +- src/apps/AudioSlicer/slicer/mainwindow_ui.cpp | 18 +- src/apps/AudioSlicer/slicer/mainwindow_ui.h | 4 +- src/apps/AudioSlicer/slicer/slicer.cpp | 175 +++++++----------- src/apps/AudioSlicer/slicer/slicer.h | 4 +- src/apps/AudioSlicer/slicer/workthread.cpp | 6 +- src/apps/AudioSlicer/slicer/workthread.h | 4 +- 7 files changed, 92 insertions(+), 125 deletions(-) diff --git a/src/apps/AudioSlicer/slicer/mainwindow.cpp b/src/apps/AudioSlicer/slicer/mainwindow.cpp index e09cf65d..ab9275db 100644 --- a/src/apps/AudioSlicer/slicer/mainwindow.cpp +++ b/src/apps/AudioSlicer/slicer/mainwindow.cpp @@ -67,7 +67,7 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWi auto validator = new QRegularExpressionValidator(QRegularExpression("\\d+")); ui->lineEditThreshold->setValidator(new QDoubleValidator()); - ui->lineEditMinLen->setValidator(validator); + ui->lineEditMaxLen->setValidator(validator); ui->lineEditMinInterval->setValidator(validator); ui->lineEditHopSize->setValidator(validator); ui->lineEditMaxSilence->setValidator(validator); @@ -272,7 +272,7 @@ void MainWindow::slot_start() { auto runnable = new WorkThread(path, ui->lineEditOutputDir->text(), ui->lineEditThreshold->text().toDouble(), - ui->lineEditMinLen->text().toLongLong(), + ui->lineEditMaxLen->text().toLongLong(), ui->lineEditMinInterval->text().toLongLong(), ui->lineEditHopSize->text().toLongLong(), ui->lineEditMaxSilence->text().toLongLong(), @@ -404,7 +404,7 @@ void MainWindow::setProcessing(bool processing) { ui->btnRemoveListItem->setEnabled(enabled); ui->btnClearList->setEnabled(enabled); ui->lineEditThreshold->setEnabled(enabled); - ui->lineEditMinLen->setEnabled(enabled); + ui->lineEditMaxLen->setEnabled(enabled); ui->lineEditMinInterval->setEnabled(enabled); ui->lineEditHopSize->setEnabled(enabled); ui->lineEditMaxSilence->setEnabled(enabled); diff --git a/src/apps/AudioSlicer/slicer/mainwindow_ui.cpp b/src/apps/AudioSlicer/slicer/mainwindow_ui.cpp index c51db5d4..727c9614 100644 --- a/src/apps/AudioSlicer/slicer/mainwindow_ui.cpp +++ b/src/apps/AudioSlicer/slicer/mainwindow_ui.cpp @@ -111,16 +111,16 @@ void Ui_MainWindow::setupUi(QMainWindow *MainWindow) { formLayout->setWidget(0, QFormLayout::FieldRole, lineEditThreshold); - lblMinLen = new QLabel(gBoxParameters); - lblMinLen->setObjectName("lblMinLen"); + lblMaxLen = new QLabel(gBoxParameters); + lblMaxLen->setObjectName("lblMaxLen"); - formLayout->setWidget(1, QFormLayout::LabelRole, lblMinLen); + formLayout->setWidget(1, QFormLayout::LabelRole, lblMaxLen); - lineEditMinLen = new QLineEdit(gBoxParameters); - lineEditMinLen->setObjectName("lineEditMinLen"); - lineEditMinLen->setAlignment(Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter); + lineEditMaxLen = new QLineEdit(gBoxParameters); + lineEditMaxLen->setObjectName("lineEditMaxLen"); + lineEditMaxLen->setAlignment(Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter); - formLayout->setWidget(1, QFormLayout::FieldRole, lineEditMinLen); + formLayout->setWidget(1, QFormLayout::FieldRole, lineEditMaxLen); lblMinInterval = new QLabel(gBoxParameters); lblMinInterval->setObjectName("lblMinInterval"); @@ -373,8 +373,8 @@ void Ui_MainWindow::retranslateUi(QMainWindow *MainWindow) gBoxParameters->setTitle(QCoreApplication::translate("MainWindow", "Parameters", nullptr)); lblThreshold->setText(QCoreApplication::translate("MainWindow", "Threshold (dB)", nullptr)); lineEditThreshold->setText(QCoreApplication::translate("MainWindow", "-40", nullptr)); - lblMinLen->setText(QCoreApplication::translate("MainWindow", "Minimum Length (ms)", nullptr)); - lineEditMinLen->setText(QCoreApplication::translate("MainWindow", "5000", nullptr)); + lblMaxLen->setText(QCoreApplication::translate("MainWindow", "Maximum Length (ms)", nullptr)); + lineEditMaxLen->setText(QCoreApplication::translate("MainWindow", "20000", nullptr)); lblMinInterval->setText(QCoreApplication::translate("MainWindow", "Minimum Interval (ms)", nullptr)); lineEditMinInterval->setText(QCoreApplication::translate("MainWindow", "300", nullptr)); lblHopSize->setText(QCoreApplication::translate("MainWindow", "Hop Size (ms)", nullptr)); diff --git a/src/apps/AudioSlicer/slicer/mainwindow_ui.h b/src/apps/AudioSlicer/slicer/mainwindow_ui.h index 520d0b27..46594397 100644 --- a/src/apps/AudioSlicer/slicer/mainwindow_ui.h +++ b/src/apps/AudioSlicer/slicer/mainwindow_ui.h @@ -60,8 +60,8 @@ class Ui_MainWindow QFormLayout *formLayout; QLabel *lblThreshold; QLineEdit *lineEditThreshold; - QLabel *lblMinLen; - QLineEdit *lineEditMinLen; + QLabel *lblMaxLen; + QLineEdit *lineEditMaxLen; QLabel *lblMinInterval; QLineEdit *lineEditMinInterval; QLabel *lblHopSize; diff --git a/src/apps/AudioSlicer/slicer/slicer.cpp b/src/apps/AudioSlicer/slicer/slicer.cpp index e89adf53..8889a2fd 100644 --- a/src/apps/AudioSlicer/slicer/slicer.cpp +++ b/src/apps/AudioSlicer/slicer/slicer.cpp @@ -68,16 +68,15 @@ qint64 MovingRMS::size() { return m_numElements; } -Slicer::Slicer(SndfileHandle *decoder, double threshold, qint64 minLength, qint64 minInterval, qint64 hopSize, qint64 maxSilKept) +Slicer::Slicer(SndfileHandle *decoder, double threshold, qint64 maxLength, qint64 minInterval, qint64 hopSize, qint64 maxSilKept) { m_errCode = SlicerErrorCode::SLICER_OK; - if ((!((minLength >= minInterval) && (minInterval >= hopSize))) || (maxSilKept < hopSize)) + if (!((minInterval >= hopSize) && (maxSilKept >= hopSize) && (maxLength >= maxSilKept * 2))) { - // The following condition must be satisfied: m_minLength >= m_minInterval >= m_hopSize - // The following condition must be satisfied: m_maxSilKept >= m_hopSize + // The following condition must be satisfied: (m_minInterval >= m_hopSize) and (m_maxLength / 2 >= m_maxSilKept >= m_hopSize) m_errCode = SlicerErrorCode::SLICER_INVALID_ARGUMENT; m_errMsg = "ValueError: The following conditions must be satisfied: " - "(min_length >= min_interval >= hop_size) and (max_sil_kept >= hop_size)."; + "(min_interval >= hop_size) and (max_length / 2 >= max_sil_kept >= hop_size)."; return; } @@ -99,7 +98,7 @@ Slicer::Slicer(SndfileHandle *decoder, double threshold, qint64 minLength, qint6 m_threshold = std::pow(10, threshold / 20.0); m_hopSize = divIntRound(hopSize * (qint64)sr, (qint64)1000); m_winSize = std::min(divIntRound(minInterval * (qint64)sr, (qint64)1000), (qint64)4 * m_hopSize); - m_minLength = divIntRound(minLength * (qint64)sr, (qint64)1000 * m_hopSize); + m_maxLength = divIntRound(maxLength * (qint64)sr, (qint64)1000 * m_hopSize); m_minInterval = divIntRound(minInterval * (qint64)sr, (qint64)1000 * m_hopSize); m_maxSilKept = divIntRound(maxSilKept * (qint64)sr, (qint64)1000 * m_hopSize); } @@ -115,7 +114,7 @@ MarkerList Slicer::slice() qint64 frames = m_decoder->frames(); int channels = m_decoder->channels(); - if ((frames + m_hopSize - 1) / m_hopSize <= m_minLength) + if ((frames + m_hopSize - 1) / m_hopSize <= m_maxLength) { return {{ 0, frames }}; } @@ -181,118 +180,86 @@ MarkerList Slicer::slice() //std::vector rms_list = get_rms(samples, (qint64) m_winSize, (qint64) m_hopSize); - MarkerList sil_tags; - qint64 silence_start = 0; - bool has_silence_start = false; - qint64 clip_start = 0; + // Scan for silence - qint64 pos = 0, pos_l = 0, pos_r = 0; + MarkerList silences; + qint64 silence_start = -1<<30; + bool was_silent = true; // If 0 is not silent, it detects an inf-length silence [-inf, 0) as opening guard. for (qint64 i = 0; i < rms_list.size(); i++) { - double rms = rms_list[i]; - // Keep looping while frame is silent. - if (rms < m_threshold) - { - // Record start of silent frames. - if (!has_silence_start) - { + if (rms_list[i] < m_threshold) + { // This frame is silent. + if (!was_silent) + { // Start of silence. + was_silent = true; silence_start = i; - has_silence_start = true; } - continue; - } - // Keep looping while frame is not silent and silence start has not been recorded. - if (!has_silence_start) - { - continue; - } - // Clear recorded silence start if interval is not enough or clip is too short - bool is_leading_silence = ((silence_start == 0) && (i > m_maxSilKept)); - bool need_slice_middle = ( - ( (i - silence_start) >= m_minInterval) && - ( (i - clip_start) >= m_minLength) ); - if ((!is_leading_silence) && (!need_slice_middle)) - { - has_silence_start = false; - continue; } - - // Need slicing. Record the range of silent frames to be removed. - if ((i - silence_start) <= m_maxSilKept) - { - pos = argmin_range_view(rms_list, silence_start, i + 1); - if (silence_start == 0) - { - sil_tags.emplace_back(0, pos); - } - else - { - sil_tags.emplace_back(pos, pos); - } - clip_start = pos; - } - else if ((i - silence_start) <= (m_maxSilKept * 2)) - { - pos = argmin_range_view(rms_list, i - m_maxSilKept, silence_start + m_maxSilKept + 1); - pos_l = argmin_range_view(rms_list, silence_start, silence_start + m_maxSilKept + 1); - pos_r = argmin_range_view(rms_list, i - m_maxSilKept, i + 1); - if (silence_start == 0) - { - clip_start = pos_r; - sil_tags.emplace_back(0, clip_start); - } - else - { - clip_start = std::max(pos_r, pos); - sil_tags.emplace_back(std::min(pos_l, pos), clip_start); - } - } - else { - pos_l = argmin_range_view(rms_list, silence_start, silence_start + m_maxSilKept + 1); - pos_r = argmin_range_view(rms_list, i - m_maxSilKept, i + 1); - if (silence_start == 0) { - sil_tags.emplace_back(0, pos_r); - } - else { - sil_tags.emplace_back(pos_l, pos_r); + else + { // This frame is not silent. + if (was_silent) + { // End of silence range [silence_start, i). + was_silent = false; + if (silence_start < 0) + { // Prepend inf-length silence as left boundary guard. + qint64 pos_r = argmin_range_view(rms_list, i - m_maxSilKept, i + 1); + silences.emplace_back(-1<<30, pos_r); + } + else + { + // Find break points in this silence. + qint64 pos_l = argmin_range_view(rms_list, silence_start, std::min(silence_start + m_maxSilKept, i) + 1); + qint64 pos_r = argmin_range_view(rms_list, std::max(silence_start, i - m_maxSilKept), i + 1); + // assert(pos_l <= pos_r); // by simple mathematics + silences.emplace_back(pos_l, pos_r); + } } - clip_start = pos_r; } - has_silence_start = false; + + if (silences.empty()) { + return {}; // entirely silent } - // Deal with trailing silence. - qint64 total_frames = rms_list.size(); - if (has_silence_start && ((total_frames - silence_start) >= m_minInterval)) { - qint64 silence_end = std::min(total_frames - 1, silence_start + m_maxSilKept); - pos = argmin_range_view(rms_list, silence_start, silence_end + 1); - sil_tags.emplace_back(pos, total_frames + 1); + // Append inf-length silence as right boundary guard. + if (!was_silent) + { + silence_start = rms_list.size(); } - // Apply and return slices. - if (sil_tags.empty()) { - return {{ 0, frames }}; + qint64 pos_l = argmin_range_view(rms_list, silence_start, silence_start + m_maxSilKept + 1); + silences.emplace_back(pos_l, 1<<30); + + std::vector> order(silences.size()); + std::vector prev(silences.size()), next(silences.size()); // doubly linked list of silences + for (qint64 i = 0; i < silences.size(); i++) + { + auto sil = silence[i]; + order[i] = std::make_pair(std::max(sil.second - sil.first, 0), i); + prev[i] = i - 1; + next[i] = i + 1; } - else { - MarkerList chunks; - qint64 begin = 0, end = 0; - qint64 s0 = sil_tags[0].first; - if (s0 > 0) { - begin = 0; - end = s0; - chunks.emplace_back(begin * m_hopSize, std::min(frames, end * m_hopSize)); - } - for (auto i = 0; i < sil_tags.size() - 1; i++) { - begin = sil_tags[i].second; - end = sil_tags[i + 1].first; - chunks.emplace_back(begin * m_hopSize, std::min(frames, end * m_hopSize)); + std::sort(order.start(), order.end()); // sort silences from short to long + + for (auto o : order) + { + if (o.first >> 29) // length is inf + { + break; } - if (sil_tags.back().second < total_frames) { - begin = sil_tags.back().second; - end = total_frames; - chunks.emplace_back(begin * m_hopSize, std::min(frames, end * m_hopSize)); + qint64 i = o.second; + qint64 pr = prev[i], nx = next[i]; + if (silences[nx].first - silences[pr].second <= m_maxLength) + { // If possible, remove this short silence. + next[pr] = nx; + prev[nx] = pr; } - return chunks; } + + MarkerList chunks; + for (qint64 i = 0; next[i] < silences.size(); i = next[i]) + { + chunks.emplace_back(silences[i].second * m_hopSize, std::min(frames, silences[next[i]].first * m_hopSize)); + } + return chunks; } SlicerErrorCode Slicer::getErrorCode() { diff --git a/src/apps/AudioSlicer/slicer/slicer.h b/src/apps/AudioSlicer/slicer/slicer.h index 66a0cf60..080c8d30 100644 --- a/src/apps/AudioSlicer/slicer/slicer.h +++ b/src/apps/AudioSlicer/slicer/slicer.h @@ -21,7 +21,7 @@ class Slicer { double m_threshold; qint64 m_hopSize; qint64 m_winSize; - qint64 m_minLength; + qint64 m_maxLength; qint64 m_minInterval; qint64 m_maxSilKept; SlicerErrorCode m_errCode; @@ -29,7 +29,7 @@ class Slicer { SndfileHandle *m_decoder; public: - explicit Slicer(SndfileHandle *decoder, double threshold = -40.0, qint64 minLength = 5000, qint64 minInterval = 300, qint64 hopSize = 20, qint64 maxSilKept = 5000); + explicit Slicer(SndfileHandle *decoder, double threshold = -40.0, qint64 maxLength = 5000, qint64 minInterval = 300, qint64 hopSize = 20, qint64 maxSilKept = 5000); MarkerList slice(); SlicerErrorCode getErrorCode(); QString getErrorMsg(); diff --git a/src/apps/AudioSlicer/slicer/workthread.cpp b/src/apps/AudioSlicer/slicer/workthread.cpp index 860640b4..4d5b7930 100644 --- a/src/apps/AudioSlicer/slicer/workthread.cpp +++ b/src/apps/AudioSlicer/slicer/workthread.cpp @@ -47,11 +47,11 @@ inline QString samplesToDecimalFormat(qint64 samples, int sampleRate); inline qint64 decimalFormatToSamples(const QStringView &decimalFormat, int sampleRate, bool *ok = nullptr); inline qint64 decimalFormatToSamples(const QString &decimalFormat, int sampleRate, bool *ok = nullptr); -WorkThread::WorkThread(const QString &filename, const QString &outPath, double threshold, qint64 minLength, qint64 minInterval, +WorkThread::WorkThread(const QString &filename, const QString &outPath, double threshold, qint64 maxLength, qint64 minInterval, qint64 hopSize, qint64 maxSilKept, int outputWaveFormat, bool saveAudio, bool saveMarkers, bool loadMarkers, bool overwriteMarkers, int minimumDigits, int listIndex) - : m_filename(filename), m_outPath(outPath), m_threshold(threshold), m_minLength(minLength), + : m_filename(filename), m_outPath(outPath), m_threshold(threshold), m_maxLength(maxLength), m_minInterval(minInterval), m_hopSize(hopSize), m_maxSilKept(maxSilKept), m_outputWaveFormat(outputWaveFormat), m_saveAudio(saveAudio), m_saveMarkers(saveMarkers), m_loadMarkers(loadMarkers), m_overwriteMarkers(overwriteMarkers), m_minimumDigits(minimumDigits), m_listIndex(listIndex) {} @@ -117,7 +117,7 @@ void WorkThread::run() { } if (!hasExistingMarkers) { emit oneInfo(QString("%1: calculating markers").arg(m_filename)); - Slicer slicer(&sf, m_threshold, m_minLength, m_minInterval, m_hopSize, m_maxSilKept); + Slicer slicer(&sf, m_threshold, m_maxLength, m_minInterval, m_hopSize, m_maxSilKept); if (slicer.getErrorCode() != SlicerErrorCode::SLICER_OK) { emit oneError("slicer: " + slicer.getErrorMsg()); diff --git a/src/apps/AudioSlicer/slicer/workthread.h b/src/apps/AudioSlicer/slicer/workthread.h index d02dea72..e954c281 100644 --- a/src/apps/AudioSlicer/slicer/workthread.h +++ b/src/apps/AudioSlicer/slicer/workthread.h @@ -15,7 +15,7 @@ Q_OBJECT WorkThread(const QString &filename, const QString &outPath, double threshold, - qint64 minLength, + qint64 maxLength, qint64 minInterval, qint64 hopSize, qint64 maxSilKept, @@ -32,7 +32,7 @@ Q_OBJECT QString m_filename; QString m_outPath; double m_threshold; - qint64 m_minLength; + qint64 m_maxLength; qint64 m_minInterval; qint64 m_hopSize; qint64 m_maxSilKept;