|
@@ -0,0 +1,208 @@
|
|
|
+package com.yonge.netty.server.messagehandler;
|
|
|
+
|
|
|
+import io.netty.buffer.ByteBuf;
|
|
|
+import io.netty.buffer.ByteBufUtil;
|
|
|
+import io.netty.channel.Channel;
|
|
|
+import io.netty.channel.ChannelHandler;
|
|
|
+import io.netty.channel.ChannelHandlerContext;
|
|
|
+import io.netty.channel.SimpleChannelInboundHandler;
|
|
|
+import io.netty.handler.codec.http.websocketx.BinaryWebSocketFrame;
|
|
|
+
|
|
|
+import java.io.File;
|
|
|
+
|
|
|
+import javax.sound.sampled.AudioFormat;
|
|
|
+
|
|
|
+import org.apache.commons.lang.ArrayUtils;
|
|
|
+import org.slf4j.Logger;
|
|
|
+import org.slf4j.LoggerFactory;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.stereotype.Component;
|
|
|
+
|
|
|
+import be.tarsos.dsp.AudioDispatcher;
|
|
|
+import be.tarsos.dsp.AudioEvent;
|
|
|
+import be.tarsos.dsp.io.TarsosDSPAudioFloatConverter;
|
|
|
+import be.tarsos.dsp.io.TarsosDSPAudioFormat;
|
|
|
+import be.tarsos.dsp.io.jvm.AudioDispatcherFactory;
|
|
|
+import be.tarsos.dsp.pitch.PitchDetectionHandler;
|
|
|
+import be.tarsos.dsp.pitch.PitchDetectionResult;
|
|
|
+import be.tarsos.dsp.pitch.PitchDetector;
|
|
|
+import be.tarsos.dsp.pitch.PitchProcessor;
|
|
|
+import be.tarsos.dsp.pitch.PitchProcessor.PitchEstimationAlgorithm;
|
|
|
+
|
|
|
+import com.yonge.audio.utils.ArrayUtil;
|
|
|
+import com.yonge.nettty.dto.UserChannelContext;
|
|
|
+import com.yonge.nettty.entity.MusicXmlNote;
|
|
|
+import com.yonge.netty.server.NettyChannelManager;
|
|
|
+import com.yonge.netty.server.processor.WaveformWriter;
|
|
|
+import com.yonge.netty.server.service.UserChannelContextService;
|
|
|
+
|
|
|
+@Component
|
|
|
+@ChannelHandler.Sharable
|
|
|
+public class BinaryWebSocketFrameHandler extends SimpleChannelInboundHandler<BinaryWebSocketFrame> {
|
|
|
+
|
|
|
+ private static final Logger LOGGER = LoggerFactory.getLogger(BinaryWebSocketFrameHandler.class);
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private NettyChannelManager nettyChannelManager;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private UserChannelContextService userChannelContextService;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * @describe 采样率
|
|
|
+ */
|
|
|
+ private float sampleRate = 44100;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 每个采样大小(Bit)
|
|
|
+ */
|
|
|
+ private int bitsPerSample = 16;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 通道数
|
|
|
+ */
|
|
|
+ private int channels = 1;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * @describe 采样大小
|
|
|
+ */
|
|
|
+ private int sampleSize = 1024 * 4;
|
|
|
+ /**
|
|
|
+ * @describe 帧覆盖大小
|
|
|
+ */
|
|
|
+ private int overlap = 256;
|
|
|
+
|
|
|
+ private boolean signed = true;
|
|
|
+
|
|
|
+ private boolean bigEndian = false;
|
|
|
+
|
|
|
+ private AudioFormat audioFormat = new AudioFormat(sampleRate, bitsPerSample, channels, signed, bigEndian);
|
|
|
+
|
|
|
+ private TarsosDSPAudioFloatConverter converter = TarsosDSPAudioFloatConverter.getConverter(new TarsosDSPAudioFormat(sampleRate, bitsPerSample, channels,
|
|
|
+ signed, bigEndian));
|
|
|
+
|
|
|
+ private PitchEstimationAlgorithm algorithm = PitchProcessor.PitchEstimationAlgorithm.FFT_YIN;
|
|
|
+
|
|
|
+ private PitchDetector pitchDetector = algorithm.getDetector(sampleRate, sampleSize);
|
|
|
+
|
|
|
+ /**
|
|
|
+ * @describe 有效分贝大小
|
|
|
+ */
|
|
|
+ private int validDb = 35;
|
|
|
+ /**
|
|
|
+ * @describe 有效频率
|
|
|
+ */
|
|
|
+ private int validFrequency = 20;
|
|
|
+ /**
|
|
|
+ * @describe 音准前后音分误差范围
|
|
|
+ */
|
|
|
+ private int intonationCentsRange = 3;
|
|
|
+ /**
|
|
|
+ * @describe 节奏有效阈值
|
|
|
+ */
|
|
|
+ private float cadenceValidDuty = 0.09f;
|
|
|
+ /**
|
|
|
+ * @describe 完整性有效频率误差范围
|
|
|
+ */
|
|
|
+ private int integrityFrequencyRange = 30;
|
|
|
+
|
|
|
+ private String tmpFileDir = "e:/soundRecords/";
|
|
|
+
|
|
|
+ @Override
|
|
|
+ protected void channelRead0(ChannelHandlerContext ctx, BinaryWebSocketFrame frame) throws Exception {
|
|
|
+
|
|
|
+ Channel channel = ctx.channel();
|
|
|
+
|
|
|
+ ByteBuf buf = frame.content().retain();
|
|
|
+
|
|
|
+ byte[] datas = ByteBufUtil.getBytes(buf);
|
|
|
+
|
|
|
+ String user = nettyChannelManager.getUser(channel);
|
|
|
+
|
|
|
+ UserChannelContext channelContext = userChannelContextService.getChannelContext(channel);
|
|
|
+
|
|
|
+ if (channelContext == null) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 写录音文件
|
|
|
+ WaveformWriter waveFileProcessor = channelContext.getWaveFileProcessor();
|
|
|
+ if (waveFileProcessor == null) {
|
|
|
+ File file = new File(tmpFileDir + user + "_" + System.currentTimeMillis() + ".wav");
|
|
|
+ waveFileProcessor = new WaveformWriter(file.getAbsolutePath());
|
|
|
+ channelContext.setWaveFileProcessor(waveFileProcessor);
|
|
|
+ }
|
|
|
+ waveFileProcessor.process(datas);
|
|
|
+
|
|
|
+ LOGGER.info("服务器接收到二进制消息长度[{}]", datas.length);
|
|
|
+
|
|
|
+ AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(datas, audioFormat, sampleSize, overlap);
|
|
|
+
|
|
|
+ dispatcher.addAudioProcessor(new PitchProcessor(algorithm, sampleRate, sampleSize, new PitchDetectionHandler() {
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void handlePitch(PitchDetectionResult pitchDetectionResult, AudioEvent audioEvent) {
|
|
|
+
|
|
|
+ // 获取字节流
|
|
|
+ int byteOverlap = audioEvent.getOverlap() * audioFormat.getFrameSize();
|
|
|
+ int byteStepSize = audioEvent.getBufferSize() * audioFormat.getFrameSize() - byteOverlap;
|
|
|
+ byte[] acceptDatas = ArrayUtils.subarray(audioEvent.getByteBuffer(), byteOverlap, byteStepSize);
|
|
|
+
|
|
|
+ // 粘合数据
|
|
|
+ byte[] totalBytes = ArrayUtil.mergeByte(channelContext.getBufferBytes(), acceptDatas);
|
|
|
+ channelContext.setBufferBytes(totalBytes);
|
|
|
+
|
|
|
+ LOGGER.info("新增字节数:{} 最新剩余字节长度:{}", acceptDatas.length, totalBytes.length);
|
|
|
+
|
|
|
+ // 获取当前音符信息
|
|
|
+ MusicXmlNote musicXmlNote = channelContext.getCurrentMusicNote(null);
|
|
|
+
|
|
|
+ // 计算当前音符的数据长度 公式:数据量(字节/秒)= 采样频率(Hz)× (采样位数(bit)/ 8) × 声道数
|
|
|
+ int length = (int) (audioFormat.getSampleRate() * (audioFormat.getSampleSizeInBits() / 8) * channels * musicXmlNote.getDuration() / 1000);
|
|
|
+
|
|
|
+ if (totalBytes.length >= length) {
|
|
|
+ // 处理当前音符
|
|
|
+ byte[] noteByteData = new byte[length];
|
|
|
+ System.arraycopy(totalBytes, 0, noteByteData, 0, length);
|
|
|
+
|
|
|
+ float[] noteFloatData = new float[length / audioFormat.getFrameSize()];
|
|
|
+
|
|
|
+ converter.toFloatArray(noteByteData, noteFloatData);
|
|
|
+
|
|
|
+ // 获取频率数据
|
|
|
+ float pitch = getPitch(noteFloatData, sampleSize);
|
|
|
+
|
|
|
+ LOGGER.info("第{}个音符的样本频率:{} 实际频率:{}", channelContext.getCurrentMusicNoteIndex(), musicXmlNote.getFrequency(), pitch);
|
|
|
+
|
|
|
+ // 准备处理下一个音符
|
|
|
+ channelContext.incrementMusicNoteIndex();
|
|
|
+ // 剩余未处理的数据
|
|
|
+ channelContext.setBufferBytes(ArrayUtil.extractByte(totalBytes, 0, length - 1));
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ }));
|
|
|
+ dispatcher.run();
|
|
|
+ }
|
|
|
+
|
|
|
+ private float getPitch(float[] audioBuffer, int bufferSize) {
|
|
|
+
|
|
|
+ int blankNum = audioBuffer.length % bufferSize;
|
|
|
+ float[] zeroBytes = new float[blankNum];
|
|
|
+
|
|
|
+ audioBuffer = ArrayUtil.mergeFloat(audioBuffer, zeroBytes);
|
|
|
+
|
|
|
+ int times = audioBuffer.length / bufferSize;
|
|
|
+
|
|
|
+ float totalPitch = 0f;
|
|
|
+
|
|
|
+ float[] bufferByte = new float[bufferSize];
|
|
|
+ for (int i = 0; i < times; i++) {
|
|
|
+ bufferByte = ArrayUtil.extractFloat(audioBuffer, i * bufferSize, (i + 1) * bufferSize);
|
|
|
+ totalPitch += pitchDetector.getPitch(bufferByte).getPitch();
|
|
|
+ }
|
|
|
+
|
|
|
+ return totalPitch / times;
|
|
|
+ }
|
|
|
+
|
|
|
+}
|