update

audio-analysis/src/main/java/com/yonge/nettty/dto/UserChannelContext.java

@@ -3,34 +3,43 @@ package com.yonge.nettty.dto;
 import java.util.Comparator;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import be.tarsos.dsp.AudioEvent;
+import be.tarsos.dsp.pitch.PitchDetectionHandler;
+import be.tarsos.dsp.pitch.PitchDetectionResult;
+import be.tarsos.dsp.pitch.PitchDetector;
+import be.tarsos.dsp.pitch.PitchProcessor;
+
+import com.yonge.audio.utils.ArrayUtil;
 import com.yonge.nettty.entity.MusicXmlBasicInfo;
 import com.yonge.nettty.entity.MusicXmlNote;
+import com.yonge.nettty.entity.NoteAnalysis;
 import com.yonge.netty.server.processor.WaveformWriter;
 
 /**
  * 用户通道上下文
  */
-public class UserChannelContext {
+public class UserChannelContext implements PitchDetectionHandler {
+	
+	private final static Logger LOGGER = LoggerFactory.getLogger(UserChannelContext.class);
 
 	// 曲目与musicxml对应关系
 	private ConcurrentHashMap<Integer, MusicXmlBasicInfo> songMusicXmlMap = new ConcurrentHashMap<Integer, MusicXmlBasicInfo>();
 
 	private WaveformWriter waveFileProcessor;
 
-	// 当前音符索引
-	private AtomicInteger currentNoteIndex = new AtomicInteger(0);
-
-	// 当前乐谱小节索引
-	private AtomicInteger currentMusicSectionIndex = new AtomicInteger(0);
-
-	// 缓存字节数据
+	private NoteAnalysis processingNote = new NoteAnalysis(0, 0);
+	
 	private byte[] channelBufferBytes = new byte[0];
 	
-	private byte[] noteBufferBytes = new byte[0];
+	private float[] handlerBufferBytes = new float[0];
 
+	private PitchDetector pitchDetector = PitchProcessor.PitchEstimationAlgorithm.FFT_YIN.getDetector(44100, 1024 * 4);
+	
 	public ConcurrentHashMap<Integer, MusicXmlBasicInfo> getSongMusicXmlMap() {
 		return songMusicXmlMap;
 	}
@@ -47,29 +56,20 @@ public class UserChannelContext {
 		this.waveFileProcessor = waveFileProcessor;
 	}
 
-	public int incrementMusicNoteIndex() {
-		return currentNoteIndex.incrementAndGet();
-	}
-
-	public int getCurrentMusicNoteIndex() {
-		return currentNoteIndex.intValue();
+	public NoteAnalysis getProcessingNote() {
+		return processingNote;
 	}
 
-	public int incrementMusicSectionIndex() {
-		return currentMusicSectionIndex.incrementAndGet();
-	}
-
-	public int getCurrentMusicSectionIndex() {
-		return currentMusicSectionIndex.intValue();
+	public void setProcessingNote(NoteAnalysis processingNote) {
+		this.processingNote = processingNote;
 	}
 
 	public void resetUserInfo() {
 
 		waveFileProcessor = null;
-		currentNoteIndex.set(0);
-		currentMusicSectionIndex.set(0);
+		processingNote = new NoteAnalysis(0,0);
 		channelBufferBytes = new byte[0];
-		noteBufferBytes = new byte[0];
+		handlerBufferBytes = new float[0];
 	}
 
 	public MusicXmlNote getCurrentMusicNote(Integer songId) {
@@ -83,12 +83,8 @@ public class UserChannelContext {
 			musicXmlBasicInfo = songMusicXmlMap.get(songId);
 		}
 
-		if (musicXmlBasicInfo != null && getCurrentMusicNoteIndex() <= getTotalMusicNoteIndexNum(null)) {
-			/*
-			 * if (getCurrentMusicNoteIndex() >= musicXmlBasicInfo.getMusicXmlInfos().size()) { return
-			 * musicXmlBasicInfo.getMusicXmlInfos().get(musicXmlBasicInfo.getMusicXmlInfos().size() - 1); }
-			 */
-			return musicXmlBasicInfo.getMusicXmlInfos().stream().filter(t -> t.getMusicalNotesIndex() == getCurrentMusicNoteIndex()).findFirst().get();
+		if (musicXmlBasicInfo != null && processingNote.getIndex() <= getTotalMusicNoteIndexNum(null)) {
+			return musicXmlBasicInfo.getMusicXmlInfos().stream().filter(t -> t.getMusicalNotesIndex() == processingNote.getIndex()).findFirst().get();
 		}
 
 		return null;
@@ -124,7 +120,7 @@ public class UserChannelContext {
 		}
 
 		if (musicXmlBasicInfo != null) {
-			return musicXmlBasicInfo.getMusicXmlInfos().stream().filter(t -> t.getMusicalNotesIndex() == getCurrentMusicSectionIndex())
+			return musicXmlBasicInfo.getMusicXmlInfos().stream().filter(t -> t.getMusicalNotesIndex() == processingNote.getSectionIndex())
 					.sorted(Comparator.comparing(MusicXmlNote::getMusicalNotesIndex)).collect(Collectors.toList());
 		}
 
@@ -157,12 +153,107 @@ public class UserChannelContext {
 		this.channelBufferBytes = channelBufferBytes;
 	}
 
-	public byte[] getNoteBufferBytes() {
-		return noteBufferBytes;
+	public float[] getHandlerBufferBytes() {
+		return handlerBufferBytes;
+	}
+
+	public void setHandlerBufferBytes(float[] handlerBufferBytes) {
+		this.handlerBufferBytes = handlerBufferBytes;
 	}
 
-	public void setNoteBufferBytes(byte[] noteBufferBytes) {
-		this.noteBufferBytes = noteBufferBytes;
+	@Override
+	public void handlePitch(PitchDetectionResult pitchDetectionResult, AudioEvent audioEvent) {
+		
+		double durationTime = 1000 * (audioEvent.getFloatBuffer().length) / audioEvent.getSampleRate() / 2;
+		
+		float pitch = pitchDetectionResult.getPitch();
+		
+		//LOGGER.info("pitch:{} timeStamp:{} endTimeStamp:{} durationTime:{}", pitch, audioEvent.getTimeStamp(), audioEvent.getEndTimeStamp(), durationTime);
+		
+		// 获取当前音符信息
+		MusicXmlNote musicXmlNote = getCurrentMusicNote(null);
+
+		if (musicXmlNote == null) {
+			return;
+		}
+		
+		//取出当前处理中的音符信息
+		NoteAnalysis noteAnalysis = getProcessingNote();
+		if(noteAnalysis == null){
+			noteAnalysis = new NoteAnalysis(musicXmlNote.getMusicalNotesIndex(),musicXmlNote.getMeasureIndex());
+		}
+		
+		double noteDurationTime = noteAnalysis.getDurationTime() + durationTime;
+		noteAnalysis.setDurationTime(noteDurationTime);
+		
+		if(pitch != -1){
+			noteAnalysis.setChunks(noteAnalysis.getChunks() + 1);
+			noteAnalysis.setTotalPitch(noteAnalysis.getTotalPitch() + pitch);
+		}
+		
+		setProcessingNote(noteAnalysis);
+		
+		if(noteAnalysis.getIndex() <= getTotalMusicNoteIndexNum(null) && noteDurationTime >= musicXmlNote.getDuration()){
+			
+			noteAnalysis.setAvgPitch(noteAnalysis.getTotalPitch()/noteAnalysis.getChunks());
+			
+			LOGGER.info("当前音符下标[{}] 预计频率：{} 实际频率：{} 持续时间：{}", noteAnalysis.getIndex() , musicXmlNote.getFrequency(), noteAnalysis.getAvgPitch(), noteAnalysis.getDurationTime());
+			
+			// 准备处理下一个音符
+			setProcessingNote(noteAnalysis = new NoteAnalysis(musicXmlNote.getMusicalNotesIndex() + 1,musicXmlNote.getMeasureIndex()));
+		}
+		
+
+		/*// 获取字节流
+		float[] bufferBytes = audioEvent.getFloatBuffer();
+
+		// 粘合音符数据
+		float[] totalNoteBytes = ArrayUtil.mergeFloat(getHandlerBufferBytes(), bufferBytes);
+		setHandlerBufferBytes(totalNoteBytes);
+		
+
+		// 计算当前音符的数据长度 公式：数据量（字节/秒）= 采样频率（Hz）× （采样位数（bit）/ 8） × 声道数
+		int length = (int) (44100 * (16 / 8) * 1 * musicXmlNote.getDuration() / 1000);
+
+		if (noteAnalysis.getIndex() <= getTotalMusicNoteIndexNum(null) && totalNoteBytes.length >= length) {
+			// 处理当前音符
+			float[] noteFloatData = new float[length];
+			System.arraycopy(totalNoteBytes, 0, noteFloatData, 0, length);
+			// 剩余未处理的数据
+			setHandlerBufferBytes(ArrayUtil.extractFloat(totalNoteBytes, length - 1, totalNoteBytes.length - 1));
+
+			// 获取频率数据
+			float npitch = getPitch(noteFloatData, audioEvent.getBufferSize());
+
+			LOGGER.info("第{}个音符的样本频率：{} 实际频率：{}", noteAnalysis.getIndex(), musicXmlNote.getFrequency(), npitch);
+
+			// 准备处理下一个音符
+			setProcessingNote(noteAnalysis = new NoteAnalysis(musicXmlNote.getMusicalNotesIndex() + 1,musicXmlNote.getMeasureIndex()));
+		}*/
+	}
+
+	private float getPitch(float[] audioBuffer, int bufferSize) {
+
+		int blankNum = audioBuffer.length % bufferSize;
+		float[] zeroBytes = new float[blankNum];
+
+		audioBuffer = ArrayUtil.mergeFloat(audioBuffer, zeroBytes);
+
+		int times = audioBuffer.length / bufferSize;
+
+		float totalPitch = 0f;
+
+		float[] bufferByte = new float[bufferSize];
+		for (int i = 0; i < times; i++) {
+			bufferByte = ArrayUtil.extractFloat(audioBuffer, i * bufferSize, (i + 1) * bufferSize);
+			float pitch = pitchDetector.getPitch(bufferByte).getPitch();
+			if(pitch == -1){
+				continue;
+			}
+			totalPitch += pitch;
+		}
+
+		return totalPitch / times;
 	}
 
 }

audio-analysis/src/main/java/com/yonge/nettty/entity/NoteAnalysis.java

@@ -0,0 +1,70 @@
+package com.yonge.nettty.entity;
+
+public class NoteAnalysis {
+
+	private int index;
+
+	private double durationTime;
+
+	private double avgPitch;
+
+	private int sectionIndex;
+
+	private double totalPitch;
+
+	private int chunks;
+
+	public NoteAnalysis(int index, int sectionIndex) {
+		this.index = index;
+		this.sectionIndex = sectionIndex;
+	}
+
+	public int getIndex() {
+		return index;
+	}
+
+	public void setIndex(int index) {
+		this.index = index;
+	}
+
+	public double getDurationTime() {
+		return durationTime;
+	}
+
+	public void setDurationTime(double durationTime) {
+		this.durationTime = durationTime;
+	}
+
+	public double getAvgPitch() {
+		return avgPitch;
+	}
+
+	public void setAvgPitch(double avgPitch) {
+		this.avgPitch = avgPitch;
+	}
+
+	public int getSectionIndex() {
+		return sectionIndex;
+	}
+
+	public void setSectionIndex(int sectionIndex) {
+		this.sectionIndex = sectionIndex;
+	}
+
+	public double getTotalPitch() {
+		return totalPitch;
+	}
+
+	public void setTotalPitch(double totalPitch) {
+		this.totalPitch = totalPitch;
+	}
+
+	public int getChunks() {
+		return chunks;
+	}
+
+	public void setChunks(int chunks) {
+		this.chunks = chunks;
+	}
+
+}

audio-analysis/src/main/java/com/yonge/netty/server/NettyServer.java

@@ -76,8 +76,8 @@ public class NettyServer {
 		// 设置监听端口
 		bootstrap.localAddress(new InetSocketAddress(port));
 		// 服务端 accept 队列的大小
-		bootstrap.option(ChannelOption.SO_BACKLOG, 1024);
-		bootstrap.option(ChannelOption.SO_RCVBUF, 1024*4);
+		//bootstrap.option(ChannelOption.SO_BACKLOG, 1024);
+		//bootstrap.option(ChannelOption.SO_RCVBUF, 1024*4);
 		// 允许较小的数据包的发送，降低延迟
 		bootstrap.childOption(ChannelOption.TCP_NODELAY, true);
 		// 连接到达时会创建一个通道

audio-analysis/src/main/java/com/yonge/netty/server/NioAudioInputStream.java

@@ -0,0 +1,61 @@
+package com.yonge.netty.server;
+
+import be.tarsos.dsp.io.TarsosDSPAudioFormat;
+import be.tarsos.dsp.io.TarsosDSPAudioInputStream;
+import org.springframework.stereotype.Component;
+
+import javax.sound.sampled.AudioFormat;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+/**
+ * @Author Joburgess
+ * @Date 2021/8/4 0004
+ */
+public class NioAudioInputStream implements TarsosDSPAudioInputStream {
+
+    private RandomAccessFile randomAccessFile;
+    private AudioFormat format;
+
+    private long position = 44;
+
+    public NioAudioInputStream() {
+    }
+
+    public NioAudioInputStream(RandomAccessFile randomAccessFile, AudioFormat audioFormat) {
+        this.randomAccessFile = randomAccessFile;
+        this.format = audioFormat;
+    }
+
+    @Override
+    public long skip(long bytesToSkip) throws IOException {
+        return randomAccessFile.skipBytes((int) bytesToSkip);
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        randomAccessFile.seek(position);
+        int read = randomAccessFile.read(b, off, len);
+        if(read>0){
+            position += read;
+        }
+        return read;
+    }
+
+    @Override
+    public void close() throws IOException {
+        randomAccessFile.close();
+    }
+
+    @Override
+    public TarsosDSPAudioFormat getFormat() {
+        boolean isSigned = format.getEncoding() == AudioFormat.Encoding.PCM_SIGNED;
+        TarsosDSPAudioFormat tarsosDSPFormat = new TarsosDSPAudioFormat(format.getSampleRate(), format.getSampleSizeInBits(), format.getChannels(), isSigned, format.isBigEndian());
+        return tarsosDSPFormat;
+    }
+
+    @Override
+    public long getFrameLength() {
+        return 0;
+    }
+}

audio-analysis/src/main/java/com/yonge/netty/server/messagehandler/BinaryWebSocketFrameHandler.java

@@ -12,26 +12,19 @@ import java.io.File;
 
 import javax.sound.sampled.AudioFormat;
 
-import org.apache.commons.lang.ArrayUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import be.tarsos.dsp.AudioDispatcher;
-import be.tarsos.dsp.AudioEvent;
 import be.tarsos.dsp.io.TarsosDSPAudioFloatConverter;
 import be.tarsos.dsp.io.TarsosDSPAudioFormat;
 import be.tarsos.dsp.io.jvm.AudioDispatcherFactory;
-import be.tarsos.dsp.pitch.PitchDetectionHandler;
-import be.tarsos.dsp.pitch.PitchDetectionResult;
 import be.tarsos.dsp.pitch.PitchDetector;
 import be.tarsos.dsp.pitch.PitchProcessor;
 import be.tarsos.dsp.pitch.PitchProcessor.PitchEstimationAlgorithm;
 
 import com.yonge.audio.utils.ArrayUtil;
 import com.yonge.nettty.dto.UserChannelContext;
-import com.yonge.nettty.entity.MusicXmlNote;
 import com.yonge.netty.server.NettyChannelManager;
 import com.yonge.netty.server.processor.WaveformWriter;
 import com.yonge.netty.server.service.UserChannelContextService;
@@ -40,8 +33,6 @@ import com.yonge.netty.server.service.UserChannelContextService;
 @ChannelHandler.Sharable
 public class BinaryWebSocketFrameHandler extends SimpleChannelInboundHandler<BinaryWebSocketFrame> {
 
-	private static final Logger LOGGER = LoggerFactory.getLogger(BinaryWebSocketFrameHandler.class);
-
 	@Autowired
 	private NettyChannelManager nettyChannelManager;
 
@@ -66,7 +57,7 @@ public class BinaryWebSocketFrameHandler extends SimpleChannelInboundHandler<Bin
 	/**
 	 * @describe 采样大小
 	 */
-	private int bufferSize = 1024 * 2;
+	private int bufferSize = 1024 * 4;
 	/**
 	 * @describe 帧覆盖大小
 	 */
@@ -83,8 +74,6 @@ public class BinaryWebSocketFrameHandler extends SimpleChannelInboundHandler<Bin
 
 	private PitchEstimationAlgorithm algorithm = PitchProcessor.PitchEstimationAlgorithm.FFT_YIN;
 
-	private PitchDetector pitchDetector = algorithm.getDetector(sampleRate, bufferSize);
-
 	/**
 	 * @describe 有效分贝大小
 	 */
@@ -107,7 +96,7 @@ public class BinaryWebSocketFrameHandler extends SimpleChannelInboundHandler<Bin
 	private int integrityFrequencyRange = 30;
 
 	private String tmpFileDir = "e:/soundRecords/";
-
+	
 	@Override
 	protected void channelRead0(ChannelHandlerContext ctx, BinaryWebSocketFrame frame) throws Exception {
 
@@ -134,67 +123,19 @@ public class BinaryWebSocketFrameHandler extends SimpleChannelInboundHandler<Bin
 				channelContext.setWaveFileProcessor(waveFileProcessor);
 			}
 			waveFileProcessor.process(datas);
-
-			// LOGGER.info("服务器接收到的音频消息长度[{}]", datas.length);
-
-			// 粘合数据
-			byte[] totalBytes = ArrayUtil.mergeByte(channelContext.getChannelBufferBytes(), datas);
-			channelContext.setChannelBufferBytes(totalBytes);
-
-			if (totalBytes.length >= bufferSize * audioFormat.getFrameSize()) {
-
-				// 剩余未处理的数据
-				channelContext.setChannelBufferBytes(new byte[0]);
-
-				AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(totalBytes, audioFormat, bufferSize, overlap);
-
-				dispatcher.addAudioProcessor(new PitchProcessor(algorithm, sampleRate, bufferSize, new PitchDetectionHandler() {
-
-					@Override
-					public void handlePitch(PitchDetectionResult pitchDetectionResult, AudioEvent audioEvent) {
-
-						// 获取字节流
-						int byteOverlap = audioEvent.getOverlap() * audioFormat.getFrameSize();
-						int byteStepSize = audioEvent.getBufferSize() * audioFormat.getFrameSize() - byteOverlap;
-						byte[] bufferBytes = ArrayUtils.subarray(audioEvent.getByteBuffer(), byteOverlap, byteStepSize);
-
-						// 粘合音符数据
-						byte[] totalNoteBytes = ArrayUtil.mergeByte(channelContext.getNoteBufferBytes(), bufferBytes);
-						channelContext.setNoteBufferBytes(totalNoteBytes);
-
-						LOGGER.info("新增字节数：{} 最新剩余字节长度:{} 频率:{}", bufferBytes.length, totalNoteBytes.length, pitchDetectionResult.getPitch());
-
-						// 获取当前音符信息
-						MusicXmlNote musicXmlNote = channelContext.getCurrentMusicNote(null);
-
-						if (musicXmlNote == null) {
-							return;
-						}
-
-						// 计算当前音符的数据长度 公式：数据量（字节/秒）= 采样频率（Hz）× （采样位数（bit）/ 8） × 声道数
-						int length = (int) (audioFormat.getSampleRate() * (audioFormat.getSampleSizeInBits() / 8) * channels * musicXmlNote.getDuration() / 1000);
-
-						if (channelContext.getCurrentMusicNoteIndex() <= channelContext.getTotalMusicNoteIndexNum(null) && totalNoteBytes.length >= length) {
-							// 处理当前音符
-							byte[] noteByteData = new byte[length];
-							System.arraycopy(totalNoteBytes, 0, noteByteData, 0, length);
-
-							float[] noteFloatData = new float[length / audioFormat.getFrameSize()];
-
-							converter.toFloatArray(noteByteData, noteFloatData);
-
-							// 获取频率数据
-							float pitch = getPitch(noteFloatData, bufferSize);
-
-							LOGGER.info("第{}个音符的样本频率：{} 实际频率：{}", channelContext.getCurrentMusicNoteIndex(), musicXmlNote.getFrequency(), pitch);
-
-							// 准备处理下一个音符
-							channelContext.incrementMusicNoteIndex();
-							// 剩余未处理的数据
-							channelContext.setNoteBufferBytes(ArrayUtil.extractByte(totalNoteBytes, length - 1, totalNoteBytes.length - 1));
-						}
-					}
-				}));
+			
+			channelContext.setChannelBufferBytes(ArrayUtil.mergeByte(channelContext.getChannelBufferBytes(), datas));
+			
+			int totalLength = channelContext.getChannelBufferBytes().length;
+			
+			while(totalLength >= bufferSize){
+				byte[] bufferData = ArrayUtil.extractByte(channelContext.getChannelBufferBytes(), 0, bufferSize);
+				channelContext.setChannelBufferBytes(ArrayUtil.extractByte(channelContext.getChannelBufferBytes(), bufferSize - 1, totalLength - 1));
+				
+				totalLength = channelContext.getChannelBufferBytes().length;
+
+				AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(bufferData, audioFormat, bufferSize, overlap);
+				dispatcher.addAudioProcessor(new PitchProcessor(algorithm, sampleRate, bufferSize, channelContext));
 				dispatcher.run();
 			}
 
@@ -203,24 +144,4 @@ public class BinaryWebSocketFrameHandler extends SimpleChannelInboundHandler<Bin
 		}
 	}
 
-	private float getPitch(float[] audioBuffer, int bufferSize) {
-
-		int blankNum = audioBuffer.length % bufferSize;
-		float[] zeroBytes = new float[blankNum];
-
-		audioBuffer = ArrayUtil.mergeFloat(audioBuffer, zeroBytes);
-
-		int times = audioBuffer.length / bufferSize;
-
-		float totalPitch = 0f;
-
-		float[] bufferByte = new float[bufferSize];
-		for (int i = 0; i < times; i++) {
-			bufferByte = ArrayUtil.extractFloat(audioBuffer, i * bufferSize, (i + 1) * bufferSize);
-			totalPitch += pitchDetector.getPitch(bufferByte).getPitch();
-		}
-
-		return totalPitch / times;
-	}
-
 }