update

audio-analysis/src/main/java/com/yonge/nettty/dto/HardLevelEnum.java

@@ -21,10 +21,10 @@ public enum HardLevelEnum implements BaseEnum<String, HardLevelEnum> {
 	 * 
 	 * @param msg
 	 * @param amplitudeThreshold 振幅阈值
-	 * @param frequencyOffset
-	 * @param tempoOffsetOfPercent
-	 * @param integrityRange
-	 * @param notPlayRange
+	 * @param frequencyOffset 频率法制
+	 * @param tempoOffsetOfPercent 节奏偏移量百分比（在当前范围内节奏才算正确）
+	 * @param integrityRange 完成度范围
+	 * @param notPlayRange 未演奏的范围
 	 */
 	HardLevelEnum(String msg, int amplitudeThreshold, int frequencyOffset, int tempoOffsetOfPercent, int integrityRange, int notPlayRange) {
 		this.msg = msg;

audio-analysis/src/main/java/com/yonge/nettty/dto/NoteAnalysis.java

@@ -53,8 +53,6 @@ public class NoteAnalysis {
 
 	private boolean ignore;
 	
-	private ChunkAnalysis lastChunkAnalysis;
-
 	public NoteAnalysis(int index, int sectionIndex, int frequency, double standardDurationTime) {
 		this.standardDurationTime = standardDurationTime;
 		this.index = index;
@@ -189,12 +187,4 @@ public class NoteAnalysis {
 		this.integrityScore = integrityScore;
 	}
 
-	public ChunkAnalysis getLastChunkAnalysis() {
-		return lastChunkAnalysis;
-	}
-
-	public void setLastChunkAnalysis(ChunkAnalysis lastChunkAnalysis) {
-		this.lastChunkAnalysis = lastChunkAnalysis;
-	}
-
 }

audio-analysis/src/main/java/com/yonge/nettty/dto/UserChannelContext.java

@@ -16,12 +16,9 @@ import javax.sound.sampled.AudioFormat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import be.tarsos.dsp.AudioEvent;
-import be.tarsos.dsp.pitch.PitchDetectionHandler;
-import be.tarsos.dsp.pitch.PitchDetectionResult;
-
 import com.yonge.audio.analysis.Signals;
 import com.yonge.audio.analysis.detector.YINPitchDetector;
+import com.yonge.audio.utils.ArrayUtil;
 import com.yonge.nettty.dto.NoteAnalysis.NoteErrorType;
 import com.yonge.nettty.entity.MusicXmlBasicInfo;
 import com.yonge.nettty.entity.MusicXmlNote;
@@ -31,16 +28,18 @@ import com.yonge.netty.server.processor.WaveformWriter;
 /**
  * 用户通道上下文
  */
-public class UserChannelContext implements PitchDetectionHandler {
+public class UserChannelContext {
 	
 	private final static Logger LOGGER = LoggerFactory.getLogger(UserChannelContext.class);
 	
-	private final double offsetMS = 300;
+	private final double offsetMS = 350;
 	
 	private Long recordId;
 	
 	private Integer subjectId;
 	
+	private int beatByteLength;
+	
 	// 曲目与musicxml对应关系
 	private ConcurrentHashMap<Integer, MusicXmlBasicInfo> songMusicXmlMap = new ConcurrentHashMap<Integer, MusicXmlBasicInfo>();
 
@@ -66,8 +65,23 @@ public class UserChannelContext implements PitchDetectionHandler {
 	
 	private HardLevelEnum hardLevel = HardLevelEnum.ADVANCED;
 	
-	public void init(){
-
+	public void init(String heardLevel, int subjectId, int beatByteLength) {
+		this.subjectId = subjectId;
+		this.beatByteLength = WaveformWriter.SAMPLE_RATE * WaveformWriter.BITS_PER_SAMPLE / 8 * beatByteLength / 1000;
+		//hardLevel = HardLevelEnum.valueOf(heardLevel);
+	}
+	
+	public byte[] skipHeader(byte[] datas) {
+		if (beatByteLength > 0) {
+			if (datas.length <= beatByteLength) {
+				beatByteLength -= datas.length;
+				return new byte[0];
+			}
+			byte[] data = ArrayUtil.extractByte(datas, beatByteLength, datas.length - 1);
+			beatByteLength = 0;
+			return data;
+		}
+		return datas;
 	}
 	
 	public Long getRecordId() {
@@ -110,6 +124,10 @@ public class UserChannelContext implements PitchDetectionHandler {
 		return doneNoteAnalysisList;
 	}
 
+	public int getBeatByteLength() {
+		return beatByteLength;
+	}
+
 	public void resetUserInfo() {
 		waveFileProcessor = null;
 		processingNote = new NoteAnalysis(0,0,-1);
@@ -275,7 +293,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 				if(Math.abs(chunkAnalysisList.get(chunkAnalysisList.size() - 1).getFrequency() - lastChunkAnalysis.getFrequency()) > hardLevel.getFrequencyOffset()){
 					lastChunkAnalysis.setFrequency(-1);
 				}
-				if(chunkAnalysisList.get(chunkAnalysisList.size() - 1).getAmplitude() < lastChunkAnalysis.getAmplitude()){
+				if(chunkAnalysisList.get(chunkAnalysisList.size() - 1).getAmplitude() + 2 < lastChunkAnalysis.getAmplitude()){
 					lastChunkAnalysis.setPeak(true);
 				}
 				
@@ -307,7 +325,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 					}
 				} else {
 					if (subjectId == 23) {
-						tempo = computeTempoWithAmplitude(chunkAnalysisList, lastChunkAnalysis);
+						tempo = computeTempoWithAmplitude2(chunkAnalysisList, lastChunkAnalysis);
 					} else {
 						tempo = computeTempoWithFrequency(chunkAnalysisList, lastChunkAnalysis);
 					}
@@ -361,123 +379,6 @@ public class UserChannelContext implements PitchDetectionHandler {
 		
 	}
 	
-	public void handle2(float[] samples, AudioFormat audioFormat){
-		
-		//FrequencyDetector frequencyDetector = new FrequencyDetector(samples, audioFormat.getSampleRate(), false);
-		YINPitchDetector frequencyDetector = new YINPitchDetector(samples.length/2 , audioFormat.getSampleRate());
-		int frequency = (int)frequencyDetector.getFrequency(samples);
-		
-		int splDb = (int)Signals.soundPressureLevel(samples);
-		
-		int power = (int)Signals.power(samples);
-		
-		int energy = (int)Signals.energy(samples);
-		
-		LOGGER.info("Frequency:{}  SplDb:{}  Power:{}", frequency, splDb, power);
-		
-		double durationTime = 1000 * (samples.length * 2) / audioFormat.getSampleRate() / (audioFormat.getSampleSizeInBits() / 8);
-		
-		double startTime = 0;
-		
-		if(chunkAnalysisList.size() > 0){
-			ChunkAnalysis lastestChunk = chunkAnalysisList.get(chunkAnalysisList.size() - 1);
-			startTime = lastestChunk.getEndTime();
-			
-			if(Math.abs((lastestChunk.getFrequency() - frequency)) > 10 || Math.abs(lastestChunk.getPower() - power) > 0.1){
-				
-				double avgFrequency = chunkAnalysisList.stream().collect(Collectors.averagingDouble(ChunkAnalysis::getFrequency));
-				NoteAnalysis noteAnalysis = new NoteAnalysis(chunkAnalysisList.get(0).getStartTime(), startTime, (int)avgFrequency);
-				doneNoteAnalysisList.add(noteAnalysis);//添加演奏的一个音符
-				
-				//重置
-				chunkAnalysisList.clear();
-				
-				//判断是否需要评分
-				MusicXmlSection musicXmlSection = getCurrentMusicSection(null, 0);
-				if(musicXmlSection != null){
-					if(musicXmlSection.getDuration() < startTime){
-						
-					}
-				}
-				
-			}
-		}
-		
-		chunkAnalysisList.add(new ChunkAnalysis(startTime, startTime + durationTime, frequency, splDb, power, energy));
-	}
-
-	@Override
-	public void handlePitch(PitchDetectionResult pitchDetectionResult, AudioEvent audioEvent) {
-		
-		double durationTime = 1000 * (audioEvent.getFloatBuffer().length) / audioEvent.getSampleRate() / 2;
-		
-		float pitch = pitchDetectionResult.getPitch();
-		
-		LOGGER.info("pitch:{} timeStamp:{} endTimeStamp:{} durationTime:{}", pitch, audioEvent.getTimeStamp(), audioEvent.getEndTimeStamp(), durationTime);
-		
-		// 获取当前音符信息
-		MusicXmlNote musicXmlNote = getCurrentMusicNote(null,null);
-
-		if (musicXmlNote == null) {
-			return;
-		}
-		
-		//取出当前处理中的音符信息
-		NoteAnalysis noteAnalysis = getProcessingNote();
-		if(noteAnalysis == null){
-			noteAnalysis = new NoteAnalysis(musicXmlNote.getMusicalNotesIndex(),musicXmlNote.getMeasureIndex(),(int)musicXmlNote.getFrequency());
-		}
-		
-		double noteDurationTime = noteAnalysis.getDurationTime() + durationTime;
-		noteAnalysis.setDurationTime(noteDurationTime);
-		
-		if(pitch != -1){
-			//noteAnalysis.getChunkFrequencyList().add((double) pitch);
-		}
-		
-		setProcessingNote(noteAnalysis);
-		
-		if(noteAnalysis.getMusicalNotesIndex() <= getTotalMusicNoteIndex(null) && noteDurationTime >= musicXmlNote.getDuration()){
-			
-			//noteAnalysis.setPlayFrequency(noteAnalysis.getChunkFrequencyList().stream().mapToDouble(t -> t).sum()/noteAnalysis.getChunkFrequencyList().size());
-			
-			LOGGER.info("当前音符下标[{}] 预计频率：{} 实际频率：{} 持续时间：{}", noteAnalysis.getMusicalNotesIndex() , musicXmlNote.getFrequency(), noteAnalysis.getPlayFrequency(), noteAnalysis.getDurationTime());
-			
-			// 准备处理下一个音符
-			int nextNoteIndex = musicXmlNote.getMusicalNotesIndex() + 1;
-			NoteAnalysis nextNoteAnalysis = new NoteAnalysis(nextNoteIndex, getMusicSectionIndex(null, nextNoteIndex), (int)getCurrentMusicNote(null,
-					nextNoteIndex).getFrequency());
-			setProcessingNote(nextNoteAnalysis);
-		}
-		
-
-		/*// 获取字节流
-		float[] bufferBytes = audioEvent.getFloatBuffer();
-
-		// 粘合音符数据
-		float[] totalNoteBytes = ArrayUtil.mergeFloat(getHandlerBufferBytes(), bufferBytes);
-		setHandlerBufferBytes(totalNoteBytes);
-		
-
-		// 计算当前音符的数据长度 公式：数据量（字节/秒）= 采样频率（Hz）× （采样位数（bit）/ 8） × 声道数
-		int length = (int) (44100 * (16 / 8) * 1 * musicXmlNote.getDuration() / 1000);
-
-		if (noteAnalysis.getIndex() <= getTotalMusicNoteIndexNum(null) && totalNoteBytes.length >= length) {
-			// 处理当前音符
-			float[] noteFloatData = new float[length];
-			System.arraycopy(totalNoteBytes, 0, noteFloatData, 0, length);
-			// 剩余未处理的数据
-			setHandlerBufferBytes(ArrayUtil.extractFloat(totalNoteBytes, length - 1, totalNoteBytes.length - 1));
-
-			// 获取频率数据
-			float npitch = getPitch(noteFloatData, audioEvent.getBufferSize());
-
-			LOGGER.info("第{}个音符的样本频率：{} 实际频率：{}", noteAnalysis.getIndex(), musicXmlNote.getFrequency(), npitch);
-
-			// 准备处理下一个音符
-			setProcessingNote(noteAnalysis = new NoteAnalysis(musicXmlNote.getMusicalNotesIndex() + 1,musicXmlNote.getMeasureIndex()));
-		}*/
-	}
 
 	public int evaluateForSection(int sectionIndex, int subjectId){
 
@@ -555,6 +456,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 		}
 		return result;
 	}
+	
 
 	public void evaluateForNote(NoteAnalysis noteAnalysis) {
 
@@ -635,7 +537,6 @@ public class UserChannelContext implements PitchDetectionHandler {
 					.intValue());
 		}
 	}
-	
 	private int computeFrequency(List<ChunkAnalysis> chunkAnalysisList, ChunkAnalysis lastChunkAnalysis, int offsetRange) {
 		
 		List<ChunkAnalysis> chunkList = new ArrayList<ChunkAnalysis>(chunkAnalysisList);
@@ -665,7 +566,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 			return -1;
 		}
 		
-		if(tenutoSize * 100 / chunkAnalysisList.size() >= 50){
+		if(tenutoSize * 100 / chunkAnalysisList.size() > 50){
 			return lastChunkAnalysis.getFrequency();
 		}
 		
@@ -716,7 +617,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 
 		return frequency;
 	}
-
+	
 	private boolean computeTempoWithFrequency(List<ChunkAnalysis> chunkAnalysisList, ChunkAnalysis lastChunkAnalysis){
 		
 		List<ChunkAnalysis> chunkList = new ArrayList<ChunkAnalysis>(chunkAnalysisList);
@@ -782,6 +683,58 @@ public class UserChannelContext implements PitchDetectionHandler {
 		return tempo;
 	}
 	
+	private boolean computeTempoWithAmplitude2(List<ChunkAnalysis> chunkAnalysisList, ChunkAnalysis lastChunkAnalysis) {
+
+		List<Integer> chunkAmplitudeList = chunkAnalysisList.stream().map(ChunkAnalysis::getAmplitude).collect(Collectors.toList());
+
+		if (chunkAmplitudeList.size() <= 3) {
+			return chunkAmplitudeList.stream().filter(t -> t.floatValue() > hardLevel.getAmplitudeThreshold()).count() > 0;
+		}
+		
+		chunkAmplitudeList.add(0, lastChunkAnalysis.getAmplitude());
+		
+		// 检测是否有多个波峰
+		boolean tempo = false;
+		boolean isContinue = true;
+		int firstPeakIndex = -1;
+		int peakSize = 0;
+		for (int i = 1; i < chunkAmplitudeList.size(); i++) {
+			if (chunkAmplitudeList.get(i) > hardLevel.getAmplitudeThreshold() && chunkAmplitudeList.get(i) > chunkAmplitudeList.get(i - 1) + 2) {
+				tempo = true;
+				if(firstPeakIndex == -1){
+					firstPeakIndex = i;
+					peakSize++;
+				}
+				if (isContinue == false) {
+					tempo = false;
+					peakSize++;
+					break;
+				}
+			} else {
+				if (tempo == true) {
+					isContinue = false;
+				}
+			}
+		}
+		
+		if(peakSize == 0){
+			tempo = lastChunkAnalysis.isPeak();
+		}else if(peakSize == 1){
+			tempo = true;
+		}else{
+			tempo = false;
+		}
+		
+		if (tempo) {
+			// 判断进入时间点
+			if((firstPeakIndex - 1) * 100 /chunkAmplitudeList.size() > hardLevel.getTempoOffsetOfPercent()){
+				tempo = false;
+			}
+		}
+		
+		return tempo;
+	}
+	
 	private boolean computeTempoWithAmplitude(List<ChunkAnalysis> chunkAnalysisList, ChunkAnalysis lastChunkAnalysis) {
 
 		boolean tempo = false;
@@ -809,11 +762,13 @@ public class UserChannelContext implements PitchDetectionHandler {
 					}
 				}
 			} else {
-				if (chunkAmplitudeList.get(i - 1) + 2 < chunkAmplitudeList.get(i) && chunkAmplitudeList.get(i) >= chunkAmplitudeList.get(i + 1)) {
-					peakSize++;
-					if (minPeakIndex == -1 || minPeakIndex > i) {
-						minPeakIndex = i;
-					}
+				if (chunkAmplitudeList.get(i - 1) < chunkAmplitudeList.get(i) && chunkAmplitudeList.get(i) >= chunkAmplitudeList.get(i + 1)) {
+					//if(Math.abs(chunkAmplitudeList.get(i - 1) - chunkAmplitudeList.get(i)) > 2 || Math.abs(chunkAmplitudeList.get(i) - chunkAmplitudeList.get(i + 1)) > 2){
+						peakSize++;
+						if (minPeakIndex == -1 || minPeakIndex > i) {
+							minPeakIndex = i;
+						}
+					//}
 				}
 			}
 		}

audio-analysis/src/main/java/com/yonge/nettty/entity/MusicXmlBasicInfo.java

@@ -28,6 +28,8 @@ public class MusicXmlBasicInfo {
 	private String heardLevel;
 
 	private String uuid;
+	
+	private int beatLength;
 
 	private List<MusicXmlNote> musicXmlInfos = new ArrayList<MusicXmlNote>();
 
@@ -113,6 +115,14 @@ public class MusicXmlBasicInfo {
 		this.uuid = uuid;
 	}
 
+	public int getBeatLength() {
+		return beatLength;
+	}
+
+	public void setBeatLength(int beatLength) {
+		this.beatLength = beatLength;
+	}
+
 	public List<MusicXmlNote> getMusicXmlInfos() {
 		return musicXmlInfos;
 	}

audio-analysis/src/main/java/com/yonge/netty/server/messagehandler/BinaryWebSocketFrameHandler.java

@@ -19,9 +19,6 @@ import javax.sound.sampled.AudioFormat;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import be.tarsos.dsp.pitch.PitchProcessor;
-import be.tarsos.dsp.pitch.PitchProcessor.PitchEstimationAlgorithm;
-
 import com.yonge.audio.analysis.AudioFloatConverter;
 import com.yonge.audio.utils.ArrayUtil;
 import com.yonge.nettty.dto.UserChannelContext;
@@ -60,10 +57,6 @@ public class BinaryWebSocketFrameHandler extends SimpleChannelInboundHandler<Bin
 	 * @describe 采样大小
 	 */
 	private int bufferSize = 1024 * 4;
-	/**
-	 * @describe 帧覆盖大小
-	 */
-	private int overlap = 0;
 
 	private boolean signed = true;
 
@@ -73,29 +66,6 @@ public class BinaryWebSocketFrameHandler extends SimpleChannelInboundHandler<Bin
 
 	private AudioFloatConverter converter = AudioFloatConverter.getConverter(audioFormat);
 
-	private PitchEstimationAlgorithm algorithm = PitchProcessor.PitchEstimationAlgorithm.FFT_YIN;
-
-	/**
-	 * @describe 有效分贝大小
-	 */
-	private int validDb = 35;
-	/**
-	 * @describe 有效频率
-	 */
-	private int validFrequency = 20;
-	/**
-	 * @describe 音准前后音分误差范围
-	 */
-	private int intonationCentsRange = 3;
-	/**
-	 * @describe 节奏有效阈值
-	 */
-	private float cadenceValidDuty = 0.09f;
-	/**
-	 * @describe 完整性有效频率误差范围
-	 */
-	private int integrityFrequencyRange = 30;
-
 	private String tmpFileDir = "e:/soundRecords/";
 	
 	private SimpleDateFormat sdf =new SimpleDateFormat("yyMMddHHmmSS");
@@ -117,7 +87,13 @@ public class BinaryWebSocketFrameHandler extends SimpleChannelInboundHandler<Bin
 			if (channelContext == null) {
 				return;
 			}
+			
+			datas = channelContext.skipHeader(datas);
 
+			if (datas.length == 0) {
+				return;
+			}
+			
 			// 写录音文件
 			WaveformWriter waveFileProcessor = channelContext.getWaveFileProcessor();
 			if (waveFileProcessor == null) {

audio-analysis/src/main/java/com/yonge/netty/server/messagehandler/TextWebSocketHandler.java

@@ -87,7 +87,7 @@ public class TextWebSocketHandler extends SimpleChannelInboundHandler<TextWebSoc
 			}
 
 			channelContext.getSongMusicXmlMap().put(musicXmlBasicInfo.getExamSongId(), musicXmlBasicInfo);
-			channelContext.init();
+			channelContext.init(musicXmlBasicInfo.getHeardLevel(), musicXmlBasicInfo.getSubjectId(), musicXmlBasicInfo.getBeatLength());
 
 			userChannelContextService.register(channel, channelContext);
 
@@ -207,6 +207,8 @@ public class TextWebSocketHandler extends SimpleChannelInboundHandler<TextWebSoc
 			
 			break;
 		case "checkSound": // 校音
+			
+			
 
 			break;
 

audio-analysis/src/main/java/com/yonge/netty/server/processor/WaveformWriter.java

@@ -44,13 +44,13 @@ public class WaveformWriter {
 
 	private final String fileName;
 
-	private short channelNum = 1;
+	public static final short CHANNEL_NUM = 1;
 
-	private int sampleRate = 44100;
+	public static final int SAMPLE_RATE = 44100;
 
-	private short bitsPerSample = 16;
+	public static final short BITS_PER_SAMPLE = 16;
 
-	private static final int HEADER_LENGTH = 44;
+	public static final int HEADER_LENGTH = 44;
 
 	public WaveformWriter(String fileName) {
 
@@ -79,7 +79,7 @@ public class WaveformWriter {
 
 	public void processingFinished() {
 		try {
-			WaveHeader waveHeader = new WaveHeader(WaveHeader.FORMAT_PCM, channelNum, sampleRate, bitsPerSample, (int) randomAccessFile.length()
+			WaveHeader waveHeader = new WaveHeader(WaveHeader.FORMAT_PCM, CHANNEL_NUM, SAMPLE_RATE, BITS_PER_SAMPLE, (int) randomAccessFile.length()
 					- HEADER_LENGTH);
 
 			ByteArrayOutputStream header = new ByteArrayOutputStream();