Merge branch 'audio_20220718_cadence' into online

audio-analysis/src/main/java/com/yonge/Main.java

@@ -12,16 +12,18 @@ import be.tarsos.dsp.pitch.PitchProcessor;
 import com.yonge.audio.analysis.AudioFloatConverter;
 import com.yonge.audio.analysis.detector.YINPitchDetector;
 import com.yonge.audio.utils.ArrayUtil;
+import com.yonge.netty.server.processor.WaveformWriter;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.ArrayUtils;
 
 import javax.sound.sampled.AudioFormat;
 import javax.sound.sampled.AudioInputStream;
 import javax.sound.sampled.AudioSystem;
 import javax.sound.sampled.UnsupportedAudioFileException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
+import java.io.*;
 import java.net.URL;
 import java.util.Arrays;
+import java.util.Date;
 
 /**
  * Description
@@ -44,32 +46,72 @@ public class Main {
     public static void main(String[] args){
         try{
             float sampleRate = 44100;
-            int audioBufferSize = 2048;
+            int audioBufferSize = 1024 *2;
             int bufferOverlap = 0;
             AudioFloatConverter converter = AudioFloatConverter.getConverter(audioFormat);
             //Create an AudioInputStream from my .wav file
-            URL soundURL = Main.class.getResource("/300.wav");
+            URL soundURL = Main.class.getResource("/WAV.wav");
             AudioInputStream stream = AudioSystem.getAudioInputStream(soundURL);
-            final MFCC mfccProcessor = new MFCC(audioBufferSize, stream.getFormat().getSampleRate(),
-                                                amountOfCepstrumCoef, amountOfMelFilters, lowerFilterFreq, upperFilterFreq);
+            // final MFCC mfccProcessor = new MFCC(audioBufferSize, stream.getFormat().getSampleRate(),
+            //                                     amountOfCepstrumCoef, amountOfMelFilters, lowerFilterFreq, upperFilterFreq);
 
-            FastYin detector = new FastYin(sampleRate, audioBufferSize );
+            FastYin detector = new FastYin(sampleRate, audioBufferSize *2);
             byte[] bytes = IOUtils.toByteArray(stream);
             AudioFormat format = stream.getFormat();
 
             int b = 0;
             int frequency = 0;
-            while (bytes.length > 2048 *2) {
+            File file = new File("D:\\project\\cooleshow\\audio-analysis\\target\\wav1.wav");
 
-                byte[] bufferData = ArrayUtil.extractByte(bytes, 0, 2048*2 - 1);
+            WaveformWriter waveFileProcessor = new WaveformWriter(file.getAbsolutePath());
+            byte[] bytes1 = new byte[0];
+            // for (int i = 0; i < bytes.length; i++) {
+            //     if (i%2 ==1) {
+            //         System.out.println(bytes[i] + "----------" + bytes[i-1]);
+            //     }
+            // }
+            while (bytes.length > audioBufferSize *2) {
+                byte[] bufferData = ArrayUtil.extractByte(bytes, 0, audioBufferSize*2 - 1);
+
+                bytes1 = ArrayUtil.mergeByte(bytes1, bufferData);
+                byte[] bytes2 = new byte[bytes1.length *2];
+                for (int i = 0; i < bytes1.length; i =i+2) {
+
+                    bytes2[(i+1) *2]  = bytes2[i*2] = bytes1[i];
+                    bytes2[(i+1) *2 +1] = bytes2[i*2 + 1] = bytes1[i +1];
+
+                }
+                // byte ff = bytes1[bytes1.length -1];
+                // for (int start = 0, end = bytes1.length - 2; start < end; start++, end--) {
+                //     byte temp = bytes1[end];
+                //     bytes1[end] = bytes1[start];
+                //     bytes1[start] = temp;
+                // }
+                // bytes1[bytes1.length -1] = ff;
+                //
+                // bytes1 = ArrayUtil.mergeByte(bufferData, bytes1);
+                if (bytes2.length == audioBufferSize *4) {
+                    waveFileProcessor.process(bytes2);
+
+
+                    float[] sampleFloats = new float[audioBufferSize *2];
+
+                    converter.toFloatArray(bytes2, sampleFloats);
+                    int playFrequency = (int) detector.getPitch(sampleFloats).getPitch();
+                    if (playFrequency != -1) {
+                        System.out.println("play frequency is " + playFrequency);
+                    }
+                    bytes1 = new byte[0];
+                }
 
-                float[] sampleFloats = new float[1024*2];
 
-                converter.toFloatArray(bufferData, sampleFloats);
-                int playFrequency = (int)detector.getPitch(sampleFloats).getPitch();
-                System.out.println("play frequency is " +playFrequency);
+                // YINPitchDetector frequencyDetector = new YINPitchDetector(sampleFloats.length, audioFormat.getSampleRate());
+                //
+                // playFrequency = (int) frequencyDetector.getFrequency(sampleFloats);
+                //
+                // System.out.println("frequencyDetector play frequency is " + playFrequency);
                 // ArrayUtil.extractByte(channelContext.getChannelBufferBytes(), bufferSize, totalLength - 1)
-                bytes  = ArrayUtil.extractByte(bytes, 2048*2, bytes.length - 1);
+                bytes  = ArrayUtil.extractByte(bytes, audioBufferSize, bytes.length - 1);
                 // if (b == 1) {
                 //     frequency += playFrequency;
                 //     System.out.println("play frequency is " +frequency/2);
@@ -80,6 +122,7 @@ public class Main {
                 //     b ++;
                 // }
             }
+            waveFileProcessor.processingFinished();
 
 
             //Convert into TarsosDSP API

audio-analysis/src/main/java/com/yonge/netty/dto/NoteAnalysis.java

@@ -5,7 +5,11 @@ import com.yonge.toolset.base.enums.BaseEnum;
 public class NoteAnalysis {
 
 	public enum NoteErrorType implements BaseEnum<String, NoteErrorType> {
-		RIGHT("演奏正确"), CADENCE_WRONG("节奏错误"), INTONATION_WRONG("音准错误"), INTEGRITY_WRONG("完整度不足"), NOT_PLAY("未演奏");
+		RIGHT("演奏正确"), CADENCE_WRONG("节奏错误"), INTONATION_WRONG("音准错误"), INTEGRITY_WRONG("完整度不足"), NOT_PLAY("未演奏"),
+
+		CADENCE_FAST("节奏过快"),CADENCE_SLOW("节奏过慢"),
+		INTONATION_HIGH("音准过高"),INTONATION_LOW("音准过低"),
+		;
 
 		private String msg;
 
@@ -37,7 +41,7 @@ public class NoteAnalysis {
 
 	private int playFrequency = -1;
 
-	private boolean tempo = true;
+	private int tempo = 0;
 
 	private NoteErrorType noteErrorType = NoteErrorType.RIGHT;
 
@@ -116,11 +120,11 @@ public class NoteAnalysis {
 		this.frequency = frequency;
 	}
 
-	public boolean isTempo() {
+	public int getTempo() {
 		return tempo;
 	}
 
-	public void setTempo(boolean tempo) {
+	public void setTempo(int tempo) {
 		this.tempo = tempo;
 	}
 

audio-analysis/src/main/java/com/yonge/netty/dto/NotePlayResult.java

@@ -2,7 +2,8 @@ package com.yonge.netty.dto;
 
 public class NotePlayResult {
 
-	private boolean status;
+	// 0:正确 1：音准过低 2：音准过高
+	private int status;
 	
 	private double migrationRate;
 	
@@ -10,11 +11,11 @@ public class NotePlayResult {
 		// TODO Auto-generated constructor stub
 	}
 
-	public boolean getStatus() {
+	public int getStatus() {
 		return status;
 	}
 
-	public void setStatus(boolean status) {
+	public void setStatus(int status) {
 		this.status = status;
 	}
 

audio-analysis/src/main/java/com/yonge/netty/dto/UserChannelContext.java

@@ -15,6 +15,7 @@ import java.util.stream.Collectors;
 
 import javax.sound.sampled.AudioFormat;
 
+import com.yonge.audio.analysis.AudioFloatConverter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -27,6 +28,7 @@ import com.yonge.netty.entity.MusicXmlBasicInfo;
 import com.yonge.netty.entity.MusicXmlNote;
 import com.yonge.netty.entity.MusicXmlSection;
 import com.yonge.netty.server.processor.WaveformWriter;
+import org.springframework.util.CollectionUtils;
 
 /**
  * 用户通道上下文
@@ -89,20 +91,22 @@ public class UserChannelContext {
 
 		NotePlayResult result = new NotePlayResult();
 
-		boolean status = false;
+		int status;
 		double migrationRate = 0;
 
 		if (Math.round(xmlNote.getFrequency()) == Math.round(playFrequency)) {
-			status = true;
+			status = 0;
 			migrationRate = 0;
 		} else {
 			NoteFrequencyRange noteFrequencyRange = new NoteFrequencyRange(standardFrequecy, xmlNote.getFrequency());
 
-			if (noteFrequencyRange.getMinFrequency() > playFrequency || playFrequency > noteFrequencyRange.getMaxFrequency()) {
-				status = false;
+			if (noteFrequencyRange.getMinFrequency() > playFrequency ) {
+				status = 1;
+			} else if( playFrequency > noteFrequencyRange.getMaxFrequency()){
+				status = 2;
 			} else {
 
-				status = true;
+				status = 0;
 
 				if (Math.round(playFrequency) < Math.round(xmlNote.getFrequency())) {
 					double min = Math.abs(xmlNote.getFrequency() - noteFrequencyRange.getMinFrequency()) / 2;
@@ -394,7 +398,8 @@ public class UserChannelContext {
 				}
 				
 				//判断节奏(音符持续时间内有不间断的音高，就节奏正确)
-				boolean tempo = true;
+				// 节奏  0：正常  1：错误 2：节奏慢 3：节奏快
+				int tempo = 0;
 				if (percussionList.contains(subjectId)) {
 					noteAnalysis.setPlayFrequency(-1);
 					tempo = computeTempoWithAmplitude2(musicXmlNote);
@@ -408,7 +413,7 @@ public class UserChannelContext {
 				evaluateForNote(musicXmlNote, noteAnalysis);//对当前音符评分
 
 				LOGGER.debug("当前音符下标[{}] 预计频率：{} 实际频率：{} 节奏：{}", noteAnalysis.getMusicalNotesIndex(), musicXmlNote.getFrequency(), noteAnalysis.getPlayFrequency(),
-						noteAnalysis.isTempo());
+						noteAnalysis.getTempo());
 				
 				doneNoteAnalysisList.add(noteAnalysis);
 				
@@ -540,9 +545,9 @@ public class UserChannelContext {
 		double playDurationTime = 0;
 		
 		if (percussionList.contains(subjectId)) {
-			if (noteAnalysis.getFrequency() == -1) {// 休止符
-				if (!noteAnalysis.isTempo()) {
-					noteAnalysis.setMusicalErrorType(NoteErrorType.CADENCE_WRONG);
+			if (noteAnalysis.getFrequency() == -1) { // 休止符
+				if (noteAnalysis.getTempo() != 0 ) {
+					noteAnalysis.setMusicalErrorType(setMusicalErrorTempo(noteAnalysis.getTempo()));
 				} else {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.RIGHT);
 				}
@@ -550,8 +555,8 @@ public class UserChannelContext {
 				int beatTimes = (int) chunkAnalysisList.stream().filter(t -> t.getAmplitude() > hardLevel.getAmplitudeThreshold()).count();
 				if(beatTimes == 0){
 					noteAnalysis.setMusicalErrorType(NoteErrorType.NOT_PLAY);
-				}else if (!noteAnalysis.isTempo()) {
-					noteAnalysis.setMusicalErrorType(NoteErrorType.CADENCE_WRONG);
+				}else if (noteAnalysis.getTempo() != 0) {
+					noteAnalysis.setMusicalErrorType(setMusicalErrorTempo(noteAnalysis.getTempo()));
 				} else {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.RIGHT);
 				}
@@ -564,12 +569,12 @@ public class UserChannelContext {
 
 				playDurationTime = chunkAnalysisList.stream().filter(t -> t.getFrequency() <= MIN_FREQUECY).mapToDouble(t -> t.getDurationTime()).sum();
 
-				if (!noteAnalysis.isTempo()) {
-					noteAnalysis.setMusicalErrorType(NoteErrorType.CADENCE_WRONG);
+				if (noteAnalysis.getTempo() != 0) {
+					noteAnalysis.setMusicalErrorType(setMusicalErrorTempo(noteAnalysis.getTempo()));
 				} else if (playDurationTime * 100 / durationTime < hardLevel.getIntegrityRange()) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.INTEGRITY_WRONG);
-				} else if (notePlayResult.getStatus() == false) {
-					noteAnalysis.setMusicalErrorType(NoteErrorType.INTONATION_WRONG);
+				} else if (notePlayResult.getStatus() != 0) {
+					noteAnalysis.setMusicalErrorType(setMusicalErrorStatus(notePlayResult.getStatus()));
 				} else {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.RIGHT);
 				}
@@ -583,10 +588,10 @@ public class UserChannelContext {
 				} else if (playDurationTime * 100 / durationTime < hardLevel.getIntegrityRange()) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.INTEGRITY_WRONG);
 					LOGGER.debug("完整度不足：{}", playDurationTime * 100 / durationTime);
-				} else if (!noteAnalysis.isTempo()) {
-					noteAnalysis.setMusicalErrorType(NoteErrorType.CADENCE_WRONG);
-				} else if (notePlayResult.getStatus() == false) {
-					noteAnalysis.setMusicalErrorType(NoteErrorType.INTONATION_WRONG);
+				} else if (noteAnalysis.getTempo() != 0) {
+					noteAnalysis.setMusicalErrorType(setMusicalErrorTempo(noteAnalysis.getTempo()));
+				} else if (notePlayResult.getStatus() != 0) {
+					noteAnalysis.setMusicalErrorType(setMusicalErrorStatus(notePlayResult.getStatus()));
 				} else {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.RIGHT);
 				}
@@ -610,7 +615,7 @@ public class UserChannelContext {
 			intonationScore = 0;
 		} else {
 
-			if (noteAnalysis.isTempo()) {
+			if (noteAnalysis.getTempo() == 0) {
 				tempoScore = 100;
 				noteAnalysis.setTempoScore(tempoScore);
 			}
@@ -629,7 +634,31 @@ public class UserChannelContext {
 					.intValue());
 		}
 	}
-	
+
+	// 设置 音高
+	private NoteErrorType setMusicalErrorStatus(int status) {
+		if (status == 1) {
+			return NoteErrorType.INTONATION_LOW;
+		} else if (status ==2) {
+			return NoteErrorType.INTONATION_HIGH;
+		}
+		return null;
+	}
+
+	// 设置节奏
+	private NoteErrorType setMusicalErrorTempo(int tempo) {
+		if (tempo == 1) {
+			return NoteErrorType.CADENCE_WRONG;
+		} else if (tempo == 2) {
+			return NoteErrorType.CADENCE_SLOW;
+		} else if (tempo ==3) {
+
+			return NoteErrorType.CADENCE_FAST;
+		} else {
+			return NoteErrorType.RIGHT;
+		}
+	}
+
 	private int computeFrequency(MusicXmlNote musicXmlNote) {
 		
 		double floatingRange = musicXmlNote.getDuration() * hardLevel.getTempoEffectiveRange(musicXmlNote.getDenominator()) / 100;
@@ -680,7 +709,7 @@ public class UserChannelContext {
 	 * @param musicXmlNote
 	 * @return
 	 */
-	private boolean computeTempoWithFrequency(MusicXmlNote musicXmlNote){
+	private int computeTempoWithFrequency(MusicXmlNote musicXmlNote){
 		
 		double floatingRange = musicXmlNote.getDuration() * hardLevel.getTempoEffectiveRange(musicXmlNote.getDenominator()) / 100;
 		
@@ -702,11 +731,11 @@ public class UserChannelContext {
 		List<ChunkAnalysis> chunkList = chunkAnalysisList.subList(startIndex, elementSize + startIndex);
 		
 		if(chunkList == null || chunkList.size() == 0){
-			return false;
+			return 1;
 		}
 		
 		if (musicXmlNote.getFrequency() == -1) {// 休止符
-			return chunkList.stream().filter(t -> t.getFrequency() > MIN_FREQUECY).count() <= 1;
+			return chunkList.stream().filter(t -> t.getFrequency() > MIN_FREQUECY).count() <= 1? 0:1;
 		}
 
 		ChunkAnalysis firstChunkAnalysis = chunkAnalysisList.get(0);
@@ -742,7 +771,7 @@ public class UserChannelContext {
 		
 		NoteFrequencyRange noteFrequencyRange = null;
 		ChunkAnalysis chunkAnalysis = null;
-		boolean tempo = true;
+		int tempo = 0;
 		//boolean isContinue = true;
 		//int unplayedSize = 0;
 		int firstPeakIndex = -1;
@@ -784,7 +813,7 @@ public class UserChannelContext {
 		}
 		
 		if (maxTimes * 100 / totalTimes < hardLevel.getIntegrityRange()) {
-			tempo = false;
+			tempo = 1;
 			LOGGER.debug("节奏错误原因：信号分堆后的最大数量不足指定的完成比例");
 		}
 		
@@ -827,16 +856,17 @@ public class UserChannelContext {
 		}
 		*/
 		
-		if (tempo) {
+		if (tempo == 0) {
 			// 判断进入时间点
 			if(firstPeakIndex * 100 /chunkList.size() > hardLevel.getTempoEffectiveRange(musicXmlNote.getDenominator())){
-				tempo = false;
+				// 节奏慢
+				tempo = 2;
 				LOGGER.debug("节奏错误原因：进入时间点太晚");
 			}else{
 				//判断是否与上一个音延续下来的
 				if(firstChunkAnalysis.getFrequency() > MIN_FREQUECY && lastChunkAnalysis.getFrequency() > MIN_FREQUECY){
-					tempo = new NoteFrequencyRange(standardFrequecy, firstChunkAnalysis.getFrequency()).equals(new NoteFrequencyRange(standardFrequecy, lastChunkAnalysis.getFrequency())) == false;
-					if(tempo == false){
+					tempo = new NoteFrequencyRange(standardFrequecy, firstChunkAnalysis.getFrequency()).equals(new NoteFrequencyRange(standardFrequecy, lastChunkAnalysis.getFrequency())) == false?0:1;
+					if(tempo == 1){
 						LOGGER.debug("节奏错误原因：上一个音[{}]延续下来导致的", lastChunkAnalysis.getFrequency());
 					}
 				}
@@ -846,7 +876,7 @@ public class UserChannelContext {
 		return tempo;
 	}
 	
-	private boolean computeTempoWithAmplitude2(MusicXmlNote musicXmlNote) {
+	private int computeTempoWithAmplitude2(MusicXmlNote musicXmlNote) {
 
 		double floatingRange = musicXmlNote.getDuration() * hardLevel.getTempoEffectiveRange(musicXmlNote.getDenominator()) / 100;
 		
@@ -867,7 +897,7 @@ public class UserChannelContext {
 		List<ChunkAnalysis> chunkList = chunkAnalysisList.subList(0, elementSize);
 		
 		if(chunkList == null || chunkList.size() == 0){
-			return false;
+			return 1;
 		}
 		
 		ChunkAnalysis firstChunkAnalysis = chunkAnalysisList.get(0);
@@ -878,7 +908,7 @@ public class UserChannelContext {
 		if (musicXmlNote.getFrequency() == -1) {// 休止符
 			
 			LOGGER.debug("--Amplitude:{}  Denominator:{}",chunkList.stream().map(t -> t).collect(Collectors.toList()), musicXmlNote.getDenominator());
-			return chunkList.stream().filter(t -> t.getAmplitude() > hardLevel.getAmplitudeThreshold()).count() <= 0;
+			return chunkList.stream().filter(t -> t.getAmplitude() > hardLevel.getAmplitudeThreshold()).count() <= 0 ? 0:1;
 		}
 		
 		Optional<ChunkAnalysis> chunkAnalysisOptional = totalChunkAnalysisList.stream().filter(t -> Double.doubleToLongBits(t.getEndTime()) < Double.doubleToLongBits(firstChunkAnalysis.getStartTime())).findFirst();
@@ -898,7 +928,7 @@ public class UserChannelContext {
 		LOGGER.debug("--Amplitude:{}  Denominator:{}",chunkAmplitudeList.stream().map(t -> t).collect(Collectors.toList()), musicXmlNote.getDenominator());
 		
 		// 检测是否有多个波峰
-		boolean tempo = false;
+		int tempo = 1;
 		boolean isContinue = true;
 		int firstPeakIndex = -1;
 		int peakSize = 0;
@@ -907,36 +937,36 @@ public class UserChannelContext {
 				continue;
 			}
 			if (chunkAmplitudeList.get(i) > hardLevel.getAmplitudeThreshold() && chunkAmplitudeList.get(i) > chunkAmplitudeList.get(i - 1)) {
-				tempo = true;
+				tempo = 0;
 				if(firstPeakIndex == -1){
 					firstPeakIndex = i;
 					peakSize++;
 				}
 				if (isContinue == false) {
-					tempo = false;
+					tempo = 1;
 					peakSize++;
 					break;
 				}
 			} else {
-				if (tempo == true) {
+				if (tempo == 0) {
 					isContinue = false;
 				}
 			}
 		}
 		
 		if(peakSize == 0){
-			tempo = lastChunkAnalysis.isPeak();
+			tempo = lastChunkAnalysis.isPeak() ?0:1;
 		}else if(peakSize == 1){
-			tempo = true;
+			tempo = 0;
 		}else{
-			tempo = false;
+			tempo = 1;
 		}
 		
-		if (tempo) {
+		if (tempo == 0) {
 			// 判断进入时间点
 			if((firstPeakIndex - 1) * 100 /chunkAmplitudeList.size() > hardLevel.getTempoEffectiveRange(musicXmlNote.getDenominator()) * 2){
 				LOGGER.debug("超过范围：{}", (firstPeakIndex - 1) * 100 /chunkAmplitudeList.size());
-				tempo = false;
+				tempo = 1;
 			}
 		}