update

audio-analysis/src/main/java/com/yonge/nettty/dto/HardLevelEnum.java

@@ -0,0 +1,67 @@
+package com.yonge.nettty.dto;
+
+import com.ym.mec.common.enums.BaseEnum;
+
+public enum HardLevelEnum implements BaseEnum<String, HardLevelEnum> {
+	BEGINNER("入门级", 2, 10, 40, 60, 10), ADVANCED("进阶级", 2, 10, 40, 60, 10), PERFORMER("大师级", 2, 10, 40, 60, 10);
+
+	private String msg;
+
+	private int amplitudeThreshold;
+
+	private int frequencyOffset;
+
+	private int tempoOffsetOfPercent;
+
+	private int integrityRange;
+
+	private int notPlayRange;
+
+	/**
+	 * 
+	 * @param msg
+	 * @param amplitudeThreshold 振幅阈值
+	 * @param frequencyOffset
+	 * @param tempoOffsetOfPercent
+	 * @param integrityRange
+	 * @param notPlayRange
+	 */
+	HardLevelEnum(String msg, int amplitudeThreshold, int frequencyOffset, int tempoOffsetOfPercent, int integrityRange, int notPlayRange) {
+		this.msg = msg;
+		this.amplitudeThreshold = amplitudeThreshold;
+		this.frequencyOffset = frequencyOffset;
+		this.tempoOffsetOfPercent = tempoOffsetOfPercent;
+		this.integrityRange = integrityRange;
+		this.notPlayRange = notPlayRange;
+	}
+
+	public String getMsg() {
+		return msg;
+	}
+
+	public int getAmplitudeThreshold() {
+		return amplitudeThreshold;
+	}
+
+	public int getFrequencyOffset() {
+		return frequencyOffset;
+	}
+
+	public int getTempoOffsetOfPercent() {
+		return tempoOffsetOfPercent;
+	}
+
+	public int getIntegrityRange() {
+		return integrityRange;
+	}
+
+	public int getNotPlayRange() {
+		return notPlayRange;
+	}
+
+	@Override
+	public String getCode() {
+		return this.name();
+	}
+
+}

audio-analysis/src/main/java/com/yonge/nettty/dto/NoteAnalysis.java

@@ -43,8 +43,6 @@ public class NoteAnalysis {
 
 	private NoteErrorType noteErrorType = NoteErrorType.RIGHT;
 
-	private double playDurationTime;
-	
 	private int score;
 	
 	private int intonationScore;
@@ -151,14 +149,6 @@ public class NoteAnalysis {
 		this.ignore = ignore;
 	}
 
-	public double getPlayDurationTime() {
-		return playDurationTime;
-	}
-
-	public void setPlayDurationTime(double playDurationTime) {
-		this.playDurationTime = playDurationTime;
-	}
-
 	public NoteErrorType getMusicalErrorType() {
 		return noteErrorType;
 	}

audio-analysis/src/main/java/com/yonge/nettty/dto/UserChannelContext.java

@@ -1,12 +1,12 @@
 package com.yonge.nettty.dto;
 
+import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
@@ -54,16 +54,20 @@ public class UserChannelContext implements PitchDetectionHandler {
 	
 	private List<SectionAnalysis> doneSectionAnalysisList = new ArrayList<SectionAnalysis>();
 	
+	private List<ChunkAnalysis> chunkAnalysisList = new ArrayList<ChunkAnalysis>();
+	
 	private byte[] channelBufferBytes = new byte[0];
 	
 	private double playTime;
 	
 	private double receivedTime;
 	
-	private List<ChunkAnalysis> chunkAnalysisList = new ArrayList<ChunkAnalysis>();
-	
 	private ChunkAnalysis lastChunkAnalysis;
 	
+	private List<ChunkAnalysis> lastChunkAnalysisList = new ArrayList<ChunkAnalysis>();
+	
+	private HardLevelEnum hardLevel = HardLevelEnum.ADVANCED;
+	
 	public void init(){
 
 	}
@@ -121,6 +125,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 		playTime = 0;
 		receivedTime = 0;
 		lastChunkAnalysis = null;
+		lastChunkAnalysisList = new ArrayList<ChunkAnalysis>();
 	}
 	
 	public MusicXmlBasicInfo getMusicXmlBasicInfo(Integer songId){
@@ -217,22 +222,6 @@ public class UserChannelContext implements PitchDetectionHandler {
 		return -1;
 	}
 	
-	private List<Integer> getFirstNoteIndexPerSection(Integer songId){
-		
-		List<Integer> result = new ArrayList<Integer>();
-		
-		MusicXmlBasicInfo musicXmlBasicInfo = getMusicXmlBasicInfo(songId);
-		
-		if(musicXmlBasicInfo != null){
-			
-			Map<Integer,List<MusicXmlNote>> map = musicXmlBasicInfo.getMusicXmlInfos().stream().collect(Collectors.groupingBy(MusicXmlNote :: getMeasureIndex));
-			for(Entry<Integer, List<MusicXmlNote>> entry : map.entrySet()){
-				result.add(entry.getValue().stream().map(t -> t.getMusicalNotesIndex()).reduce(Integer :: min).get());
-			}
-		}
-		return result;
-	}
-
 	public byte[] getChannelBufferBytes() {
 		return channelBufferBytes;
 	}
@@ -283,21 +272,32 @@ public class UserChannelContext implements PitchDetectionHandler {
 
 			if (playTime >= (musicXmlNote.getDuration() + musicXmlNote.getTimeStamp())) {
 
+				LOGGER.info("------ Frequency:{}  splDb:{}  Power:{}  amplitude:{} ------", playFrequency, splDb, power, amplitude);
+				
+				//每个音符最后一个块
+				lastChunkAnalysisList.add(new ChunkAnalysis(playTime - durationTime, playTime, playFrequency, splDb, power, amplitude));
+				if(noteAnalysis.getMusicalNotesIndex() > 0){
+					lastChunkAnalysis = lastChunkAnalysisList.get(noteAnalysis.getMusicalNotesIndex() - 1);
+				}
+
 				if (musicXmlNote.getDontEvaluating()) {
 					noteAnalysis.setIgnore(true);
 				}
 				
-				NoteAnalysis lastNoteAnalysis = null;
-				if (doneNoteAnalysisList.size() > 0) {
-					lastNoteAnalysis = doneNoteAnalysisList.get(doneNoteAnalysisList.size() - 1);
+				if(chunkAnalysisList.size() == 0){// 延音线
+					
 				}
-
-				noteAnalysis.setPlayFrequency(computeFrequency(chunkAnalysisList, lastNoteAnalysis, 10));
+				
+				noteAnalysis.setPlayFrequency(computeFrequency(chunkAnalysisList, lastChunkAnalysis, hardLevel.getFrequencyOffset()));
 				
 				//判断节奏(音符持续时间内有不间断的音高，就节奏正确)
 				boolean tempo = true;
 				if (musicXmlNote.getFrequency() == -1) {// 休止符
-					tempo = chunkAnalysisList.stream().filter(t -> t.getAmplitude() > 5).count() == 0;
+					if (subjectId == 23) {
+						tempo = chunkAnalysisList.stream().filter(t -> t.getAmplitude() > hardLevel.getAmplitudeThreshold()).count() <= 0;
+					}else{
+						tempo = chunkAnalysisList.stream().filter(t -> t.getFrequency() > 100).count() <= 1;
+					}
 				} else {
 					if (subjectId == 23) {
 						if (lastChunkAnalysis == null) {
@@ -306,10 +306,12 @@ public class UserChannelContext implements PitchDetectionHandler {
 							tempo = computeTempoWithAmplitude(chunkAnalysisList, lastChunkAnalysis.getAmplitude());
 						}
 					} else {
-						tempo = computeTempoWithFrequency(chunkAnalysisList, lastNoteAnalysis);
+						tempo = computeTempoWithFrequency(chunkAnalysisList, lastChunkAnalysis);
 					}
 				}
 				
+				noteAnalysis.setDurationTime(chunkAnalysisList.stream().mapToDouble(t -> t.getDurationTime()).sum());
+				
 				noteAnalysis.setTempo(tempo);
 				
 				evaluateForNote(noteAnalysis);
@@ -319,7 +321,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 				
 				doneNoteAnalysisList.add(noteAnalysis);
 				
-				lastChunkAnalysis = chunkAnalysisList.get(chunkAnalysisList.size() - 1);
+				//lastChunkAnalysis = chunkAnalysisList.get(chunkAnalysisList.size() - 1);
 				
 				chunkAnalysisList.clear();
 
@@ -338,7 +340,6 @@ public class UserChannelContext implements PitchDetectionHandler {
 				noteAnalysis = nextNoteAnalysis;
 
 			} else {
-				noteAnalysis.setDurationTime(noteAnalysis.getDurationTime() + durationTime);
 				
 				/*double skip = 0;
 				if (firstNoteIndexPerSectionList.contains(noteAnalysis.getMusicalNotesIndex())) {
@@ -348,11 +349,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 				
 				LOGGER.info("Frequency:{}  splDb:{}  Power:{}  amplitude:{}", playFrequency, splDb, power, amplitude);
 				
-				if (playFrequency < 2000 && playFrequency > 100) {
-					noteAnalysis.setPlayDurationTime(noteAnalysis.getPlayDurationTime() + durationTime);
-				}
-				
-				chunkAnalysisList.add(new ChunkAnalysis(receivedTime - durationTime, receivedTime, playFrequency, splDb, power, amplitude));
+				chunkAnalysisList.add(new ChunkAnalysis(playTime - durationTime, playTime, playFrequency, splDb, power, amplitude));
 				
 			}
 
@@ -556,50 +553,60 @@ public class UserChannelContext implements PitchDetectionHandler {
 		return result;
 	}
 
-	public void evaluateForNote(NoteAnalysis noteAnalysis){
+	public void evaluateForNote(NoteAnalysis noteAnalysis) {
+
+		double playDurationTime = 0;
 		
-		if (noteAnalysis.getFrequency() == -1) {// 休止符
+		if (subjectId == 23) {
 			if (!noteAnalysis.isTempo()) {
 				noteAnalysis.setMusicalErrorType(NoteErrorType.CADENCE_WRONG);
-			}else if (1 - noteAnalysis.getPlayDurationTime() / noteAnalysis.getDurationTime() < 0.6) {
-				noteAnalysis.setMusicalErrorType(NoteErrorType.INTEGRITY_WRONG);
-			}  else if (Math.abs(noteAnalysis.getFrequency() - noteAnalysis.getPlayFrequency()) > 10) {
-				noteAnalysis.setMusicalErrorType(NoteErrorType.INTONATION_WRONG);
 			} else {
 				noteAnalysis.setMusicalErrorType(NoteErrorType.RIGHT);
 			}
 		} else {
-			if (subjectId == 23) {
+			if (noteAnalysis.getFrequency() == -1) {// 休止符
+
+				playDurationTime = chunkAnalysisList.stream().filter(t -> t.getFrequency() <= 100).mapToDouble(t -> t.getDurationTime()).sum();
+
 				if (!noteAnalysis.isTempo()) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.CADENCE_WRONG);
+				} else if (playDurationTime * 100 / noteAnalysis.getDurationTime() < hardLevel.getIntegrityRange()) {
+					noteAnalysis.setMusicalErrorType(NoteErrorType.INTEGRITY_WRONG);
+				} else if (Math.abs(noteAnalysis.getFrequency() - noteAnalysis.getPlayFrequency()) > hardLevel.getFrequencyOffset()) {
+					noteAnalysis.setMusicalErrorType(NoteErrorType.INTONATION_WRONG);
 				} else {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.RIGHT);
 				}
 			} else {
-				if (noteAnalysis.getPlayDurationTime() / noteAnalysis.getDurationTime() < 0.1) {
+				playDurationTime = chunkAnalysisList.stream().filter(t -> t.getFrequency() > 100 && t.getFrequency() < 2000)
+						.mapToDouble(t -> t.getDurationTime()).sum();
+
+				if (playDurationTime * 100 / noteAnalysis.getDurationTime() < hardLevel.getNotPlayRange()) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.NOT_PLAY);
-				} else if (noteAnalysis.getPlayDurationTime() / noteAnalysis.getDurationTime() < 0.6) {
+				} else if (playDurationTime * 100 / noteAnalysis.getDurationTime() < hardLevel.getIntegrityRange()) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.INTEGRITY_WRONG);
 				} else if (!noteAnalysis.isTempo()) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.CADENCE_WRONG);
-				} else if (Math.abs(noteAnalysis.getFrequency() - noteAnalysis.getPlayFrequency()) > 10) {
+				} else if (Math.abs(noteAnalysis.getFrequency() - noteAnalysis.getPlayFrequency()) > hardLevel.getFrequencyOffset()) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.INTONATION_WRONG);
 				} else {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.RIGHT);
 				}
 			}
 		}
-		
-		//计算音分
+
+		// 计算音分
 		int tempoScore = 0;
 		int integrityScore = 0;
-		int intonationScore = (int) (100 - Math.abs(YINPitchDetector.hertzToAbsoluteCent(noteAnalysis.getPlayFrequency()) - YINPitchDetector.hertzToAbsoluteCent(noteAnalysis.getFrequency()))*10/17);
-		if (intonationScore < 0){
-            intonationScore = 0;
-        }else if(intonationScore > 100){
-            intonationScore = 100;
-        }
-		
+		int intonationScore = 100 - new BigDecimal(Math.abs(YINPitchDetector.hertzToAbsoluteCent(noteAnalysis.getPlayFrequency())
+				- YINPitchDetector.hertzToAbsoluteCent(noteAnalysis.getFrequency()))).multiply(new BigDecimal(10)).divide(new BigDecimal(17), 2)
+				.setScale(0, BigDecimal.ROUND_UP).intValue();
+		if (intonationScore < 0) {
+			intonationScore = 0;
+		} else if (intonationScore > 100) {
+			intonationScore = 100;
+		}
+
 		if (noteAnalysis.getMusicalErrorType() == NoteErrorType.NOT_PLAY) {
 			intonationScore = 0;
 		} else {
@@ -609,7 +616,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 				noteAnalysis.setTempoScore(tempoScore);
 			}
 
-			double durationPercent = noteAnalysis.getPlayDurationTime() / noteAnalysis.getDurationTime();
+			double durationPercent = playDurationTime / noteAnalysis.getDurationTime();
 			if (durationPercent >= 0.7) {
 				integrityScore = 100;
 			} else if (durationPercent < 0.7 && durationPercent >= 0.5) {
@@ -619,13 +626,14 @@ public class UserChannelContext implements PitchDetectionHandler {
 		}
 		noteAnalysis.setIntonationScore(intonationScore);
 		if (subjectId == 23) {
-			noteAnalysis.setScore((int)tempoScore);
-		}else{
-			noteAnalysis.setScore((int)((intonationScore + tempoScore + integrityScore) / 3));
+			noteAnalysis.setScore(tempoScore);
+		} else {
+			noteAnalysis.setScore(new BigDecimal(intonationScore + tempoScore + integrityScore).divide(new BigDecimal(3), 2).setScale(0, BigDecimal.ROUND_UP)
+					.intValue());
 		}
 	}
 	
-	private double computeFrequency(List<ChunkAnalysis> chunkAnalysisList, NoteAnalysis lastNoteAnalysis, int offsetRange) {
+	private double computeFrequency(List<ChunkAnalysis> chunkAnalysisList, ChunkAnalysis lastChunkAnalysis, int offsetRange) {
 
 		List<Double> chunkFrequencyList = chunkAnalysisList.stream().map(t -> t.getFrequency()).filter(t -> t.doubleValue() > 100 && t.doubleValue() < 2000)
 				.collect(Collectors.toList());
@@ -634,14 +642,20 @@ public class UserChannelContext implements PitchDetectionHandler {
 			return -1;
 		}
 		
-		// 剔除上一个音延续到下一个音
-		double lastFrequency = lastNoteAnalysis.getFrequency();
-		Iterator<Double> iterable = chunkFrequencyList.iterator();
-		while(iterable.hasNext()){
-			if(Math.abs(lastFrequency - iterable.next()) > 10){
-				break;
+		// 剔除上一个音延续下来的信号
+		if (lastChunkAnalysis != null) {
+			double lastFrequency = lastChunkAnalysis.getFrequency();
+			Iterator<Double> iterable = chunkFrequencyList.iterator();
+			while (iterable.hasNext()) {
+				if (Math.abs(lastFrequency - iterable.next()) > offsetRange) {
+					break;
+				}
+				iterable.remove();
+			}
+
+			if (chunkFrequencyList.size() == 0) {
+				return lastFrequency;
 			}
-			iterable.remove();
 		}
 		
 		// 排序
@@ -681,7 +695,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 			}
 		}
 
-		if (chunkFrequencyList.size() < 3 || maxChunkSize * 100 / chunkFrequencyList.size() < 40) {
+		if (chunkFrequencyList.size() < 3) {
 			frequency = chunkFrequencyList.stream().collect(Collectors.summingDouble(t -> t)) / chunkFrequencyList.size();
 		}
 		
@@ -692,17 +706,39 @@ public class UserChannelContext implements PitchDetectionHandler {
 		return frequency;
 	}
 
-	private boolean computeTempoWithFrequency(List<ChunkAnalysis> chunkAnalysisList, NoteAnalysis lastNoteAnalysis){
+	private boolean computeTempoWithFrequency(List<ChunkAnalysis> chunkAnalysisList, ChunkAnalysis lastChunkAnalysis){
+		
+		List<ChunkAnalysis> chunkList = new ArrayList<ChunkAnalysis>(chunkAnalysisList);
+		
+		// 剔除上一个音延续下来的信号
+		if (lastChunkAnalysis != null) {
+			double lastFrequency = lastChunkAnalysis.getFrequency();
+			Iterator<ChunkAnalysis> iterable = chunkList.iterator();
+			while (iterable.hasNext()) {
+				if (Math.abs(lastFrequency - iterable.next().getFrequency()) > hardLevel.getFrequencyOffset()) {
+					break;
+				}
+				iterable.remove();
+			}
+		}
+		
+		if(chunkList.size() == 0){
+			return false;
+		}
+		
 		ChunkAnalysis chunkAnalysis = null;
 		boolean tempo = false;
 		boolean isContinue = true;
-		boolean lastestNotePlayStatus = true;
 		int unplayedSize = 0;
-		for (int i = 0; i < chunkAnalysisList.size(); i++) {
-			chunkAnalysis = chunkAnalysisList.get(i);
+		int firstPeakIndex = -1;
+		for (int i = 0; i < chunkList.size(); i++) {
+			chunkAnalysis = chunkList.get(i);
 			if (chunkAnalysis != null) {
 				if (chunkAnalysis.getFrequency() > 100) {
 					tempo = true;
+					if(firstPeakIndex == -1){
+						firstPeakIndex = i;
+					}
 					if (isContinue == false) {
 						if (chunkAnalysisList.size() < 5) {
 							if (unplayedSize > 0) {
@@ -710,7 +746,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 								break;
 							}
 						} else {
-							if ((unplayedSize * 100 / chunkAnalysisList.size()) > 10 || unplayedSize > 2) {
+							if ((unplayedSize * 100 / chunkAnalysisList.size()) > hardLevel.getNotPlayRange() || unplayedSize > 3) {
 								tempo = false;
 								break;
 							}
@@ -722,29 +758,13 @@ public class UserChannelContext implements PitchDetectionHandler {
 						unplayedSize++;
 					}
 				}
-				if(i == chunkAnalysisList.size() - 1){
-					lastestNotePlayStatus = false;
-				}
 			}
 		}
 		
-		if (tempo && lastestNotePlayStatus) {
-			// 获取上一个音符信息
-			if (lastNoteAnalysis != null) {
-				double pf = lastNoteAnalysis.getPlayFrequency();
-				int continueSize = 0;
-				for (int i = 0; i < chunkAnalysisList.size(); i++) {
-					chunkAnalysis = chunkAnalysisList.get(i);
-					if (chunkAnalysis != null) {
-						if (Math.abs(chunkAnalysis.getFrequency() - pf) > 10) {
-							break;
-						}
-						continueSize++;
-					}
-				}
-				if(continueSize * 100 / chunkAnalysisList.size() > 40){
-					tempo = false;
-				}
+		if (tempo) {
+			// 判断进入时间点
+			if((chunkAnalysisList.size() - chunkList.size() + firstPeakIndex + 1) * 100 /chunkAnalysisList.size() > hardLevel.getTempoOffsetOfPercent()){
+				tempo = false;
 			}
 		}
 		
@@ -757,15 +777,18 @@ public class UserChannelContext implements PitchDetectionHandler {
 
 		List<Float> chunkAmplitudeList = chunkAnalysisList.stream().map(ChunkAnalysis::getAmplitude).collect(Collectors.toList());
 
+		// 剔除余波
 		if (chunkAmplitudeList.size() < 3) {
-			return chunkAmplitudeList.stream().filter(t -> t.floatValue() > 3).count() > 0;
+			return chunkAmplitudeList.stream().filter(t -> t.floatValue() > hardLevel.getAmplitudeThreshold()).count() > 0;
 		}
 
 		// 检测是否有多个波峰
 		int peakSize = 0;
 		int minPeakIndex = -1;
+		int notPlaySize = 0;
 		for (int i = 0; i < chunkAmplitudeList.size(); i++) {
-			if (chunkAmplitudeList.get(i) < 3) {
+			if (chunkAmplitudeList.get(i) < 2) {
+				notPlaySize++;
 				continue;
 			}
 			if (i == 0) {
@@ -790,11 +813,15 @@ public class UserChannelContext implements PitchDetectionHandler {
 			}
 		}
 
-		tempo = peakSize <= 1;
+		if(notPlaySize == chunkAmplitudeList.size()){
+			tempo = false;
+		}else{
+			tempo = peakSize <= 1;
+		}
 
 		// 检测是否延迟进入
 		if (tempo == true) {
-			if ((minPeakIndex + 1) * 100 / chunkAmplitudeList.size() > 40 && chunkAmplitudeList.size() > 3) {
+			if ((minPeakIndex + 1) * 100 / chunkAmplitudeList.size() > hardLevel.getTempoOffsetOfPercent() && chunkAmplitudeList.size() > 3) {
 				tempo = false;
 			}
 		}
@@ -802,31 +829,4 @@ public class UserChannelContext implements PitchDetectionHandler {
 		return tempo;
 	}
 	
-	private boolean getPeak(List<Float> chunkList, float last){
-		
-		if(chunkList.size()<3){
-			return true;
-		}
-		
-		int peakSize = 0;
-		for(int i = 0;i<chunkList.size() - 1;i++){
-			if(i == 0){
-				if(chunkList.get(i) > last){
-					peakSize++;
-				}
-			}
-			else if(chunkList.get(i-1) < chunkList.get(i) && chunkList.get(i) >= chunkList.get(i + 1)){
-				peakSize++;
-			}
-			
-			if(i == chunkList.size() - 2){
-				if(chunkList.get(i) < chunkList.get(i + 1)){
-					peakSize++;
-				}
-			}
-		}
-		
-		return peakSize == 1;
-	}
-	
 }