update

audio-analysis/src/main/java/com/yonge/nettty/dto/ChunkAnalysis.java

@@ -8,15 +8,17 @@ public class ChunkAnalysis {
 
 	private double durationTime;
 
-	private double frequency;
+	private int frequency;
 
-	private double splDb;
+	private int splDb;
 
-	private float power;
+	private int power;
 	
-	private float amplitude;
+	private int amplitude;
+	
+	private boolean isPeak;
 
-	public ChunkAnalysis(double startTime, double endTime, double frequency, double splDb, float power, float amplitude) {
+	public ChunkAnalysis(double startTime, double endTime, int frequency, int splDb, int power, int amplitude) {
 		this.startTime = startTime;
 		this.endTime = endTime;
 		this.frequency = frequency;
@@ -26,7 +28,7 @@ public class ChunkAnalysis {
 		this.durationTime = endTime - startTime;
 	}
 
-	public ChunkAnalysis(double frequency, double splDb, float power) {
+	public ChunkAnalysis(int frequency, int splDb, int power) {
 		this.frequency = frequency;
 		this.splDb = splDb;
 		this.power = power;
@@ -56,35 +58,43 @@ public class ChunkAnalysis {
 		this.durationTime = durationTime;
 	}
 
-	public double getFrequency() {
+	public int getFrequency() {
 		return frequency;
 	}
 
-	public void setFrequency(double frequency) {
+	public void setFrequency(int frequency) {
 		this.frequency = frequency;
 	}
 
-	public double getSplDb() {
+	public int getSplDb() {
 		return splDb;
 	}
 
-	public void setSplDb(double splDb) {
+	public void setSplDb(int splDb) {
 		this.splDb = splDb;
 	}
 
-	public float getPower() {
+	public int getPower() {
 		return power;
 	}
 
-	public void setPower(float power) {
+	public void setPower(int power) {
 		this.power = power;
 	}
 
-	public float getAmplitude() {
+	public int getAmplitude() {
 		return amplitude;
 	}
 
-	public void setAmplitude(float amplitude) {
+	public void setAmplitude(int amplitude) {
 		this.amplitude = amplitude;
 	}
+
+	public boolean isPeak() {
+		return isPeak;
+	}
+
+	public void setPeak(boolean isPeak) {
+		this.isPeak = isPeak;
+	}
 }

audio-analysis/src/main/java/com/yonge/nettty/dto/HardLevelEnum.java

@@ -3,7 +3,7 @@ package com.yonge.nettty.dto;
 import com.ym.mec.common.enums.BaseEnum;
 
 public enum HardLevelEnum implements BaseEnum<String, HardLevelEnum> {
-	BEGINNER("入门级", 2, 10, 40, 60, 10), ADVANCED("进阶级", 2, 10, 40, 60, 10), PERFORMER("大师级", 2, 10, 40, 60, 10);
+	BEGINNER("入门级", 2, 10, 40, 60, 10), ADVANCED("进阶级", 5, 5, 50, 60, 10), PERFORMER("大师级", 2, 10, 40, 60, 10);
 
 	private String msg;
 

audio-analysis/src/main/java/com/yonge/nettty/dto/NoteAnalysis.java

@@ -35,9 +35,9 @@ public class NoteAnalysis {
 
 	private double durationTime;
 
-	private double frequency;
+	private int frequency;
 
-	private double playFrequency = -1;
+	private int playFrequency = -1;
 
 	private boolean tempo = true;
 
@@ -55,14 +55,14 @@ public class NoteAnalysis {
 	
 	private ChunkAnalysis lastChunkAnalysis;
 
-	public NoteAnalysis(int index, int sectionIndex, double frequency, double standardDurationTime) {
+	public NoteAnalysis(int index, int sectionIndex, int frequency, double standardDurationTime) {
 		this.standardDurationTime = standardDurationTime;
 		this.index = index;
 		this.sectionIndex = sectionIndex;
 		this.frequency = frequency;
 	}
 
-	public NoteAnalysis(double startTime, double endTime, double playFrequency) {
+	public NoteAnalysis(double startTime, double endTime, int playFrequency) {
 		this.startTime = startTime;
 		this.endTime = endTime;
 		this.durationTime = endTime - startTime;
@@ -113,15 +113,15 @@ public class NoteAnalysis {
 		return playFrequency;
 	}
 
-	public void setPlayFrequency(double playFrequency) {
+	public void setPlayFrequency(int playFrequency) {
 		this.playFrequency = playFrequency;
 	}
 
-	public double getFrequency() {
+	public int getFrequency() {
 		return frequency;
 	}
 
-	public void setFrequency(double frequency) {
+	public void setFrequency(int frequency) {
 		this.frequency = frequency;
 	}
 

audio-analysis/src/main/java/com/yonge/nettty/dto/UserChannelContext.java

@@ -62,8 +62,6 @@ public class UserChannelContext implements PitchDetectionHandler {
 	
 	private double receivedTime;
 	
-	private ChunkAnalysis lastChunkAnalysis;
-	
 	private List<ChunkAnalysis> lastChunkAnalysisList = new ArrayList<ChunkAnalysis>();
 	
 	private HardLevelEnum hardLevel = HardLevelEnum.ADVANCED;
@@ -124,7 +122,6 @@ public class UserChannelContext implements PitchDetectionHandler {
 		subjectId = null;
 		playTime = 0;
 		receivedTime = 0;
-		lastChunkAnalysis = null;
 		lastChunkAnalysisList = new ArrayList<ChunkAnalysis>();
 	}
 	
@@ -238,10 +235,10 @@ public class UserChannelContext implements PitchDetectionHandler {
 		
 		YINPitchDetector frequencyDetector = new YINPitchDetector(samples.length , audioFormat.getSampleRate());
 
-		double playFrequency = frequencyDetector.getFrequency(samples);
-		double splDb = Signals.soundPressureLevel(samples);
-		float power = Signals.power(samples);
-		float amplitude = Signals.norm(samples);
+		int playFrequency = (int) frequencyDetector.getFrequency(samples);
+		int splDb = (int) Signals.soundPressureLevel(samples);
+		int power = (int) Signals.power(samples);
+		int amplitude = (int) Signals.norm(samples);
 		
 		double durationTime = 1000 * (samples.length * 2) / audioFormat.getSampleRate() / (audioFormat.getSampleSizeInBits() / 8);
 		
@@ -263,7 +260,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 		//取出当前处理中的音符信息
 		NoteAnalysis noteAnalysis = getProcessingNote();
 		if(noteAnalysis == null || noteAnalysis.getDurationTime() == 0) {
-			noteAnalysis = new NoteAnalysis(musicXmlNote.getMusicalNotesIndex(), musicXmlNote.getMeasureIndex(), musicXmlNote.getFrequency(), musicXmlNote.getDuration());
+			noteAnalysis = new NoteAnalysis(musicXmlNote.getMusicalNotesIndex(), musicXmlNote.getMeasureIndex(), (int)musicXmlNote.getFrequency(), musicXmlNote.getDuration());
 		}
 		
 		evaluatingSectionIndex.set(noteAnalysis.getSectionIndex());
@@ -274,10 +271,20 @@ public class UserChannelContext implements PitchDetectionHandler {
 
 				LOGGER.info("------ Frequency:{}  splDb:{}  Power:{}  amplitude:{} ------", playFrequency, splDb, power, amplitude);
 				
+				ChunkAnalysis lastChunkAnalysis = new ChunkAnalysis(playTime - durationTime, playTime, playFrequency, splDb, power, amplitude);
+				if(Math.abs(chunkAnalysisList.get(chunkAnalysisList.size() - 1).getFrequency() - lastChunkAnalysis.getFrequency()) > hardLevel.getFrequencyOffset()){
+					lastChunkAnalysis.setFrequency(-1);
+				}
+				if(chunkAnalysisList.get(chunkAnalysisList.size() - 1).getAmplitude() < lastChunkAnalysis.getAmplitude()){
+					lastChunkAnalysis.setPeak(true);
+				}
+				
 				//每个音符最后一个块
-				lastChunkAnalysisList.add(new ChunkAnalysis(playTime - durationTime, playTime, playFrequency, splDb, power, amplitude));
+				lastChunkAnalysisList.add(lastChunkAnalysis);
 				if(noteAnalysis.getMusicalNotesIndex() > 0){
 					lastChunkAnalysis = lastChunkAnalysisList.get(noteAnalysis.getMusicalNotesIndex() - 1);
+				}else{
+					lastChunkAnalysis = new ChunkAnalysis(0, 0, -1, 0, 0, 0);
 				}
 
 				if (musicXmlNote.getDontEvaluating()) {
@@ -300,11 +307,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 					}
 				} else {
 					if (subjectId == 23) {
-						if (lastChunkAnalysis == null) {
-							tempo = computeTempoWithAmplitude(chunkAnalysisList, 0);
-						} else {
-							tempo = computeTempoWithAmplitude(chunkAnalysisList, lastChunkAnalysis.getAmplitude());
-						}
+						tempo = computeTempoWithAmplitude(chunkAnalysisList, lastChunkAnalysis);
 					} else {
 						tempo = computeTempoWithFrequency(chunkAnalysisList, lastChunkAnalysis);
 					}
@@ -335,7 +338,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 					standDuration = nextMusicXmlNote.getDuration();
 				}
 				
-				NoteAnalysis nextNoteAnalysis = new NoteAnalysis(nextNoteIndex, getMusicSectionIndex(null, nextNoteIndex), nextNoteFrequence, standDuration);
+				NoteAnalysis nextNoteAnalysis = new NoteAnalysis(nextNoteIndex, getMusicSectionIndex(null, nextNoteIndex), (int)nextNoteFrequence, standDuration);
 
 				noteAnalysis = nextNoteAnalysis;
 
@@ -362,13 +365,13 @@ public class UserChannelContext implements PitchDetectionHandler {
 		
 		//FrequencyDetector frequencyDetector = new FrequencyDetector(samples, audioFormat.getSampleRate(), false);
 		YINPitchDetector frequencyDetector = new YINPitchDetector(samples.length/2 , audioFormat.getSampleRate());
-		double frequency = frequencyDetector.getFrequency(samples);
+		int frequency = (int)frequencyDetector.getFrequency(samples);
 		
-		double splDb = Signals.soundPressureLevel(samples);
+		int splDb = (int)Signals.soundPressureLevel(samples);
 		
-		float power = Signals.power(samples);
+		int power = (int)Signals.power(samples);
 		
-		float energy = Signals.energy(samples);
+		int energy = (int)Signals.energy(samples);
 		
 		LOGGER.info("Frequency:{}  SplDb:{}  Power:{}", frequency, splDb, power);
 		
@@ -383,7 +386,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 			if(Math.abs((lastestChunk.getFrequency() - frequency)) > 10 || Math.abs(lastestChunk.getPower() - power) > 0.1){
 				
 				double avgFrequency = chunkAnalysisList.stream().collect(Collectors.averagingDouble(ChunkAnalysis::getFrequency));
-				NoteAnalysis noteAnalysis = new NoteAnalysis(chunkAnalysisList.get(0).getStartTime(), startTime, avgFrequency);
+				NoteAnalysis noteAnalysis = new NoteAnalysis(chunkAnalysisList.get(0).getStartTime(), startTime, (int)avgFrequency);
 				doneNoteAnalysisList.add(noteAnalysis);//添加演奏的一个音符
 				
 				//重置
@@ -422,7 +425,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 		//取出当前处理中的音符信息
 		NoteAnalysis noteAnalysis = getProcessingNote();
 		if(noteAnalysis == null){
-			noteAnalysis = new NoteAnalysis(musicXmlNote.getMusicalNotesIndex(),musicXmlNote.getMeasureIndex(),musicXmlNote.getFrequency());
+			noteAnalysis = new NoteAnalysis(musicXmlNote.getMusicalNotesIndex(),musicXmlNote.getMeasureIndex(),(int)musicXmlNote.getFrequency());
 		}
 		
 		double noteDurationTime = noteAnalysis.getDurationTime() + durationTime;
@@ -442,7 +445,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 			
 			// 准备处理下一个音符
 			int nextNoteIndex = musicXmlNote.getMusicalNotesIndex() + 1;
-			NoteAnalysis nextNoteAnalysis = new NoteAnalysis(nextNoteIndex, getMusicSectionIndex(null, nextNoteIndex), getCurrentMusicNote(null,
+			NoteAnalysis nextNoteAnalysis = new NoteAnalysis(nextNoteIndex, getMusicSectionIndex(null, nextNoteIndex), (int)getCurrentMusicNote(null,
 					nextNoteIndex).getFrequency());
 			setProcessingNote(nextNoteAnalysis);
 		}
@@ -486,7 +489,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 		//取出当前小节的所有音符
 		List<NoteAnalysis> noteAnalysisList = doneNoteAnalysisList.stream().filter(t -> t.getSectionIndex() == sectionIndex).collect(Collectors.toList());
 		
-		long ignoreSize = doneNoteAnalysisList.stream().filter(t -> t.isIgnore()).count();
+		long ignoreSize = noteAnalysisList.stream().filter(t -> t.isIgnore()).count();
 
 		SectionAnalysis sectionAnalysis = new SectionAnalysis();
 		sectionAnalysis.setIndex(sectionIndex);
@@ -633,40 +636,48 @@ public class UserChannelContext implements PitchDetectionHandler {
 		}
 	}
 	
-	private double computeFrequency(List<ChunkAnalysis> chunkAnalysisList, ChunkAnalysis lastChunkAnalysis, int offsetRange) {
-
-		List<Double> chunkFrequencyList = chunkAnalysisList.stream().map(t -> t.getFrequency()).filter(t -> t.doubleValue() > 100 && t.doubleValue() < 2000)
-				.collect(Collectors.toList());
+	private int computeFrequency(List<ChunkAnalysis> chunkAnalysisList, ChunkAnalysis lastChunkAnalysis, int offsetRange) {
 		
-		if (chunkFrequencyList.size() == 0) {
-			return -1;
-		}
+		List<ChunkAnalysis> chunkList = new ArrayList<ChunkAnalysis>(chunkAnalysisList);
 		
+		int tenutoSize = 0;
 		// 剔除上一个音延续下来的信号
 		if (lastChunkAnalysis != null) {
-			double lastFrequency = lastChunkAnalysis.getFrequency();
-			Iterator<Double> iterable = chunkFrequencyList.iterator();
+			int lastFrequency = lastChunkAnalysis.getFrequency();
+			Iterator<ChunkAnalysis> iterable = chunkList.iterator();
 			while (iterable.hasNext()) {
-				if (Math.abs(lastFrequency - iterable.next()) > offsetRange) {
+				if (Math.abs(lastFrequency - iterable.next().getFrequency()) > offsetRange) {
 					break;
 				}
 				iterable.remove();
+				tenutoSize++;
 			}
 
-			if (chunkFrequencyList.size() == 0) {
-				return lastFrequency;
+			if (chunkList.size() == 0) {
+				return lastFrequency < 100 ? -1 : lastFrequency;
 			}
 		}
+
+		List<Integer> chunkFrequencyList = chunkList.stream().map(t -> t.getFrequency()).filter(t -> t.doubleValue() > 100 && t.doubleValue() < 2000)
+				.collect(Collectors.toList());
+		
+		if (chunkFrequencyList.size() == 0) {
+			return -1;
+		}
+		
+		if(tenutoSize * 100 / chunkAnalysisList.size() >= 50){
+			return lastChunkAnalysis.getFrequency();
+		}
 		
 		// 排序
 		chunkFrequencyList = chunkFrequencyList.stream().sorted().collect(Collectors.toList());
 		
-		double tempFrequency = chunkFrequencyList.get(0), totalFrequency = chunkFrequencyList.get(0);
+		int tempFrequency = chunkFrequencyList.get(0), totalFrequency = chunkFrequencyList.get(0);
 
 		int maxChunkSize = 0;
-		double frequency = chunkFrequencyList.get(0);
+		int frequency = chunkFrequencyList.get(0);
 		int chunkSize = 1;
-		double avgFrequency = chunkFrequencyList.get(0);
+		int avgFrequency = chunkFrequencyList.get(0);
 		for (int i = 1; i < chunkFrequencyList.size(); i++) {
 			tempFrequency = chunkFrequencyList.get(i);
 
@@ -696,7 +707,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 		}
 
 		if (chunkFrequencyList.size() < 3) {
-			frequency = chunkFrequencyList.stream().collect(Collectors.summingDouble(t -> t)) / chunkFrequencyList.size();
+			frequency = (int)(chunkFrequencyList.stream().collect(Collectors.summingDouble(t -> t)) / chunkFrequencyList.size());
 		}
 		
 		if(frequency < 100){
@@ -746,7 +757,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 								break;
 							}
 						} else {
-							if ((unplayedSize * 100 / chunkAnalysisList.size()) > hardLevel.getNotPlayRange() || unplayedSize > 3) {
+							if ((unplayedSize * 100 / chunkAnalysisList.size()) > hardLevel.getNotPlayRange() || unplayedSize > 1) {
 								tempo = false;
 								break;
 							}
@@ -763,7 +774,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 		
 		if (tempo) {
 			// 判断进入时间点
-			if((chunkAnalysisList.size() - chunkList.size() + firstPeakIndex + 1) * 100 /chunkAnalysisList.size() > hardLevel.getTempoOffsetOfPercent()){
+			if((chunkAnalysisList.size() - chunkList.size() + firstPeakIndex) * 100 /chunkAnalysisList.size() > hardLevel.getTempoOffsetOfPercent()){
 				tempo = false;
 			}
 		}
@@ -771,32 +782,26 @@ public class UserChannelContext implements PitchDetectionHandler {
 		return tempo;
 	}
 	
-	private boolean computeTempoWithAmplitude(List<ChunkAnalysis> chunkAnalysisList, float lastChunkAmplitude) {
+	private boolean computeTempoWithAmplitude(List<ChunkAnalysis> chunkAnalysisList, ChunkAnalysis lastChunkAnalysis) {
 
 		boolean tempo = false;
 
-		List<Float> chunkAmplitudeList = chunkAnalysisList.stream().map(ChunkAnalysis::getAmplitude).collect(Collectors.toList());
+		List<Integer> chunkAmplitudeList = chunkAnalysisList.stream().map(ChunkAnalysis::getAmplitude).collect(Collectors.toList());
 
-		// 剔除余波
 		if (chunkAmplitudeList.size() < 3) {
 			return chunkAmplitudeList.stream().filter(t -> t.floatValue() > hardLevel.getAmplitudeThreshold()).count() > 0;
 		}
+		
+		chunkAmplitudeList.add(0, lastChunkAnalysis.getAmplitude());
 
 		// 检测是否有多个波峰
 		int peakSize = 0;
 		int minPeakIndex = -1;
-		int notPlaySize = 0;
-		for (int i = 0; i < chunkAmplitudeList.size(); i++) {
-			if (chunkAmplitudeList.get(i) < 2) {
-				notPlaySize++;
+		for (int i = 1; i < chunkAmplitudeList.size(); i++) {
+			if (chunkAmplitudeList.get(i) < hardLevel.getAmplitudeThreshold()) {
 				continue;
 			}
-			if (i == 0) {
-				if (lastChunkAmplitude > 0 && chunkAmplitudeList.get(i) > lastChunkAmplitude) {
-					peakSize++;
-					minPeakIndex = i;
-				}
-			} else if (i == chunkAmplitudeList.size() - 1) {
+			if (i == chunkAmplitudeList.size() - 1) {
 				if (chunkAmplitudeList.get(i) > chunkAmplitudeList.get(i - 1)) {
 					peakSize++;
 					if (minPeakIndex == -1 || minPeakIndex > i) {
@@ -804,7 +809,7 @@ public class UserChannelContext implements PitchDetectionHandler {
 					}
 				}
 			} else {
-				if (chunkAmplitudeList.get(i - 1) < chunkAmplitudeList.get(i) && chunkAmplitudeList.get(i) >= chunkAmplitudeList.get(i + 1)) {
+				if (chunkAmplitudeList.get(i - 1) + 2 < chunkAmplitudeList.get(i) && chunkAmplitudeList.get(i) >= chunkAmplitudeList.get(i + 1)) {
 					peakSize++;
 					if (minPeakIndex == -1 || minPeakIndex > i) {
 						minPeakIndex = i;
@@ -813,15 +818,19 @@ public class UserChannelContext implements PitchDetectionHandler {
 			}
 		}
 
-		if(notPlaySize == chunkAmplitudeList.size()){
-			tempo = false;
-		}else{
-			tempo = peakSize <= 1;
+		if (peakSize == 1) {
+			if (lastChunkAnalysis.isPeak() == false) {
+				tempo = true;
+			}
+		} else if (peakSize == 0) {
+			if (lastChunkAnalysis.isPeak()) {
+				tempo = true;
+			}
 		}
 
 		// 检测是否延迟进入
 		if (tempo == true) {
-			if ((minPeakIndex + 1) * 100 / chunkAmplitudeList.size() > hardLevel.getTempoOffsetOfPercent() && chunkAmplitudeList.size() > 3) {
+			if (minPeakIndex * 100 / chunkAmplitudeList.size() > hardLevel.getTempoOffsetOfPercent() && chunkAmplitudeList.size() > 3) {
 				tempo = false;
 			}
 		}

audio-analysis/src/main/java/com/yonge/netty/server/messagehandler/TextWebSocketHandler.java

@@ -12,6 +12,7 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Map.Entry;
 import java.util.stream.Collectors;
 
@@ -172,7 +173,7 @@ public class TextWebSocketHandler extends SimpleChannelInboundHandler<TextWebSoc
 					musicalNotesPlayStats.put("examSongId", musicXmlBasicInfo.getExamSongId());
 					musicalNotesPlayStats.put("xmlUrl", musicXmlBasicInfo.getXmlUrl());
 					
-					musicalNotesPlayStats.put("notesData", channelContext.getDoneNoteAnalysisList());
+					musicalNotesPlayStats.put("notesData", channelContext.getDoneNoteAnalysisList().stream().filter(t -> t.isIgnore() == false).collect(Collectors.toList()));
 					scoreData.put("musicalNotesPlayStats", musicalNotesPlayStats);
 					sysMusicCompareRecord.setScoreData(JSON.toJSONString(scoreData));
 					
@@ -192,7 +193,18 @@ public class TextWebSocketHandler extends SimpleChannelInboundHandler<TextWebSoc
 
 			break;
 		case "videoUpload": // 上传音频
-
+			SysMusicCompareRecord musicCompareRecord = null;
+			if (dataObj.containsKey("recordId")) {
+				musicCompareRecord = sysMusicCompareRecordService.get(dataObj.getLong("recordId"));
+			}
+			if (Objects.nonNull(musicCompareRecord) && dataObj.containsKey("filePath")) {
+				musicCompareRecord.setVideoFilePath(dataObj.getString("filePath"));
+				sysMusicCompareRecordService.update(musicCompareRecord);
+			} else {
+				musicCompareRecord.setVideoFilePath(musicCompareRecord.getRecordFilePath());
+				sysMusicCompareRecordService.update(musicCompareRecord);
+			}
+			
 			break;
 		case "checkSound": // 校音