update

audio-analysis/src/main/java/com/yonge/netty/dto/NoteFrequencyRange.java

@@ -0,0 +1,69 @@
+package com.yonge.netty.dto;
+
+import java.math.BigDecimal;
+
+/**
+ * 一个音符的频率范围，包含最大值和最小值
+ */
+public class NoteFrequencyRange {
+
+	private double minFrequency;
+
+	private double maxFrequency;
+
+	public NoteFrequencyRange(double standardFrequecy, double frequency) {
+		int midiNoteSize = 128;
+		double[] midiNoteFrequencies = new double[midiNoteSize];
+
+		for (int x = 0; x < midiNoteSize; ++x) {
+			midiNoteFrequencies[x] = new BigDecimal(standardFrequecy).multiply(
+					new BigDecimal(Math.pow(2, new BigDecimal(x - 69).divide(new BigDecimal(12), 6, BigDecimal.ROUND_HALF_UP).doubleValue()))).doubleValue();
+
+			if (midiNoteFrequencies[x] >= frequency) {
+				if (midiNoteFrequencies[x] - frequency > frequency - midiNoteFrequencies[x - 1]) {
+					// frequency演奏的是上一个音符
+					maxFrequency = midiNoteFrequencies[x - 1] + (midiNoteFrequencies[x] - midiNoteFrequencies[x - 1]) / 2;
+					minFrequency = midiNoteFrequencies[x - 1] - (midiNoteFrequencies[x - 1] - midiNoteFrequencies[x - 2]) / 2;
+				} else {
+					// frequency演奏的是当前音符
+					midiNoteFrequencies[x + 1] = new BigDecimal(standardFrequecy).multiply(
+							new BigDecimal(Math.pow(2, new BigDecimal((x + 1) - 69).divide(new BigDecimal(12), 6, BigDecimal.ROUND_HALF_UP).doubleValue())))
+							.doubleValue();
+					maxFrequency = midiNoteFrequencies[x] + (midiNoteFrequencies[x + 1] - midiNoteFrequencies[x]) / 2;
+					minFrequency = midiNoteFrequencies[x] - (midiNoteFrequencies[x] - midiNoteFrequencies[x - 1]) / 2;
+				}
+				break;
+			}
+		}
+	}
+
+	public NoteFrequencyRange(double frequency) {
+		new NoteFrequencyRange(442, frequency);
+	}
+
+	public double getMinFrequency() {
+		return minFrequency;
+	}
+
+	public void setMinFrequency(double minFrequency) {
+		this.minFrequency = minFrequency;
+	}
+
+	public double getMaxFrequency() {
+		return maxFrequency;
+	}
+
+	public void setMaxFrequency(double maxFrequency) {
+		this.maxFrequency = maxFrequency;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (obj instanceof NoteFrequencyRange) {
+			NoteFrequencyRange nfr = (NoteFrequencyRange) obj;
+			return this.minFrequency == nfr.minFrequency && this.maxFrequency == nfr.maxFrequency;
+		}
+		return false;
+	}
+
+}

audio-analysis/src/main/java/com/yonge/netty/dto/NotePlayResult.java

@@ -0,0 +1,28 @@
+package com.yonge.netty.dto;
+
+public class NotePlayResult {
+
+	private boolean status;
+	
+	private double migrationRate;
+	
+	public NotePlayResult() {
+		// TODO Auto-generated constructor stub
+	}
+
+	public boolean getStatus() {
+		return status;
+	}
+
+	public void setStatus(boolean status) {
+		this.status = status;
+	}
+
+	public double getMigrationRate() {
+		return migrationRate;
+	}
+
+	public void setMigrationRate(double migrationRate) {
+		this.migrationRate = migrationRate;
+	}
+}

audio-analysis/src/main/java/com/yonge/netty/dto/UserChannelContext.java

@@ -4,7 +4,6 @@ import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -33,7 +32,9 @@ public class UserChannelContext {
 	
 	private final static Logger LOGGER = LoggerFactory.getLogger(UserChannelContext.class);
 	
-	private int offsetMS = 350;
+	private double standardFrequecy = 442;
+	
+	private int offsetMS;
 	
 	private String platform;
 	
@@ -45,6 +46,8 @@ public class UserChannelContext {
 	
 	private int beatByteLength;
 	
+	private boolean delayPrcessed;
+	
 	// 曲目与musicxml对应关系
 	private ConcurrentHashMap<Integer, MusicXmlBasicInfo> songMusicXmlMap = new ConcurrentHashMap<Integer, MusicXmlBasicInfo>();
 
@@ -68,6 +71,41 @@ public class UserChannelContext {
 	
 	private HardLevelEnum hardLevel = HardLevelEnum.ADVANCED;
 	
+	private NotePlayResult queryNoteFrequency(MusicXmlNote xmlNote, double playFrequency) {
+
+		NotePlayResult result = new NotePlayResult();
+
+		boolean status = false;
+		double migrationRate = 0;
+
+		if (Math.round(xmlNote.getFrequency()) == Math.round(playFrequency)) {
+			status = true;
+			migrationRate = 0;
+		} else {
+			NoteFrequencyRange noteFrequencyRange = new NoteFrequencyRange(standardFrequecy, xmlNote.getFrequency());
+
+			if (noteFrequencyRange.getMinFrequency() > playFrequency || playFrequency > noteFrequencyRange.getMaxFrequency()) {
+				status = false;
+			} else {
+
+				status = true;
+
+				if (Math.round(playFrequency) < Math.round(xmlNote.getFrequency())) {
+					double min = Math.abs(xmlNote.getFrequency() - noteFrequencyRange.getMinFrequency()) / 2;
+					migrationRate = Math.abs(playFrequency - xmlNote.getFrequency()) / min;
+				} else {
+					double max = Math.abs(xmlNote.getFrequency() - noteFrequencyRange.getMaxFrequency()) / 2;
+					migrationRate = Math.abs(playFrequency - xmlNote.getFrequency()) / max;
+				}
+			}
+		}
+
+		result.setStatus(status);
+		result.setMigrationRate(migrationRate);
+
+		return result;
+	}
+	
 	public void init(String platform, String heardLevel, int subjectId, int beatDuration) {
 		this.platform = platform;
 		this.subjectId = subjectId;
@@ -151,6 +189,7 @@ public class UserChannelContext {
 		recordId = null;
 		playTime = 0;
 		receivedTime = 0;
+		delayPrcessed = false;
 	}
 	
 	public MusicXmlBasicInfo getMusicXmlBasicInfo(Integer songId){
@@ -311,6 +350,13 @@ public class UserChannelContext {
 			}
 			totalChunkAnalysisList.add(chunkAnalysis);
 			
+			if(delayPrcessed == false && chunkAnalysis.getFrequency() > 100){
+				
+				delayPrcessed = true;
+				//计算延迟偏移值
+				playTime = musicXmlNote.getTimeStamp() + durationTime;
+			}
+			
 			if (playTime >= (musicXmlNote.getDuration() + musicXmlNote.getTimeStamp())) {
 
 				if (musicXmlNote.getDontEvaluating()) {
@@ -466,6 +512,9 @@ public class UserChannelContext {
 				}
 			}
 		} else {
+			
+			NotePlayResult notePlayResult = queryNoteFrequency(musicXmlNote, noteAnalysis.getPlayFrequency());
+			
 			if (noteAnalysis.getFrequency() == -1) {// 休止符
 
 				playDurationTime = chunkAnalysisList.stream().filter(t -> t.getFrequency() <= 100).mapToDouble(t -> t.getDurationTime()).sum();
@@ -474,7 +523,7 @@ public class UserChannelContext {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.CADENCE_WRONG);
 				} else if (playDurationTime * 100 / noteAnalysis.getDurationTime() < hardLevel.getIntegrityRange()) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.INTEGRITY_WRONG);
-				} else if (Math.abs(noteAnalysis.getFrequency() - noteAnalysis.getPlayFrequency()) > hardLevel.getFrequencyThreshold()) {
+				} else if (notePlayResult.getStatus() == false) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.INTONATION_WRONG);
 				} else {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.RIGHT);
@@ -489,7 +538,7 @@ public class UserChannelContext {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.INTEGRITY_WRONG);
 				} else if (!noteAnalysis.isTempo()) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.CADENCE_WRONG);
-				} else if (Math.abs(noteAnalysis.getFrequency() - noteAnalysis.getPlayFrequency()) > hardLevel.getFrequencyThreshold()) {
+				} else if (notePlayResult.getStatus() == false) {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.INTONATION_WRONG);
 				} else {
 					noteAnalysis.setMusicalErrorType(NoteErrorType.RIGHT);
@@ -552,95 +601,26 @@ public class UserChannelContext {
 		LOGGER.info("-------startTime:{}  endTime:{}------", firstChunkAnalysis.getStartTime(), chunkAnalysisList.get(chunkAnalysisList.size() - 1)
 				.getEndTime());
 		
-		Optional<ChunkAnalysis> chunkAnalysisOptional = totalChunkAnalysisList.stream().filter(t -> Double.doubleToLongBits(t.getEndTime()) == Double.doubleToLongBits(firstChunkAnalysis.getStartTime())).findFirst();
-
-		ChunkAnalysis lastChunkAnalysis = null;
-		if (chunkAnalysisOptional.isPresent()) {
-			lastChunkAnalysis = chunkAnalysisOptional.get();
-		}
-		if(lastChunkAnalysis == null){
-			lastChunkAnalysis = new ChunkAnalysis(0, 0, -1, 0, 0, 0);
-		}
-		
 		List<ChunkAnalysis> chunkList = new ArrayList<ChunkAnalysis>(chunkAnalysisList);
 		
-		int tenutoSize = 0;
-		// 剔除上一个音延续下来的信号
-		if (lastChunkAnalysis != null) {
-			int lastFrequency = lastChunkAnalysis.getFrequency();
-			Iterator<ChunkAnalysis> iterable = chunkList.iterator();
-			while (iterable.hasNext()) {
-				if (Math.abs(lastFrequency - iterable.next().getFrequency()) > hardLevel.getFrequencyThreshold()) {
-					break;
-				}
-				iterable.remove();
-				tenutoSize++;
-			}
-
-			if (chunkList.size() == 0) {
-				return lastFrequency < 100 ? -1 : lastFrequency;
-			}
-		}
-
 		List<Integer> chunkFrequencyList = chunkList.stream().map(t -> t.getFrequency()).filter(t -> t.doubleValue() > 100 && t.doubleValue() < 2000)
 				.collect(Collectors.toList());
 		
 		if (chunkFrequencyList.size() == 0) {
 			return -1;
 		}
-		
-		if(tenutoSize * 100 / chunkAnalysisList.size() > 50){
-			return lastChunkAnalysis.getFrequency();
-		}
-		
-		// 排序
-		chunkFrequencyList = chunkFrequencyList.stream().sorted().collect(Collectors.toList());
-		
-		int tempFrequency = chunkFrequencyList.get(0), totalFrequency = chunkFrequencyList.get(0);
-
-		int maxChunkSize = 0;
-		int frequency = chunkFrequencyList.get(0);
-		int chunkSize = 1;
-		int avgFrequency = chunkFrequencyList.get(0);
-		for (int i = 1; i < chunkFrequencyList.size(); i++) {
-			tempFrequency = chunkFrequencyList.get(i);
-
-			if (Math.abs(avgFrequency - tempFrequency) > hardLevel.getFrequencyThreshold()) {
-
-				avgFrequency = totalFrequency / chunkSize;
-
-				if (maxChunkSize < chunkSize) {
-					maxChunkSize = chunkSize;
-					frequency = avgFrequency;
-				}
-
-				chunkSize = 1;
-				avgFrequency = tempFrequency;
-				totalFrequency = tempFrequency;
-			} else {
-				chunkSize++;
-				totalFrequency += tempFrequency;
-			}
-
-			if (i == chunkFrequencyList.size() - 1) {
-				if (maxChunkSize <= chunkSize) {
-					maxChunkSize = chunkSize;
-					frequency = totalFrequency / chunkSize;
-				}
-			}
-		}
 
-		if (chunkFrequencyList.size() < 3) {
-			frequency = (int)chunkFrequencyList.get(chunkFrequencyList.size() - 1);
-		}
-		
-		if(frequency < 100){
-			frequency = -1;
-		}
+		int frequency = (int) (chunkFrequencyList.stream().mapToInt(t -> t).sum() / chunkFrequencyList.size());
 
 		return frequency;
 	}
 	
+	/**
+	 * 时值范围内有且只有一个音，且不能间断，且在合理范围内需开始演奏
+	 * 与上一个音相同时，2个音之间需要间断
+	 * @param musicXmlNote
+	 * @return
+	 */
 	private boolean computeTempoWithFrequency(MusicXmlNote musicXmlNote){
 		
 		double floatingRange = musicXmlNote.getDuration() * hardLevel.getTempoEffectiveRange(musicXmlNote.getDenominator()) / 100;
@@ -672,22 +652,11 @@ public class UserChannelContext {
 		
 		List<ChunkAnalysis> chunkList = new ArrayList<ChunkAnalysis>(chunkAnalysisList);
 		
-		// 剔除上一个音延续下来的信号
-		if (lastChunkAnalysis != null) {
-			double lastFrequency = lastChunkAnalysis.getFrequency();
-			Iterator<ChunkAnalysis> iterable = chunkList.iterator();
-			while (iterable.hasNext()) {
-				if (Math.abs(lastFrequency - iterable.next().getFrequency()) > hardLevel.getFrequencyThreshold()) {
-					break;
-				}
-				iterable.remove();
-			}
-		}
-		
 		if(chunkList.size() == 0){
 			return false;
 		}
 		
+		NoteFrequencyRange noteFrequencyRange = null;
 		ChunkAnalysis chunkAnalysis = null;
 		boolean tempo = false;
 		boolean isContinue = true;
@@ -697,21 +666,23 @@ public class UserChannelContext {
 			chunkAnalysis = chunkList.get(i);
 			if (chunkAnalysis != null) {
 				if (chunkAnalysis.getFrequency() > 100) {
+					
 					tempo = true;
-					if(firstPeakIndex == -1){
+					if (firstPeakIndex == -1) {
 						firstPeakIndex = i;
+						noteFrequencyRange = new NoteFrequencyRange(standardFrequecy, chunkAnalysis.getFrequency());
+					} else if (noteFrequencyRange.getMinFrequency() > chunkAnalysis.getFrequency()
+							|| chunkAnalysis.getFrequency() > noteFrequencyRange.getMaxFrequency()) {
+						// 判断是否是同一个音
+						tempo = false;
+						break;
 					}
 					if (isContinue == false) {
-						if (chunkAnalysisList.size() < 5) {
+						if ((i + 1) / chunkAnalysisList.size() < hardLevel.getIntegrityRange()) {
 							if (unplayedSize > 0) {
 								tempo = false;
 								break;
 							}
-						} else {
-							if ((unplayedSize * 100 / chunkAnalysisList.size()) > hardLevel.getNotPlayRange() || unplayedSize > 1) {
-								tempo = false;
-								break;
-							}
 						}
 					}
 				} else {
@@ -725,8 +696,13 @@ public class UserChannelContext {
 		
 		if (tempo) {
 			// 判断进入时间点
-			if((chunkAnalysisList.size() - chunkList.size() + firstPeakIndex) * 100 /chunkAnalysisList.size() > hardLevel.getTempoEffectiveRange(musicXmlNote.getDenominator()) * 2){
+			if(firstPeakIndex * 100 /chunkAnalysisList.size() > hardLevel.getTempoEffectiveRange(musicXmlNote.getDenominator()) * 2){
 				tempo = false;
+			}else{
+				//判断是否与上一个音延续下来的
+				if(firstChunkAnalysis.getFrequency() > 100 && lastChunkAnalysis.getFrequency() > 100){
+					return new NoteFrequencyRange(standardFrequecy, firstChunkAnalysis.getFrequency()).equals(new NoteFrequencyRange(standardFrequecy, lastChunkAnalysis.getFrequency())) == false;
+				}
 			}
 		}
 		
@@ -809,21 +785,14 @@ public class UserChannelContext {
 	}
 	
 	public static void main(String[] args) {
-		UserChannelContext context = new UserChannelContext();
-		
-		//int[] frequencys = {286,291,291,291,291,291,291};
-		int[] frequencys = {312,43,295,294,294,295};
-		
-		List<ChunkAnalysis> chunkAnalysisList = new ArrayList<ChunkAnalysis>();
-		for(int f : frequencys) {
-			chunkAnalysisList.add(new ChunkAnalysis(f, 0, 0));
+		double[] midi = new double[128];;
+		int standardPitch = 440; // a is 440 hz...
+		for (int x = 0; x < midi.length; ++x)
+		{
+		   midi[x] = new BigDecimal(standardPitch).multiply(new BigDecimal(Math.pow(2, new BigDecimal(x-69).divide(new BigDecimal(12),6,BigDecimal.ROUND_HALF_UP).doubleValue()))).doubleValue();
+		   System.out.println("x=" + x +"  "+ midi[x]);
 		}
 		
-		MusicXmlNote musicXmlNote = new MusicXmlNote();
-		musicXmlNote.setDenominator(1);
-		
-		//System.out.println(context.computeFrequency(chunkAnalysisList, lastChunkAnalysis, 5));
-		System.out.println(context.computeTempoWithFrequency(musicXmlNote));
 	}
 	
 }

audio-analysis/src/main/java/com/yonge/netty/entity/MusicXmlNote.java

@@ -18,7 +18,7 @@ public class MusicXmlNote {
 	private float nextFrequency;
 
 	// 上一个音的频率（不是乐谱中上一个音符的频率）
-	private float lastFrequency;
+	private float prevFrequency;
 
 	// 当前音符所在的小节下标（从0开始）
 	private int measureIndex;
@@ -56,12 +56,12 @@ public class MusicXmlNote {
 		this.frequency = frequency;
 	}
 
-	public float getLastFrequency() {
-		return lastFrequency;
+	public float getPrevFrequency() {
+		return prevFrequency;
 	}
 
-	public void setLastFrequency(float lastFrequency) {
-		this.lastFrequency = lastFrequency;
+	public void setPrevFrequency(float prevFrequency) {
+		this.prevFrequency = prevFrequency;
 	}
 
 	public float getNextFrequency() {

audio-analysis/src/main/java/com/yonge/netty/server/service/AudioCompareHandler.java

@@ -92,6 +92,8 @@ public class AudioCompareHandler implements MessageHandler {
 	private String tmpFileDir = "/mdata/soundCompare/";
 
 	private SimpleDateFormat sdf = new SimpleDateFormat("yyMMddHHmmSS");
+	
+	private boolean handlerSwitch = false;
 
 	@Override
 	public String getAction() {
@@ -228,7 +230,8 @@ public class AudioCompareHandler implements MessageHandler {
 			
 			Integer offsetTime = dataObj.getInteger("offsetTime");
 			if(offsetTime != null){
-				//channelContext.setOffsetMS(offsetTime);
+				channelContext.setOffsetMS(offsetTime);
+				handlerSwitch = true;
 			}
 
 			break;
@@ -281,6 +284,26 @@ public class AudioCompareHandler implements MessageHandler {
 		channelContext.setChannelBufferBytes(ArrayUtil.mergeByte(channelContext.getChannelBufferBytes(), datas));
 
 		int totalLength = channelContext.getChannelBufferBytes().length;
+		
+		if (handlerSwitch == false) {
+			return false;
+		}
+		
+		if (channelContext.getOffsetMS() > 0) {
+			// 减掉空白
+			int durationTime = (int) (1000 * totalLength / audioFormat.getSampleRate() / (audioFormat.getSampleSizeInBits() / 8));
+
+			if (durationTime > channelContext.getOffsetMS()) {
+				int beatByteLength = WaveformWriter.SAMPLE_RATE * WaveformWriter.BITS_PER_SAMPLE / 8 * channelContext.getOffsetMS() / 1000;
+				channelContext.setChannelBufferBytes(ArrayUtil.extractByte(channelContext.getChannelBufferBytes(), beatByteLength, totalLength - 1));
+				channelContext.setOffsetMS(0);
+			} else {
+				channelContext.setOffsetMS(channelContext.getOffsetMS() - durationTime);
+				return false;
+			}
+		}
+		
+		totalLength = channelContext.getChannelBufferBytes().length;
 
 		while (totalLength >= bufferSize) {
 			byte[] bufferData = ArrayUtil.extractByte(channelContext.getChannelBufferBytes(), 0, bufferSize - 1);