feat:小节评分

mec-biz/src/main/java/com/ym/mec/biz/dal/dto/MusicPitchDetailDto.java

@@ -11,6 +11,9 @@ public class MusicPitchDetailDto {
     @ApiModelProperty("时间戳ms")
     private int timeStamp;
 
+    @ApiModelProperty("结束时间戳")
+    private int endTimeStamp;
+
     @ApiModelProperty("持续时长ms")
     private int duration;
 
@@ -20,6 +23,14 @@ public class MusicPitchDetailDto {
     @ApiModelProperty("小节数")
     private int measureIndex;
 
+    public int getEndTimeStamp() {
+        return endTimeStamp;
+    }
+
+    public void setEndTimeStamp(int endTimeStamp) {
+        this.endTimeStamp = endTimeStamp;
+    }
+
     public MusicPitchDetailDto() {
     }
 
@@ -28,6 +39,11 @@ public class MusicPitchDetailDto {
         this.frequency = frequency;
     }
 
+    public MusicPitchDetailDto(int timeStamp, int endTimeStamp) {
+        this.timeStamp = timeStamp;
+        this.endTimeStamp = endTimeStamp;
+    }
+
     public MusicPitchDetailDto(int timeStamp, int duration, float frequency) {
         this.timeStamp = timeStamp;
         this.duration = duration;

mec-biz/src/main/java/com/ym/mec/biz/dal/dto/SoundCompareHelper.java

@@ -25,7 +25,7 @@ public class SoundCompareHelper {
     private Map<Integer, List<MusicPitchDetailDto>> measureXmlInfoMap = new HashMap<>();
 
     @ApiModelProperty(value = "小节结束时间字典")
-    private Map<Integer, Integer> measureEndTime = new HashMap<>();
+    private Map<Integer, MusicPitchDetailDto> measureEndTime = new HashMap<>();
 
     @ApiModelProperty(value = "录音音频信息")
     private List<MusicPitchDetailDto> recordMeasurePithInfo = new ArrayList<>();
@@ -88,11 +88,11 @@ public class SoundCompareHelper {
         this.measureXmlInfoMap = measureXmlInfoMap;
     }
 
-    public Map<Integer, Integer> getMeasureEndTime() {
+    public Map<Integer, MusicPitchDetailDto> getMeasureEndTime() {
         return measureEndTime;
     }
 
-    public void setMeasureEndTime(Map<Integer, Integer> measureEndTime) {
+    public void setMeasureEndTime(Map<Integer, MusicPitchDetailDto> measureEndTime) {
         this.measureEndTime = measureEndTime;
     }
 

mec-biz/src/main/java/com/ym/mec/biz/handler/WebSocketHandler.java

@@ -22,6 +22,7 @@ import org.springframework.web.socket.*;
 import org.springframework.web.socket.handler.AbstractWebSocketHandler;
 
 import javax.sound.sampled.AudioFormat;
+import javax.sound.sampled.UnsupportedAudioFileException;
 import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
@@ -102,8 +103,9 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
                 userSoundInfoMap.get(phone).setMusicScoreId(bodyObject.getInteger("id"));
                 userSoundInfoMap.get(phone).setMeasureXmlInfoMap(musicXmlInfos.stream().collect(Collectors.groupingBy(MusicPitchDetailDto::getMeasureIndex)));
                 for (Map.Entry<Integer, List<MusicPitchDetailDto>> userMeasureXmlInfoEntry : userSoundInfoMap.get(phone).getMeasureXmlInfoMap().entrySet()) {
-                    MusicPitchDetailDto musicPitchDetailDto = userMeasureXmlInfoEntry.getValue().stream().max(Comparator.comparing(MusicPitchDetailDto::getTimeStamp)).get();
-                    userSoundInfoMap.get(phone).getMeasureEndTime().put(userMeasureXmlInfoEntry.getKey(), musicPitchDetailDto.getTimeStamp()+musicPitchDetailDto.getDuration());
+                    MusicPitchDetailDto firstPitch = userMeasureXmlInfoEntry.getValue().stream().min(Comparator.comparing(MusicPitchDetailDto::getTimeStamp)).get();
+                    MusicPitchDetailDto lastPitch = userMeasureXmlInfoEntry.getValue().stream().max(Comparator.comparing(MusicPitchDetailDto::getTimeStamp)).get();
+                    userSoundInfoMap.get(phone).getMeasureEndTime().put(userMeasureXmlInfoEntry.getKey(), new MusicPitchDetailDto(firstPitch.getTimeStamp(), lastPitch.getTimeStamp() + lastPitch.getDuration()));
                 }
                 break;
             case SoundSocketService.RECORD_START:
@@ -121,7 +123,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
                     Integer lastMeasureIndex = userSoundInfoMap.get(phone).getMeasureEndTime().keySet().stream().min(Integer::compareTo).get();
                     double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(audioFormat.getFrameSize()*audioFormat.getFrameRate())*1000;
                     //如果结束时时长大于某小节，则此小节需要评分
-                    if(recordTime>userSoundInfoMap.get(phone).getMeasureEndTime().get(lastMeasureIndex)){
+                    if(recordTime>userSoundInfoMap.get(phone).getMeasureEndTime().get(lastMeasureIndex).getEndTimeStamp()){
                         measureCompare(phone, lastMeasureIndex);
                         userSoundInfoMap.get(phone).getMeasureEndTime().remove(lastMeasureIndex);
                     }
@@ -134,7 +136,8 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
                 break;
             case SoundSocketService.PROXY_MESSAGE:
                 if(bodyObject.containsKey(SoundSocketService.OFFSET_TIME)){
-                    userSoundInfoMap.get(phone).setOffsetTime(bodyObject.getIntValue(SoundSocketService.OFFSET_TIME));
+                    int offsetTime = bodyObject.getIntValue(SoundSocketService.OFFSET_TIME);
+                    calOffsetTime(phone, offsetTime);
                 }
                 break;
             default:
@@ -152,19 +155,9 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
             userSoundInfoMap.get(phone).getAccessFile().write(message.getPayload().array());
         }
 
-        byte[] newByte = new byte[userSoundInfoMap.get(phone).getPreDataArray().length+message.getPayloadLength()];
-        System.arraycopy(userSoundInfoMap.get(phone).getPreDataArray(), 0, newByte, 0, userSoundInfoMap.get(phone).getPreDataArray().length);
-        System.arraycopy(message.getPayload().array(), 0, newByte, userSoundInfoMap.get(phone).getPreDataArray().length, message.getPayloadLength());
-        userSoundInfoMap.get(phone).setPreDataArray(message.getPayload().array());
-
-        double preDurationTime = userSoundInfoMap.get(phone).getPreDataArray().length/(audioFormat.getFrameSize()*audioFormat.getFrameRate())*1000;
-
         List<MusicPitchDetailDto> recordInfo = new ArrayList<>();
         AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), audioFormat, simpleSize, 128);
         dispatcher.addAudioProcessor(new PitchProcessor(algo, simpleRate, simpleSize, (pitchDetectionResult, audioEvent) -> {
-//            if(audioEvent.getTimeStamp()*1000<=preDurationTime){
-//                return;
-//            }
             int timeStamp = (int) (userSoundInfoMap.get(phone).getMeasureStartTime() + audioEvent.getTimeStamp()*1000);
             float pitch = pitchDetectionResult.getPitch();
 //            LOGGER.info("频率：{}, {}", timeStamp, pitch);
@@ -176,12 +169,14 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
         }
 
         double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(audioFormat.getFrameSize()*audioFormat.getFrameRate())*1000;
+
         userSoundInfoMap.get(phone).setMeasureStartTime(recordTime);
         userSoundInfoMap.get(phone).getRecordMeasurePithInfo().addAll(recordInfo);
-        for (Map.Entry<Integer, Integer> userMeasureEndTimeMapEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) {
-            int ot = (int) (userMeasureEndTimeMapEntry.getValue()*0.1);
-            if((recordTime - userSoundInfoMap.get(phone).getOffsetTime())>(userMeasureEndTimeMapEntry.getValue()+ot)){
-                measureCompare(phone, userMeasureEndTimeMapEntry.getKey());
+        for (Map.Entry<Integer, MusicPitchDetailDto> userMeasureEndTimeMapEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) {
+            int ot = (int) (userMeasureEndTimeMapEntry.getValue().getEndTimeStamp()*0.1);
+            if(recordTime>(userMeasureEndTimeMapEntry.getValue().getEndTimeStamp()+ot)){
+//                measureCompare(phone, userMeasureEndTimeMapEntry.getKey());
+                measureCompare2(phone, userMeasureEndTimeMapEntry.getValue());
                 userSoundInfoMap.get(phone).getMeasureEndTime().remove(userMeasureEndTimeMapEntry.getKey());
                 break;
             }
@@ -222,6 +217,22 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
     }
 
     /**
+     * @describe 处理时间偏移
+     * @author Joburgess
+     * @date 2021/7/5 0005
+     * @param phone:
+     * @param offsetTime:
+     * @return void
+     */
+    private void calOffsetTime(String phone, int offsetTime){
+        userSoundInfoMap.get(phone).setOffsetTime(offsetTime);
+        for (Map.Entry<Integer, MusicPitchDetailDto> musicPitchDetailDtoEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) {
+            musicPitchDetailDtoEntry.getValue().setTimeStamp(musicPitchDetailDtoEntry.getValue().getTimeStamp() + offsetTime);
+            musicPitchDetailDtoEntry.getValue().setEndTimeStamp(musicPitchDetailDtoEntry.getValue().getEndTimeStamp() + offsetTime);
+        }
+    }
+
+    /**
      * @describe 保存录音数据，并生成wav头信息
      * @author Joburgess
      * @date 2021/6/25 0025
@@ -290,8 +301,8 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
 
             for (MusicPitchDetailDto musicXmlInfo : userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex)) {
                 int ot5 = (int) (musicXmlInfo.getDuration()*0.1);
-                int startTimeStamp = musicXmlInfo.getTimeStamp() - ot5 + userSoundInfoMap.get(phone).getOffsetTime();
-                int endTimeStamp = musicXmlInfo.getTimeStamp() + ot5 + musicXmlInfo.getDuration() + userSoundInfoMap.get(phone).getOffsetTime();
+                int startTimeStamp = musicXmlInfo.getTimeStamp() - ot5;
+                int endTimeStamp = musicXmlInfo.getTimeStamp() + ot5;
 
                 //时间范围内有效音准数量
                 float recordValidIntonationNum = 0;
@@ -392,6 +403,164 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
         WS_CLIENTS.get(phone).sendMessage(new TextMessage(JSON.toJSONString(createPushInfo("measureScore", measureIndex, intonation, cadence, integrity))));
     }
 
+    private void measureCompare2(String phone, MusicPitchDetailDto measureTimeInfo) throws IOException, UnsupportedAudioFileException {
+        //小节总时长
+        int measureTime = measureTimeInfo.getEndTimeStamp()-measureTimeInfo.getTimeStamp();
+        int ot = (int) (measureTime * 0.1);
+        //小节时长占用字节数
+        int measureByteNum = (int) (measureTime*(audioFormat.getFrameSize()*audioFormat.getFrameRate()));
+
+        List<MusicPitchDetailDto> recordInfo = new ArrayList<>();
+        byte[] bytes = new byte[measureByteNum];
+        userSoundInfoMap.get(phone).getAccessFile().seek(userSoundInfoMap.get(phone).getAccessFile().length()-measureByteNum);
+        userSoundInfoMap.get(phone).getAccessFile().readFully(bytes);
+
+        AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(bytes, audioFormat, simpleSize, 128);
+        dispatcher.addAudioProcessor(new PitchProcessor(algo, simpleRate, simpleSize, (pitchDetectionResult, audioEvent) -> {
+            int timeStamp = (int) (measureTimeInfo.getTimeStamp() - ot + audioEvent.getTimeStamp()*1000);
+            float pitch = pitchDetectionResult.getPitch();
+            recordInfo.add(new MusicPitchDetailDto(timeStamp, pitch));
+        }));
+        dispatcher.run();
+        userSoundInfoMap.get(phone).getRecordMeasurePithInfo().addAll(recordInfo);
+        LOGGER.info("小节评分频率{}：{}", measureTimeInfo.getMeasureIndex(), JSON.toJSONString(recordInfo));
+        scoreCal(phone, measureTimeInfo, recordInfo);
+    }
+
+    private void scoreCal(String phone, MusicPitchDetailDto measureTimeInfo, List<MusicPitchDetailDto> recordPitchDetails) throws IOException {
+        //相似度
+        BigDecimal intonation = BigDecimal.ZERO;
+        //节奏
+        BigDecimal cadence = BigDecimal.ZERO;
+        //完整度
+        BigDecimal integrity = BigDecimal.ZERO;
+
+        try {
+            //最低有效频率
+            float minValidFrequency = 20;
+
+            //音准匹配数量
+            float intonationNum = 0;
+            //音准匹配误差范围
+            float intonationErrRange = 15;
+            //音准有效阈值
+            float intonationValidDuty = 0.1f;
+
+            //节奏匹配数量
+            float cadenceNum = 0;
+            //节奏有效阈值
+            float cadenceValidDuty = 0.1f;
+
+            //完整性数量
+            float integrityNum = 0;
+            //完整性误差范围
+            float integrityRange = 30;
+            //完整性有效阈值
+            float integrityValidDuty = 0.5f;
+
+            int totalCompareNum = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureTimeInfo.getMeasureIndex()).size();
+
+            for (MusicPitchDetailDto musicXmlInfo : userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureTimeInfo.getMeasureIndex())) {
+                int ot5 = (int) (musicXmlInfo.getDuration()*0.1);
+                int startTimeStamp = musicXmlInfo.getTimeStamp() - ot5;
+                int endTimeStamp = musicXmlInfo.getTimeStamp() + ot5;
+
+                //时间范围内有效音准数量
+                float recordValidIntonationNum = 0;
+                //时间范围内有效节奏数量
+                float cadenceValidNum = 0;
+                //时间范围内有效音频数量
+                float integrityValidNum = 0;
+                //时间范围内匹配次数
+                float compareNum = 0;
+                int faultNum = 0;
+                for (int i = 0; i < recordPitchDetails.size(); i++) {
+                    MusicPitchDetailDto recordInfo = recordPitchDetails.get(i);
+                    if(recordInfo.getTimeStamp()>(startTimeStamp-ot5)&&Math.abs((recordInfo.getFrequency()-musicXmlInfo.getFrequency()))>20){
+                        faultNum++;
+                    }else{
+                        if(faultNum<6)
+                            faultNum = 0;
+                    }
+                    if(faultNum<6){
+                        continue;
+                    }
+                    //如果在时间范围之外直接跳过
+                    if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
+                        continue;
+                    }
+//                    LOGGER.info("{}频率({}-{})：{}, {}", recordInfo.getTimeStamp(), startTimeStamp, endTimeStamp, musicXmlInfo.getFrequency(), recordInfo.getFrequency());
+                    compareNum++;
+                    //如果在最低有效频率以下则跳过
+                    if(recordInfo.getFrequency()<minValidFrequency&&musicXmlInfo.getFrequency()!=-1){
+                        continue;
+                    }
+                    cadenceValidNum++;
+                    if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
+                        continue;
+                    }
+                    //如果频率差值在节奏误差范围内
+                    if(Math.abs(recordInfo.getFrequency()-musicXmlInfo.getFrequency())<=integrityRange){
+                        integrityValidNum++;
+                    }
+                    //如果频率差值在音准误差范围内
+                    if(Math.abs(recordInfo.getFrequency()-musicXmlInfo.getFrequency())<=intonationErrRange){
+                        recordValidIntonationNum++;
+                    }
+                }
+                //有效音频占比
+                float integrityDuty = integrityValidNum/compareNum;
+                //有效音高占比
+                float intonationDuty = recordValidIntonationNum/compareNum;
+                //有效节奏占比
+                float cadenceDuty = cadenceValidNum/compareNum;
+                //节奏
+                if(cadenceDuty>=cadenceValidDuty){
+                    cadenceNum++;
+                    if(intonationDuty>=intonationValidDuty){
+                        intonationNum++;
+                    }
+                    if(integrityDuty>=integrityValidDuty){
+                        integrityNum++;
+                    }
+                }
+            }
+
+            intonation = new BigDecimal(intonationNum).divide(new BigDecimal(totalCompareNum), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
+            cadence = new BigDecimal(cadenceNum).divide(new BigDecimal(totalCompareNum), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
+            integrity = new BigDecimal(integrityNum).divide(new BigDecimal(totalCompareNum), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
+        } catch (ArithmeticException e){
+            LOGGER.info("无musicXml信息");
+        }
+
+        if(userSoundInfoMap.get(phone).getUserScoreMap().containsKey("intonation")){
+            userSoundInfoMap.get(phone).getUserScoreMap().put("intonation", intonation.add(userSoundInfoMap.get(phone).getUserScoreMap().get("intonation")));
+        }else{
+            userSoundInfoMap.get(phone).getUserScoreMap().put("intonation", intonation);
+        }
+
+        if(userSoundInfoMap.get(phone).getUserScoreMap().containsKey("cadence")){
+            userSoundInfoMap.get(phone).getUserScoreMap().put("cadence", cadence.add(userSoundInfoMap.get(phone).getUserScoreMap().get("cadence")));
+        }else{
+            userSoundInfoMap.get(phone).getUserScoreMap().put("cadence", cadence);
+        }
+
+        if(userSoundInfoMap.get(phone).getUserScoreMap().containsKey("integrity")){
+            userSoundInfoMap.get(phone).getUserScoreMap().put("integrity", integrity.add(userSoundInfoMap.get(phone).getUserScoreMap().get("integrity")));
+        }else{
+            userSoundInfoMap.get(phone).getUserScoreMap().put("integrity", integrity);
+        }
+
+        Map<String, BigDecimal> scoreData = new HashMap<>();
+        scoreData.put("intonation", intonation);
+        scoreData.put("cadence", cadence);
+        scoreData.put("integrity", integrity);
+
+        userSoundInfoMap.get(phone).getUserMeasureScoreMap().put(measureTimeInfo.getMeasureIndex(), scoreData);
+
+        WS_CLIENTS.get(phone).sendMessage(new TextMessage(JSON.toJSONString(createPushInfo("measureScore", measureTimeInfo.getMeasureIndex(), intonation, cadence, integrity))));
+    }
+
     /**
      * @describe 计算最终评分
      * @author Joburgess