feat:小节评分

mec-biz/src/main/java/com/ym/mec/biz/dal/dto/MusicPitchDetailDto.java

@@ -23,6 +23,28 @@ public class MusicPitchDetailDto {
     @ApiModelProperty("小节数")
     private int measureIndex;
 
+    @ApiModelProperty("音符数")
+    private int musicalNotesIndex;
+
+    @ApiModelProperty("不需要评分")
+    private Boolean dontEvaluating;
+
+    public Boolean getDontEvaluating() {
+        return dontEvaluating;
+    }
+
+    public void setDontEvaluating(Boolean dontEvaluating) {
+        this.dontEvaluating = dontEvaluating;
+    }
+
+    public int getMusicalNotesIndex() {
+        return musicalNotesIndex;
+    }
+
+    public void setMusicalNotesIndex(int musicalNotesIndex) {
+        this.musicalNotesIndex = musicalNotesIndex;
+    }
+
     public int getEndTimeStamp() {
         return endTimeStamp;
     }

mec-biz/src/main/java/com/ym/mec/biz/dal/dto/SoundCompareHelper.java

@@ -35,11 +35,22 @@ public class SoundCompareHelper {
 
     private Map<Integer, Map<String, BigDecimal>> userMeasureScoreMap = new HashMap<>();
 
+    @ApiModelProperty(value = "音符频率字典")
+    private Map<Integer, Float> musicalNotePitchMap = new HashMap<>();
+
     @ApiModelProperty(value = "偏移时间量，解决客户端录音播放不同步导致的声音留白")
     private int offsetTime;
 
     private byte[] preDataArray = new byte[0];
 
+    public Map<Integer, Float> getMusicalNotePitchMap() {
+        return musicalNotePitchMap;
+    }
+
+    public void setMusicalNotePitchMap(Map<Integer, Float> musicalNotePitchMap) {
+        this.musicalNotePitchMap = musicalNotePitchMap;
+    }
+
     public byte[] getPreDataArray() {
         return preDataArray;
     }

mec-biz/src/main/java/com/ym/mec/biz/handler/WebSocketHandler.java

@@ -100,12 +100,16 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
             case SoundSocketService.MUSIC_XML:
                 userSoundInfoMap.put(phone, new SoundCompareHelper());
                 List<MusicPitchDetailDto> musicXmlInfos = JSON.parseArray(bodyObject.getString("musicXmlInfos"), MusicPitchDetailDto.class);
+                musicXmlInfos = musicXmlInfos.stream().filter(m->!m.getDontEvaluating()).collect(Collectors.toList());
                 userSoundInfoMap.get(phone).setMusicScoreId(bodyObject.getInteger("id"));
                 userSoundInfoMap.get(phone).setMeasureXmlInfoMap(musicXmlInfos.stream().collect(Collectors.groupingBy(MusicPitchDetailDto::getMeasureIndex)));
+                musicXmlInfos.forEach(e->userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(e.getMusicalNotesIndex(), e.getFrequency()));
                 for (Map.Entry<Integer, List<MusicPitchDetailDto>> userMeasureXmlInfoEntry : userSoundInfoMap.get(phone).getMeasureXmlInfoMap().entrySet()) {
                     MusicPitchDetailDto firstPitch = userMeasureXmlInfoEntry.getValue().stream().min(Comparator.comparing(MusicPitchDetailDto::getTimeStamp)).get();
                     MusicPitchDetailDto lastPitch = userMeasureXmlInfoEntry.getValue().stream().max(Comparator.comparing(MusicPitchDetailDto::getTimeStamp)).get();
-                    userSoundInfoMap.get(phone).getMeasureEndTime().put(userMeasureXmlInfoEntry.getKey(), new MusicPitchDetailDto(firstPitch.getTimeStamp(), lastPitch.getTimeStamp() + lastPitch.getDuration()));
+                    MusicPitchDetailDto musicPitchDetailDto = new MusicPitchDetailDto(firstPitch.getTimeStamp(), lastPitch.getTimeStamp() + lastPitch.getDuration());
+                    musicPitchDetailDto.setDuration(musicPitchDetailDto.getEndTimeStamp()-musicPitchDetailDto.getTimeStamp());
+                    userSoundInfoMap.get(phone).getMeasureEndTime().put(userMeasureXmlInfoEntry.getKey(), musicPitchDetailDto);
                 }
                 break;
             case SoundSocketService.RECORD_START:
@@ -155,29 +159,28 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
             userSoundInfoMap.get(phone).getAccessFile().write(message.getPayload().array());
         }
 
-//        List<MusicPitchDetailDto> recordInfo = new ArrayList<>();
-//        AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), audioFormat, simpleSize, 128);
-//        dispatcher.addAudioProcessor(new PitchProcessor(algo, simpleRate, simpleSize, (pitchDetectionResult, audioEvent) -> {
-//            int timeStamp = (int) (userSoundInfoMap.get(phone).getMeasureStartTime() + audioEvent.getTimeStamp()*1000);
-//            float pitch = pitchDetectionResult.getPitch();
+        List<MusicPitchDetailDto> recordInfo = new ArrayList<>();
+        AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), audioFormat, simpleSize, 128);
+        dispatcher.addAudioProcessor(new PitchProcessor(algo, simpleRate, simpleSize, (pitchDetectionResult, audioEvent) -> {
+            int timeStamp = (int) (userSoundInfoMap.get(phone).getMeasureStartTime() + audioEvent.getTimeStamp()*1000);
+            float pitch = pitchDetectionResult.getPitch();
 //            LOGGER.info("频率：{}, {}", timeStamp, pitch);
-//            recordInfo.add(new MusicPitchDetailDto(timeStamp, pitch));
-//        }));
-//        dispatcher.run();
-//        if(Objects.isNull(userSoundInfoMap.get(phone).getAccessFile())){
-//            return;
-//        }
+            recordInfo.add(new MusicPitchDetailDto(timeStamp, pitch));
+        }));
+        dispatcher.run();
+        if(Objects.isNull(userSoundInfoMap.get(phone).getAccessFile())){
+            return;
+        }
 
         double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(audioFormat.getFrameSize()*audioFormat.getFrameRate())*1000;
-
         userSoundInfoMap.get(phone).setMeasureStartTime(recordTime);
-//        userSoundInfoMap.get(phone).getRecordMeasurePithInfo().addAll(recordInfo);
+        userSoundInfoMap.get(phone).getRecordMeasurePithInfo().addAll(recordInfo);
         for (Map.Entry<Integer, MusicPitchDetailDto> userMeasureEndTimeMapEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) {
-            int ot = (int) (userMeasureEndTimeMapEntry.getValue().getEndTimeStamp()*0.1);
+            int ot = (int) (userMeasureEndTimeMapEntry.getValue().getDuration()*0.1);
             if(recordTime>(userMeasureEndTimeMapEntry.getValue().getEndTimeStamp()+ot)){
-                LOGGER.info("频分开始{}：{}", recordTime, userSoundInfoMap.get(phone).getAccessFile().length());
-//                measureCompare(phone, userMeasureEndTimeMapEntry.getKey());
-                measureCompare2(phone, userMeasureEndTimeMapEntry.getValue());
+//                LOGGER.info("频分开始{}：{}, {}, {}", recordTime, userSoundInfoMap.get(phone).getAccessFile().length(), ot, JSON.toJSONString(userMeasureEndTimeMapEntry.getValue()));
+                measureCompare(phone, userMeasureEndTimeMapEntry.getKey());
+//                measureCompare2(phone, userMeasureEndTimeMapEntry.getValue());
                 userSoundInfoMap.get(phone).getMeasureEndTime().remove(userMeasureEndTimeMapEntry.getKey());
                 break;
             }
@@ -282,14 +285,14 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
             //音准匹配误差范围
             float intonationErrRange = 15;
             //音准有效阈值
-            float intonationValidDuty = 0.1f;
+            float intonationValidDuty = 0.2f;
 
             //节奏匹配数量
             float cadenceNum = 0;
             //节奏匹配误差范围
             float cadenceErrRange = 30;
             //节奏有效阈值
-            float cadenceValidDuty = 0.1f;
+            float cadenceValidDuty = 0.2f;
 
             //完整性数量
             float integrityNum = 0;
@@ -300,10 +303,11 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
 
             int totalCompareNum = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size();
 
-            for (MusicPitchDetailDto musicXmlInfo : userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex)) {
+            for (int i = 0; i < userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size(); i++) {
+                MusicPitchDetailDto musicXmlInfo = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).get(i);
                 int ot5 = (int) (musicXmlInfo.getDuration()*0.1);
                 int startTimeStamp = musicXmlInfo.getTimeStamp() - ot5;
-                int endTimeStamp = musicXmlInfo.getTimeStamp() + ot5;
+                int endTimeStamp = musicXmlInfo.getTimeStamp() + musicXmlInfo.getDuration() + ot5;
 
                 //时间范围内有效音准数量
                 float recordValidIntonationNum = 0;
@@ -313,22 +317,44 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
                 float integrityValidNum = 0;
                 //时间范围内匹配次数
                 float compareNum = 0;
-                int faultNum = 0;
-                for (int i = 0; i < userSoundInfoMap.get(phone).getRecordMeasurePithInfo().size(); i++) {
-                    MusicPitchDetailDto recordInfo = userSoundInfoMap.get(phone).getRecordMeasurePithInfo().get(i);
-                    if(recordInfo.getTimeStamp()>(startTimeStamp-ot5)&&Math.abs((recordInfo.getFrequency()-musicXmlInfo.getFrequency()))>20){
-                        faultNum++;
-                    }else{
-                        if(faultNum<6)
-                            faultNum = 0;
+
+                boolean newMeasure = false;
+                float preMusicalNotesPitch = 0;
+                if(userSoundInfoMap.get(phone).getMusicalNotePitchMap().containsKey(musicXmlInfo.getMusicalNotesIndex()-1)){
+                    preMusicalNotesPitch = userSoundInfoMap.get(phone).getMusicalNotePitchMap().get(musicXmlInfo.getMusicalNotesIndex()-1);
+                }
+                int newNum = 0;
+
+                for (MusicPitchDetailDto recordInfo : userSoundInfoMap.get(phone).getRecordMeasurePithInfo()) {
+                    if(musicXmlInfo.getMusicalNotesIndex()==0){
+                        newMeasure = true;
                     }
-                    if(faultNum<6){
+                    if(newMeasure){
+                        break;
+                    }
+                    if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
                         continue;
                     }
+                    if(Math.abs(recordInfo.getFrequency()-preMusicalNotesPitch)>10){
+                        newNum++;
+                    }
+                    if(newNum>=2){
+                        newMeasure = true;
+                    }
+                }
+
+                List<Integer> musicalNotesPitchs = new ArrayList<>();
+
+                for (int j = 0; j < userSoundInfoMap.get(phone).getRecordMeasurePithInfo().size(); j++) {
+                    MusicPitchDetailDto recordInfo = userSoundInfoMap.get(phone).getRecordMeasurePithInfo().get(j);
                     //如果在时间范围之外直接跳过
                     if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
                         continue;
                     }
+                    musicalNotesPitchs.add((int) recordInfo.getFrequency());
+                    if(!newMeasure){
+                        continue;
+                    }
 //                    LOGGER.info("{}频率({}-{})：{}, {}", recordInfo.getTimeStamp(), startTimeStamp, endTimeStamp, musicXmlInfo.getFrequency(), recordInfo.getFrequency());
                     compareNum++;
                     //如果在最低有效频率以下则跳过
@@ -348,9 +374,15 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
                         recordValidIntonationNum++;
                     }
                 }
-//                for (MusicPitchDetailDto recordInfo : userSoundInfoMap.get(phone).getRecordMeasurePithInfo()) {
-//
-//                }
+
+                if(CollectionUtils.isEmpty(musicalNotesPitchs)){
+                    userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(musicXmlInfo.getMusicalNotesIndex(), (float) 0);
+                }else{
+                    Map<Integer, Long> collect = musicalNotesPitchs.stream().collect(Collectors.groupingBy(Integer::intValue, Collectors.counting()));
+                    Integer pitch = collect.entrySet().stream().max(Comparator.comparing(e -> e.getValue())).get().getKey();
+                    userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(musicXmlInfo.getMusicalNotesIndex(), (float) pitch);
+                }
+
                 //有效音频占比
                 float integrityDuty = integrityValidNum/compareNum;
                 //有效音高占比
@@ -408,16 +440,14 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
         //小节总时长
         double measureTime = measureTimeInfo.getEndTimeStamp()-measureTimeInfo.getTimeStamp();
         double ot = measureTime * 0.1;
-        if(ot<=measureTimeInfo.getTimeStamp()){
-            measureTime += ot;
-        }
+        measureTime += ot;
         //小节时长占用字节数
         int measureByteNum = (int) (measureTime/1000*(audioFormat.getFrameSize()*audioFormat.getFrameRate()));
 
         List<MusicPitchDetailDto> recordInfo = new ArrayList<>();
         byte[] bytes = new byte[measureByteNum];
         long startOffset = (userSoundInfoMap.get(phone).getAccessFile().length()-measureByteNum);
-        userSoundInfoMap.get(phone).getAccessFile().seek(startOffset-10);
+        userSoundInfoMap.get(phone).getAccessFile().seek(startOffset);
 //        userSoundInfoMap.get(phone).getAccessFile().seek(0);
         userSoundInfoMap.get(phone).getAccessFile().readFully(bytes);