feat:云教练音准与节奏计算逻辑调整

mec-biz/src/main/java/com/ym/mec/biz/dal/dto/SoundCompareHelper.java

@@ -4,11 +4,11 @@ import be.tarsos.dsp.AudioEvent;
 import be.tarsos.dsp.SilenceDetector;
 import be.tarsos.dsp.pitch.PitchDetectionHandler;
 import be.tarsos.dsp.pitch.PitchDetectionResult;
+import com.alibaba.fastjson.JSON;
 import com.ym.mec.biz.dal.enums.DeviceTypeEnum;
 import com.ym.mec.biz.dal.enums.HeardLevelEnum;
 import com.ym.mec.biz.service.impl.SoundCompareHandler;
 import io.swagger.annotations.ApiModelProperty;
-import org.springframework.util.CollectionUtils;
 
 import java.io.File;
 import java.io.RandomAccessFile;
@@ -17,6 +17,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * @Author Joburgess
@@ -59,8 +60,8 @@ public class SoundCompareHelper implements PitchDetectionHandler {
     @ApiModelProperty(value = "小节结束时间字典")
     private Map<Integer, MusicPitchDetailDto> measureEndTime = new HashMap<>();
 
-    @ApiModelProperty(value = "录音音频信息")
-    private List<MusicPitchDetailDto> recordMeasurePithInfo = new ArrayList<>();
+//    @ApiModelProperty(value = "录音音频信息")
+//    private List<MusicPitchDetailDto> recordMeasurePithInfo = new ArrayList<>();
 
     @ApiModelProperty(value = "小节分数记录")
     private Map<String, BigDecimal> userScoreMap = new HashMap<>();
@@ -86,6 +87,15 @@ public class SoundCompareHelper implements PitchDetectionHandler {
 
     private String xmlUrl;
 
+    /** 演奏音频数据中频率变更明显持续数量 */
+    private int obviousChangeNum = 0;
+
+    /** 演奏音频中但前小节所有频率数据 */
+    private List<MusicPitchDetailDto> currPitchInfos = new ArrayList<>();
+    private List<MusicPitchDetailDto> currTmpPitchInfos = new ArrayList<>();
+
+    private List<MusicPitchDetailDto> recordMeasurePitchInfos = new ArrayList<>();
+
     /**
      * @describe 分贝检测器
      */
@@ -251,14 +261,6 @@ public class SoundCompareHelper implements PitchDetectionHandler {
         this.musicalNotesPlayStats = musicalNotesPlayStats;
     }
 
-    public List<MusicPitchDetailDto> getRecordMeasurePithInfo() {
-        return recordMeasurePithInfo;
-    }
-
-    public void setRecordMeasurePithInfo(List<MusicPitchDetailDto> recordMeasurePithInfo) {
-        this.recordMeasurePithInfo = recordMeasurePithInfo;
-    }
-
     public Map<String, BigDecimal> getUserScoreMap() {
         return userScoreMap;
     }
@@ -275,29 +277,82 @@ public class SoundCompareHelper implements PitchDetectionHandler {
         this.userMeasureScoreMap = userMeasureScoreMap;
     }
 
+    public List<MusicPitchDetailDto> getRecordMeasurePitchInfos() {
+        return recordMeasurePitchInfos;
+    }
+
+    public void setRecordMeasurePitchInfos(List<MusicPitchDetailDto> recordMeasurePitchInfos) {
+        this.recordMeasurePitchInfos = recordMeasurePitchInfos;
+    }
+
     @Override
     public void handlePitch(PitchDetectionResult pitchDetectionResult, AudioEvent audioEvent) {
         int timeStamp = (int) (measureStartTime + audioEvent.getTimeStamp()*1000);
         float pitch = pitchDetectionResult.getPitch();
+        double decibel = silenceDetector.currentSPL();
+        //初始化偏移时间
         if(offsetTime == -1 && !DeviceTypeEnum.IOS.equals(deviceType) && pitch>0){
-            int preTimeStamp = CollectionUtils.isEmpty(recordMeasurePithInfo)?0:recordMeasurePithInfo.get(recordMeasurePithInfo.size()-1).getTimeStamp();
-            offsetTime = timeStamp - (timeStamp - preTimeStamp)/2;
+            offsetTime = timeStamp;
             for (MusicPitchDetailDto musicXmlInfo : musicXmlInfos) {
                 if(!musicXmlInfo.getDontEvaluating()){
-                    if(offsetTime > musicXmlInfo.getTimeStamp())
+                    if(offsetTime > musicXmlInfo.getTimeStamp()) {
                         offsetTime = offsetTime - musicXmlInfo.getTimeStamp();
+                    }
                     break;
                 }
             }
+            musicXmlInfos.forEach(e->e.setTimeStamp(e.getTimeStamp()+offsetTime));
             for (Map.Entry<Integer, MusicPitchDetailDto> musicPitchDetailDtoEntry : measureEndTime.entrySet()) {
                 musicPitchDetailDtoEntry.getValue().setTimeStamp(musicPitchDetailDtoEntry.getValue().getTimeStamp() + offsetTime);
                 musicPitchDetailDtoEntry.getValue().setEndTimeStamp(musicPitchDetailDtoEntry.getValue().getEndTimeStamp() + offsetTime);
             }
         }
-        if(silenceDetector.currentSPL()< SoundCompareHandler.soundCompareConfig.validDb){
+
+        if(decibel < SoundCompareHandler.soundCompareConfig.validDb){
+            decibel = SoundCompareHandler.soundCompareConfig.validDb;
             pitch = -1;
         }
-//            LOGGER.info("时间：{}, 频率：{}, 分贝：{}", timeStamp, pitch, silenceDetecor.currentSPL());
-        recordMeasurePithInfo.add(new MusicPitchDetailDto(timeStamp, pitch, silenceDetector.currentSPL()));
+        if(pitch==-1){
+            decibel = SoundCompareHandler.soundCompareConfig.validDb;
+        }
+//        SoundCompareHandler.LOGGER.info("时间：{}, 频率：{}, 分贝：{}", timeStamp, pitch, decibel);
+//        recordMeasurePithInfo.add(new MusicPitchDetailDto(timeStamp, pitch, silenceDetector.currentSPL()));
+
+        Double avgPitch = currPitchInfos.stream().skip(1).collect(Collectors.averagingDouble(MusicPitchDetailDto::getFrequency));
+        Double avgDb = currPitchInfos.stream().skip(1).collect(Collectors.averagingDouble(MusicPitchDetailDto::getDecibel));
+        long count = currPitchInfos.stream().filter(p -> p.getFrequency() < 0).count();
+        if(currPitchInfos.size()>0&&count/currPitchInfos.size()<0.3){
+            avgPitch = currPitchInfos.stream().skip(1).filter(p->p.getFrequency()>0).collect(Collectors.averagingDouble(MusicPitchDetailDto::getFrequency));
+        }
+        if(Math.abs(avgPitch-pitch)>10||Math.abs(decibel-avgDb)>5){
+            obviousChangeNum++;
+        }else{
+            currTmpPitchInfos.clear();
+            obviousChangeNum=0;
+        }
+
+        if(obviousChangeNum>1){
+//            SoundCompareHandler.LOGGER.info("----------------------------{}----{}", avgPitch, recordMeasurePitchInfos.size());
+//            SoundCompareHandler.LOGGER.info("----------------------------{}", JSON.toJSONString(currPitchInfos.stream().map(MusicPitchDetailDto::getFrequency).collect(Collectors.toList())));
+//            SoundCompareHandler.LOGGER.info("----------------------------{}", JSON.toJSONString(currTmpPitchInfos.stream().map(MusicPitchDetailDto::getFrequency).collect(Collectors.toList())));
+
+            if(avgPitch>0&&currPitchInfos.size()>2){
+                MusicPitchDetailDto measureDetail = new MusicPitchDetailDto(currPitchInfos.get(0).getTimeStamp(), avgPitch.floatValue(), avgDb);
+                measureDetail.setEndTimeStamp(currTmpPitchInfos.get(0).getTimeStamp());
+                measureDetail.setDuration(measureDetail.getEndTimeStamp()-measureDetail.getTimeStamp());
+                recordMeasurePitchInfos.add(measureDetail);
+            }
+
+            currPitchInfos.clear();
+            currPitchInfos.addAll(currTmpPitchInfos);
+            currTmpPitchInfos.clear();
+            obviousChangeNum = 0;
+        }
+        if(obviousChangeNum>0){
+            currTmpPitchInfos.add(new MusicPitchDetailDto(timeStamp, pitch, decibel));
+        }else{
+            currPitchInfos.add(new MusicPitchDetailDto(timeStamp, pitch, decibel));
+        }
+
     }
 }

mec-biz/src/main/java/com/ym/mec/biz/dal/enums/HeardLevelEnum.java

@@ -3,9 +3,9 @@ package com.ym.mec.biz.dal.enums;
 import com.ym.mec.common.enums.BaseEnum;
 
 public enum HeardLevelEnum implements BaseEnum<String, HeardLevelEnum> {
-    BEGINNER("BEGINNER","入门级", 9, 0.05f, 0),
-    ADVANCED("ADVANCED","进阶级", 3, 0.09f, 0),
-    PERFORMER("PERFORMER","大师级", 1, 0.3f, 0);
+    BEGINNER("BEGINNER","入门级", 0.1f, 0.7f, 0),
+    ADVANCED("ADVANCED","进阶级", 1, 0.9f, 0),
+    PERFORMER("PERFORMER","大师级", 3, 0.93f, 0);
 
     private String code;
 

mec-biz/src/main/java/com/ym/mec/biz/service/impl/SoundCompareHandler.java

@@ -6,6 +6,7 @@ import be.tarsos.dsp.pitch.PitchProcessor;
 import be.tarsos.dsp.util.PitchConverter;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.serializer.SerializerFeature;
 import com.ym.mec.biz.dal.config.SoundCompareConfig;
 import com.ym.mec.biz.dal.dao.SysMusicScoreAccompanimentDao;
 import com.ym.mec.biz.dal.dto.*;
@@ -53,7 +54,7 @@ import static com.ym.mec.biz.service.SoundSocketService.VIDEO_UPDATE;
 @Service
 public class SoundCompareHandler implements WebSocketEventHandler {
 
-    private final Logger LOGGER = LoggerFactory.getLogger(SoundCompareHandler.class);
+    public static final Logger LOGGER = LoggerFactory.getLogger(SoundCompareHandler.class);
 
     private BigDecimal oneHundred = new BigDecimal(100);
 
@@ -218,7 +219,7 @@ public class SoundCompareHandler implements WebSocketEventHandler {
             double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(soundCompareConfig.audioFormat.getFrameSize()*soundCompareConfig.audioFormat.getFrameRate())*1000;
             userSoundInfoMap.get(phone).setMeasureStartTime(recordTime);
             for (Map.Entry<Integer, MusicPitchDetailDto> userMeasureEndTimeMapEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) {
-                if(recordTime>(userMeasureEndTimeMapEntry.getValue().getEndTimeStamp())){
+                if(recordTime>(userMeasureEndTimeMapEntry.getValue().getEndTimeStamp()+100)){
                     if(userMeasureEndTimeMapEntry.getValue().getDontEvaluating()){
                         continue;
                     }else{
@@ -255,6 +256,7 @@ public class SoundCompareHandler implements WebSocketEventHandler {
             musicPitchDetailDtoEntry.getValue().setTimeStamp(musicPitchDetailDtoEntry.getValue().getTimeStamp() + offsetTime);
             musicPitchDetailDtoEntry.getValue().setEndTimeStamp(musicPitchDetailDtoEntry.getValue().getEndTimeStamp() + offsetTime);
         }
+        userSoundInfoMap.get(phone).getMusicXmlInfos().forEach(e->e.setTimeStamp(e.getTimeStamp()+offsetTime));
     }
 
     /**
@@ -341,111 +343,30 @@ public class SoundCompareHandler implements WebSocketEventHandler {
             for (int i = 0; i < userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size(); i++) {
                 MusicPitchDetailDto musicXmlInfo = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).get(i);
 
+                int startTimeStamp = musicXmlInfo.getTimeStamp();
+                int endTimeStamp = musicXmlInfo.getTimeStamp() + musicXmlInfo.getDuration();
+
                 int ot5 = (int) (musicXmlInfo.getDuration()*0.1);
-                int startTimeStamp = musicXmlInfo.getTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime() + ot5;
-                int endTimeStamp = musicXmlInfo.getTimeStamp()  + userSoundInfoMap.get(phone).getOffsetTime() + musicXmlInfo.getDuration() - ot5;
-
-                //时间范围内有效节奏数量
-                float cadenceValidNum = 0;
-                //时间范围内有效音频数量
-                float integrityValidNum = 0;
-                //时间范围内匹配次数
-                float compareNum = 0;
-
-                List<MusicPitchDetailDto> measureSoundPitchInfos = new ArrayList<>();
-
-                for (int j = 0; j < userSoundInfoMap.get(phone).getRecordMeasurePithInfo().size(); j++) {
-                    MusicPitchDetailDto recordInfo = userSoundInfoMap.get(phone).getRecordMeasurePithInfo().get(j);
-                    //如果在时间范围之外直接跳过
-                    if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
-                        continue;
-                    }
-                    measureSoundPitchInfos.add(recordInfo);
-                    compareNum++;
-                    //如果在最低有效频率以下则跳过
-                    if(recordInfo.getFrequency()<soundCompareConfig.validFrequency&&musicXmlInfo.getFrequency()!=-1){
-                        continue;
-                    }
-                    cadenceValidNum++;
-                    //如果频率差值在节奏误差范围内
-//                    if(Math.abs(recordInfo.getFrequency()-musicXmlInfo.getFrequency())<=soundCompareConfig.integrityFrequencyRange){
-//                        integrityValidNum++;
-//                    }
-                }
 
-                //非正常频率次数
-                int errPitchNum = 0;
-                //分贝变化次数
-                int decibelChangeNum = 0;
-
-                if(CollectionUtils.isEmpty(measureSoundPitchInfos)){
-                    userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(musicXmlInfo.getMusicalNotesIndex(), (float) 0);
-                }else{
-                    Map<Integer, Long> collect = measureSoundPitchInfos.stream().map(pitch -> (int)pitch.getFrequency()).collect(Collectors.groupingBy(Integer::intValue, Collectors.counting()));
-                    //出现次数最多的频率
-                    Integer pitch = collect.entrySet().stream().max(Comparator.comparing(e -> e.getValue())).get().getKey();
-                    //当前频率
-                    double cf = -1;
-                    //频率持续数量
-                    int fnum = 0;
-                    //是否演奏中
-                    boolean ing = false;
-                    //当前分贝
-                    double cd = 0;
-                    //分贝变化方向，-1变小，1变大
-                    int dcd = -1;
-                    //分贝持续数量
-                    int dnum = 0;
-                    for (MusicPitchDetailDto musicalNotesPitch : measureSoundPitchInfos) {
-                        //计算频率断层次数
-                        if (Math.abs(musicalNotesPitch.getFrequency() - cf) > 20){
-                            fnum ++;
-                        }
-                        if (fnum>=5){
-                            cf = musicalNotesPitch.getFrequency();
-                            fnum = 0;
-                            if (cf != -1){
-                                errPitchNum ++;
-                                ing = true;
-                                cd = musicalNotesPitch.getDecibel();
-                            }
-                        }
-                        //计算声音大小断层册数
-                        if(ing && Math.abs(musicalNotesPitch.getDecibel() - cd) > 10){
-                            dnum ++;
-                        }
-                        if (dnum > 2){
-                            int tdcd = cd > musicalNotesPitch.getDecibel() ? -1 : 1;
-                            cd = musicalNotesPitch.getDecibel();
-                            dnum = 0;
-                            if (tdcd != dcd) {
-                                decibelChangeNum++;
-                            }
-                            dcd = tdcd;
-                        }
-                    }
-                    userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(musicXmlInfo.getMusicalNotesIndex(), (float) pitch);
-                }
+                List<MusicPitchDetailDto> recordPitchs = userSoundInfoMap.get(phone).getRecordMeasurePitchInfos().stream().filter(m -> Math.abs(startTimeStamp-m.getTimeStamp())<ot5 || (m.getTimeStamp() >= startTimeStamp && m.getTimeStamp() <= endTimeStamp)).collect(Collectors.toList());
 
                 boolean cadenceRight = false;
                 boolean intonationRight = false;
                 boolean integrityRight = false;
 
-                //有效节奏占比
-                float cadenceDuty = cadenceValidNum/compareNum;
-                //如果频率出现断层或这个音量出现断层，则当前音符节奏无效
-                if(errPitchNum>=2 || decibelChangeNum>1){
-                    cadenceDuty = 0;
+                float integrityDuty = 0;
+                if(recordPitchs.size()>0){
+                    integrityDuty = recordPitchs.get(0).getDuration()/(float)musicXmlInfo.getDuration();
                 }
                 //节奏
-                if(cadenceDuty>=userSoundInfoMap.get(phone).getHeardLevel().getCadenceRange()){
+                if(recordPitchs.size()==1||integrityDuty>userSoundInfoMap.get(phone).getHeardLevel().getCadenceRange()){
                     cadenceNum++;
                     cadenceRight = true;
                 }
                 //音准、完成度
-                if (cadenceRight && !CollectionUtils.isEmpty(measureSoundPitchInfos)){
+                if (cadenceRight){
                     //音准
-                    Double avgPitch = measureSoundPitchInfos.stream().filter(pitch -> Math.abs((pitch.getFrequency()-musicXmlInfo.getFrequency()))<15).collect(Collectors.averagingDouble(pitch -> pitch.getFrequency()));
+                    float avgPitch = recordPitchs.get(0).getFrequency();
                     //音分
                     double recordCents = 0;
                     if (avgPitch > 0){
@@ -455,32 +376,24 @@ public class SoundCompareHandler implements WebSocketEventHandler {
                     if(musicXmlInfo.getFrequency()>0){
                         cents =  PitchConverter.hertzToAbsoluteCent(musicXmlInfo.getFrequency());
                     }
-                    double score = 100 - Math.round(Math.abs(cents - recordCents)) + userSoundInfoMap.get(phone).getHeardLevel().getIntonationCentsRange();
+                    double score = 100 - Math.round(Math.abs(cents - recordCents)) + 3;
                     if (score < 0){
                         score = 0;
                     }else if(score > 100){
                         score = 100;
                     }
 
-                    score = Math.pow(score/100f,3)*100;
+                    score = Math.pow(score/100f, userSoundInfoMap.get(phone).getHeardLevel().getIntonationCentsRange())*100;
 
                     intonationScore += score;
-                    musicXmlInfo.setAvgFrequency(avgPitch.floatValue());
+                    musicXmlInfo.setAvgFrequency(avgPitch);
                     intonationRight = score>70;
-
-                    if(score>0){
-                        integrityValidNum = measureSoundPitchInfos.stream().filter(pitch -> Math.abs((pitch.getFrequency()-musicXmlInfo.getFrequency()))<15).count();
-                    }else{
-                        integrityValidNum = 0;
-                    }
                 }
                 //完成度
-                if(integrityValidNum > compareNum){
-                    integrityValidNum = compareNum;
+                if(cadenceRight){
+                    integrityScore += integrityDuty;
+                    integrityRight = integrityDuty>0.7;
                 }
-                float integrityDuty = integrityValidNum/compareNum;
-                integrityScore += integrityDuty;
-                integrityRight = integrityDuty>0.7;
 
                 if(!cadenceRight){
                     userSoundInfoMap.get(phone).getMusicalNotesPlayStats().add(new MusicalNotesPlayStatDto(musicXmlInfo.getMusicalNotesIndex(), MusicalErrorTypeEnum.CADENCE_WRONG));
@@ -540,15 +453,15 @@ public class SoundCompareHandler implements WebSocketEventHandler {
         BigDecimal integrity = BigDecimal.ZERO;
 
         if(currentCompareNum>0){
-            intonation = userSoundInfoMap.get(phone).getUserScoreMap().get("intonation").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_DOWN);
-            cadence = userSoundInfoMap.get(phone).getUserScoreMap().get("cadence").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_DOWN);
-            integrity = userSoundInfoMap.get(phone).getUserScoreMap().get("integrity").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_DOWN);
+            intonation = userSoundInfoMap.get(phone).getUserScoreMap().get("intonation").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_UP);
+            cadence = userSoundInfoMap.get(phone).getUserScoreMap().get("cadence").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_UP);
+            integrity = userSoundInfoMap.get(phone).getUserScoreMap().get("integrity").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_UP);
         }
 
         //计算分数并推送
         createPushInfo(phone, "overall", -1, intonation, cadence, integrity);
 
-        LOGGER.info("评分数据：{}", JSON.toJSONString(userSoundInfoMap.get(phone)));
+        LOGGER.info("评分数据：{}", JSON.toJSONString(userSoundInfoMap.get(phone), SerializerFeature.DisableCircularReferenceDetect));
     }
 
     /**
@@ -581,7 +494,7 @@ public class SoundCompareHandler implements WebSocketEventHandler {
 
         userSoundInfoMap.get(phone).getUserMeasureScoreMap().put(measureIndex, result);
 
-        LOGGER.info("小节频分：{}", JSON.toJSONString(webSocketInfo));
+        LOGGER.info("小节评分：{}", JSON.toJSONString(webSocketInfo));
 
         //推送结果
         WebSocketHandler.sendTextMessage(phone, webSocketInfo);