feat:小节评分

mec-biz/src/main/java/com/ym/mec/biz/dal/dto/MusicPitchDetailDto.java

@@ -0,0 +1,57 @@
+package com.ym.mec.biz.dal.dto;
+
+import io.swagger.annotations.ApiModelProperty;
+
+/**
+ * @Author Joburgess
+ * @Date 2021/6/5 0005
+ */
+public class MusicPitchDetailDto {
+
+    @ApiModelProperty("时间戳ms")
+    private int timeStamp;
+
+    @ApiModelProperty("持续时长ms")
+    private int duration;
+
+    @ApiModelProperty("频率Hz")
+    private float frequency;
+
+    public MusicPitchDetailDto() {
+    }
+
+    public MusicPitchDetailDto(int timeStamp, float frequency) {
+        this.timeStamp = timeStamp;
+        this.frequency = frequency;
+    }
+
+    public MusicPitchDetailDto(int timeStamp, int duration, float frequency) {
+        this.timeStamp = timeStamp;
+        this.duration = duration;
+        this.frequency = frequency;
+    }
+
+    public int getTimeStamp() {
+        return timeStamp;
+    }
+
+    public void setTimeStamp(int timeStamp) {
+        this.timeStamp = timeStamp;
+    }
+
+    public int getDuration() {
+        return duration;
+    }
+
+    public void setDuration(int duration) {
+        this.duration = duration;
+    }
+
+    public float getFrequency() {
+        return frequency;
+    }
+
+    public void setFrequency(float frequency) {
+        this.frequency = frequency;
+    }
+}

mec-biz/src/main/java/com/ym/mec/biz/service/SoundService.java

@@ -1,9 +1,12 @@
 package com.ym.mec.biz.service;
 
+import com.ym.mec.biz.dal.dto.MusicPitchDetailDto;
 import com.ym.mec.common.entity.HttpResponseResult;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.multipart.MultipartFile;
 
+import java.util.List;
+
 /**
  * @Author Joburgess
  * @Date 2021/5/19 0019
@@ -12,4 +15,6 @@ public interface SoundService {
 
     HttpResponseResult compare(MultipartFile record, Integer musicScoreId);
 
+    HttpResponseResult measureCompare(List<MusicPitchDetailDto> musicXmlInfos, MultipartFile record);
+
 }

mec-biz/src/main/java/com/ym/mec/biz/service/impl/SoundServiceImpl.java

@@ -10,8 +10,10 @@ import be.tarsos.dsp.io.TarsosDSPAudioInputStream;
 import be.tarsos.dsp.io.jvm.AudioDispatcherFactory;
 import be.tarsos.dsp.onsets.ComplexOnsetDetector;
 import be.tarsos.dsp.onsets.OnsetHandler;
+import be.tarsos.dsp.pitch.PitchProcessor;
 import com.ym.mec.biz.dal.dao.SysMusicScoreAccompanimentDao;
 import com.ym.mec.biz.dal.dao.SysMusicScoreDao;
+import com.ym.mec.biz.dal.dto.MusicPitchDetailDto;
 import com.ym.mec.biz.dal.entity.SysMusicScore;
 import com.ym.mec.biz.dal.entity.SysMusicScoreAccompaniment;
 import com.ym.mec.biz.service.SoundService;
@@ -29,6 +31,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.sound.sampled.AudioFormat;
+import javax.sound.sampled.AudioSystem;
 import javax.sound.sampled.UnsupportedAudioFileException;
 import java.io.File;
 import java.io.IOException;
@@ -102,6 +105,29 @@ public class SoundServiceImpl implements SoundService {
         return pitchs;
     }
 
+    /**
+     * @describe 音频时域信息提取
+     * @author Joburgess
+     * @date 2021/5/19 0019
+     * @return java.util.List<java.lang.Double>
+     */
+    public List<MusicPitchDetailDto> pitchExtractor(File audioFile) throws IOException, UnsupportedAudioFileException {
+        List<MusicPitchDetailDto> result = new ArrayList<>();
+        PitchProcessor.PitchEstimationAlgorithm algo = PitchProcessor.PitchEstimationAlgorithm.FFT_YIN;
+
+        float samplerate = AudioSystem.getAudioFileFormat(audioFile).getFormat().getSampleRate();
+        int size = 256;
+        int overlap = 128;
+        AudioDispatcher dispatcher = AudioDispatcherFactory.fromFile(audioFile, size, overlap);
+        dispatcher.addAudioProcessor(new PitchProcessor(algo, samplerate, size, (pitchDetectionResult, audioEvent) -> {
+            int timeStamp = (int) (audioEvent.getTimeStamp()*1000);
+            float pitch = pitchDetectionResult.getPitch();
+            result.add(new MusicPitchDetailDto(timeStamp, pitch));
+        }));
+        dispatcher.run();
+        return result;
+    }
+
     private AudioDispatcher getFromByteArray(byte[] bytes, int size, int overlap) throws UnsupportedAudioFileException {
         AudioFormat audioFormat = new AudioFormat(sampleRate, 16, 1, true, false);
         AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(bytes, audioFormat, size, overlap);
@@ -233,4 +259,124 @@ public class SoundServiceImpl implements SoundService {
         result.put("integrity", integrity);
         return BaseController.succeed(result);
     }
+
+    @Override
+    public HttpResponseResult measureCompare(List<MusicPitchDetailDto> musicXmlInfos, MultipartFile record) {
+        Map<String, BigDecimal> result = new HashMap<>();
+
+        String tmpDir = FileUtils.getTempDirectoryPath() + "/";
+        //存储录音文件
+        String filePath_r =  tmpDir + idGeneratorService.generatorId("sound") + "_r.wav";
+
+        LOGGER.info("录音文件大小：{}MB, 保存路径：{}", record.getSize()/1048576f, filePath_r);
+
+        BigDecimal oneHundred = new BigDecimal(100);
+        //总分
+        BigDecimal score = BigDecimal.ZERO;
+        //相似度
+        BigDecimal intonation = BigDecimal.ZERO;
+        //节奏
+        BigDecimal cadence = BigDecimal.ZERO;
+        //完整度
+        BigDecimal integrity = BigDecimal.ZERO;
+        File f_r = null;
+        try {
+            f_r = new File(filePath_r);
+            FileUtils.copyToFile(record.getInputStream(), f_r);
+
+            List<MusicPitchDetailDto> recordInfos = pitchExtractor(f_r);
+
+            //最低有效频率
+            float minValidFrequency = 20;
+
+            //有效音频数量
+            float validNum = 0;
+            //音频有效阈值
+            float validDuty = 0.7f;
+
+            //音准匹配数量
+            float intonationNum = 0;
+            //音准匹配误差范围
+            float intonationErrRange = 10;
+            //音准有效阈值
+            float intonationValidDuty = 0.7f;
+
+            //节奏匹配数量
+            float cadenceNum = 0;
+            //节奏匹配误差范围
+            float cadenceErrRange = 100;
+            //节奏有效阈值
+            float cadenceValidDuty = 0.7f;
+
+            for (MusicPitchDetailDto musicXmlInfo : musicXmlInfos) {
+                int startTimeStamp = musicXmlInfo.getTimeStamp();
+                int endTimeStamp = musicXmlInfo.getTimeStamp()+musicXmlInfo.getDuration();
+
+                //时间范围内有效音准数量
+                float recordValidIntonationNum = 0;
+                //时间范围内有效节奏数量
+                float cadenceValidNum = 0;
+                //时间范围内有效音频数量
+                float recordValidNum = 0;
+                //时间范围内匹配次数
+                float compareNum = 0;
+                for (MusicPitchDetailDto recordInfo : recordInfos) {
+                    //如果在时间范围之外直接跳过
+                    if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
+                        continue;
+                    }
+                    compareNum++;
+                    //如果在最低有效频率以下则跳过
+                    if(recordInfo.getFrequency()<minValidFrequency){
+                        continue;
+                    }
+                    recordValidNum++;
+                    //如果频率差值在节奏误差范围内
+                    if(recordInfo.getFrequency()-musicXmlInfo.getFrequency()<=cadenceErrRange){
+                        cadenceValidNum++;
+                    }
+                    //如果频率差值在音准误差范围内
+                    if(recordInfo.getFrequency()-musicXmlInfo.getFrequency()<=intonationErrRange){
+                        recordValidIntonationNum++;
+                    }
+                }
+                //有效音频占比
+                float recordValidDuty = recordValidNum/compareNum;
+                if(recordValidDuty<validDuty){
+                    continue;
+                }
+                validNum++;
+                //有效音高占比
+                float intonationDuty = recordValidIntonationNum/compareNum;
+                //有效节奏占比
+                float cadenceDuty = cadenceValidNum/compareNum;
+                if(intonationDuty>=intonationValidDuty){
+                    intonationNum++;
+                }
+                if(cadenceDuty>=cadenceValidDuty){
+                    cadenceNum++;
+                }
+            }
+
+            intonation = new BigDecimal(intonationNum).divide(new BigDecimal(musicXmlInfos.size()), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
+            cadence = new BigDecimal(cadenceNum).divide(new BigDecimal(musicXmlInfos.size()), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
+            integrity = new BigDecimal(validNum).divide(new BigDecimal(musicXmlInfos.size()), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
+
+            score = intonation.multiply(new BigDecimal(0.4)).add(cadence.multiply(new BigDecimal(0.4))).add(integrity.multiply(new BigDecimal(0.2))).setScale(0, BigDecimal.ROUND_HALF_UP);
+        } catch (UnsupportedAudioFileException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }finally {
+            if(f_r!=null){
+                f_r.deleteOnExit();
+            }
+        }
+
+        result.put("score", score);
+        result.put("intonation", intonation);
+        result.put("cadence", cadence);
+        result.put("integrity", integrity);
+        return BaseController.succeed(result);
+    }
 }

mec-teacher/src/main/java/com/ym/mec/teacher/controller/SoundController.java

@@ -1,5 +1,6 @@
 package com.ym.mec.teacher.controller;
 
+import com.ym.mec.biz.dal.dto.MusicPitchDetailDto;
 import com.ym.mec.biz.service.SoundService;
 import com.ym.mec.common.controller.BaseController;
 import com.ym.mec.common.entity.HttpResponseResult;
@@ -14,6 +15,8 @@ import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
+import java.util.List;
+
 /**
  * @Author Joburgess
  * @Date 2021/5/19 0019
@@ -32,4 +35,10 @@ public class SoundController extends BaseController {
         return soundService.compare(record, musicScoreId);
     }
 
+    @ApiOperation(value = "小节评分")
+    @PostMapping("measureCompare")
+    public HttpResponseResult measureCompare(List<MusicPitchDetailDto> musicXmlInfos, MultipartFile record){
+        return soundService.measureCompare(musicXmlInfos, record);
+    }
+
 }