feat:智能打分

mec-biz/src/main/java/com/ym/mec/biz/service/SoundService.java

@@ -0,0 +1,14 @@
+package com.ym.mec.biz.service;
+
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.multipart.MultipartFile;
+
+/**
+ * @Author Joburgess
+ * @Date 2021/5/19 0019
+ **/
+public interface SoundService {
+
+    void compare(MultipartFile record, Integer musicScoreId);
+
+}

mec-biz/src/main/java/com/ym/mec/biz/service/impl/SoundServiceImpl.java

@@ -0,0 +1,175 @@
+package com.ym.mec.biz.service.impl;
+
+import be.tarsos.dsp.AudioDispatcher;
+import be.tarsos.dsp.AudioEvent;
+import be.tarsos.dsp.AudioProcessor;
+import be.tarsos.dsp.SilenceDetector;
+import be.tarsos.dsp.beatroot.BeatRootOnsetEventHandler;
+import be.tarsos.dsp.io.PipedAudioStream;
+import be.tarsos.dsp.io.TarsosDSPAudioInputStream;
+import be.tarsos.dsp.io.jvm.AudioDispatcherFactory;
+import be.tarsos.dsp.onsets.ComplexOnsetDetector;
+import be.tarsos.dsp.onsets.OnsetHandler;
+import com.alibaba.fastjson.JSON;
+import com.ym.mec.biz.dal.dao.SysMusicScoreDao;
+import com.ym.mec.biz.dal.entity.SysMusicScore;
+import com.ym.mec.biz.service.SoundService;
+import com.ym.mec.common.exception.BizException;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.web.multipart.MultipartFile;
+
+import javax.sound.sampled.AudioFileFormat;
+import javax.sound.sampled.AudioFormat;
+import javax.sound.sampled.AudioSystem;
+import javax.sound.sampled.UnsupportedAudioFileException;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * @Author Joburgess
+ * @Date 2021/5/19 0019
+ */
+@Service
+public class SoundServiceImpl implements SoundService {
+
+    private float sampleRate = 44100;
+
+    @Autowired
+    private SysMusicScoreDao sysMusicScoreDao;
+
+    /**
+     * @describe 音频节拍信息提取
+     * @author Joburgess
+     * @date 2021/5/19 0019
+     * @return
+     */
+    private List<Double> beatExtractor(byte[] bytes, String url) throws UnsupportedAudioFileException, IOException {
+        List<Double> times = new ArrayList<>();
+        int size = 256;
+        int overlap = 128;
+        AudioDispatcher dispatcher = StringUtils.isBlank(url)?getFromByteArray(bytes, size, overlap):getFromUtl(url, size, overlap);
+
+        ComplexOnsetDetector detector = new ComplexOnsetDetector(size);
+        BeatRootOnsetEventHandler handler = new BeatRootOnsetEventHandler();
+        detector.setHandler(handler);
+
+        dispatcher.addAudioProcessor(detector);
+        dispatcher.run();
+
+        handler.trackBeats(new OnsetHandler() {
+            @Override
+            public void handleOnset(double time, double salience) {
+                times.add(time);
+            }
+        });
+        return times;
+    }
+
+    /**
+     * @describe 音频分贝信息提取
+     * @author Joburgess
+     * @date 2021/5/19 0019
+     * @param bytes: 文件字节
+     * @return java.util.List<java.lang.Double>
+     */
+    private List<Double> soundPressureLevelExtractor(byte[] bytes, String url) throws UnsupportedAudioFileException, IOException {
+        List<Double> pitchs = new ArrayList<>();
+        int size = 2048;
+        int overlap = 0;
+        final SilenceDetector silenceDetecor = new SilenceDetector();
+        AudioDispatcher dispatcher = StringUtils.isBlank(url)?getFromByteArray(bytes, size, overlap):getFromUtl(url, size, overlap);
+        dispatcher.addAudioProcessor(silenceDetecor);
+        dispatcher.addAudioProcessor(new AudioProcessor() {
+            @Override
+            public void processingFinished() {
+            }
+
+            @Override
+            public boolean process(AudioEvent audioEvent) {
+                pitchs.add(Double.isInfinite(silenceDetecor.currentSPL())?0:silenceDetecor.currentSPL());
+                return true;
+            }
+        });
+        dispatcher.run();
+        return pitchs;
+    }
+
+    private AudioDispatcher getFromByteArray(byte[] bytes, int size, int overlap) throws UnsupportedAudioFileException {
+        AudioFormat audioFormat = new AudioFormat(sampleRate, 16, 1, true, false);
+        AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(bytes, audioFormat, size, overlap);
+        return dispatcher;
+    }
+
+    private AudioDispatcher getFromUtl(String url, int size, int overlap) throws UnsupportedAudioFileException, IOException {
+        AudioDispatcher dispatcher = AudioDispatcherFactory.fromURL(new URL(url), size, overlap);
+        return dispatcher;
+    }
+
+    @Override
+    public void compare(MultipartFile record, Integer musicScoreId) {
+        SysMusicScore sysMusicScore = sysMusicScoreDao.get(musicScoreId);
+        if(Objects.isNull(sysMusicScore)|| StringUtils.isBlank(sysMusicScore.getUrl())){
+            throw new BizException("伴奏信息错误");
+        }
+
+        try {
+            AudioFileFormat sourceFile = AudioSystem.getAudioFileFormat(new URL(sysMusicScore.getUrl()));
+            double l_s = sourceFile.getFrameLength()/sourceFile.getFormat().getFrameRate();
+            System.out.printf("源长度：%.2f \n", l_s);
+            AudioFileFormat recordFile = AudioSystem.getAudioFileFormat(record.getInputStream());
+            double r_s = sourceFile.getFrameLength()/sourceFile.getFormat().getFrameRate();
+            System.out.printf("录音长度：%.2f \n", r_s);
+
+            //相似度
+            List<Double> pitchs_s = soundPressureLevelExtractor(null, sysMusicScore.getUrl());
+            List<Double> pitchs_r = soundPressureLevelExtractor(record.getBytes(), null);
+
+            int maxLength = pitchs_s.size();
+            if(pitchs_r.size()<maxLength){
+                maxLength = pitchs_r.size();
+            }
+
+            Double pitchSize = Double.valueOf(0);
+            Double allPitchGap = Double.valueOf(0);
+            for(int i=0;i<maxLength;i++){
+                Double pitch1 = Math.abs(pitchs_s.get(i));
+                Double pitch2 = Math.abs(pitchs_r.get(i));
+                Double pitchGap = Math.abs(pitch1-pitch2);
+                if(pitchGap>pitch1){
+                    pitchGap = pitch1;
+                }
+                allPitchGap+=pitchGap;
+                pitchSize+=pitch1;
+            }
+            Double intonation = 1-(allPitchGap/pitchSize);
+            System.out.printf("音准：%.2f \r\n", intonation);
+
+            //节奏
+            List<Double> times_s = beatExtractor(null, sysMusicScore.getUrl());
+            List<Double> times_r = beatExtractor(record.getBytes(), null);
+
+            float sameTimes = 0;
+            for (Double time1 : times_s) {
+                for (Double time2 : times_r) {
+                    if(Math.abs(time2-time1)<0.1){
+                        sameTimes++;
+                    }
+                }
+            }
+            Double cadence = Double.valueOf(sameTimes/times_s.size());
+            System.out.printf("节奏：%.2f", cadence);
+        } catch (UnsupportedAudioFileException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }finally {
+            return;
+        }
+    }
+}

mec-common/common-core/pom.xml

@@ -66,5 +66,11 @@
 			<artifactId>emoji-java</artifactId>
 		</dependency>
 
+
+		<dependency>
+			<groupId>com.github.dragoon000320</groupId>
+			<artifactId>tarsosdsp</artifactId>
+			<version>1.0</version>
+		</dependency>
 	</dependencies>
 </project>

mec-teacher/src/main/java/com/ym/mec/teacher/controller/SoundController.java

@@ -1,17 +1,36 @@
 package com.ym.mec.teacher.controller;
 
+import be.tarsos.dsp.AudioDispatcher;
+import be.tarsos.dsp.AudioEvent;
+import be.tarsos.dsp.AudioProcessor;
+import be.tarsos.dsp.SilenceDetector;
+import be.tarsos.dsp.beatroot.BeatRootOnsetEventHandler;
+import be.tarsos.dsp.io.jvm.AudioDispatcherFactory;
+import be.tarsos.dsp.onsets.ComplexOnsetDetector;
+import be.tarsos.dsp.onsets.OnsetHandler;
+import com.netflix.discovery.converters.Auto;
+import com.ym.mec.biz.service.SoundService;
 import com.ym.mec.common.controller.BaseController;
 import com.ym.mec.common.entity.HttpResponseResult;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
 
+import javax.servlet.http.HttpServletRequest;
+import javax.sound.sampled.UnsupportedAudioFileException;
+import java.io.File;
+import java.io.IOException;
 import java.text.DecimalFormat;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -23,17 +42,27 @@ import java.util.Map;
 @RestController
 public class SoundController extends BaseController {
 
+    private final Logger LOGGER = LoggerFactory.getLogger(SoundController.class);
+
+    @Autowired
+    private SoundService soundService;
+
     @ApiOperation(value = "评分")
     @PostMapping("compare")
     public HttpResponseResult compare(@RequestParam("record") MultipartFile record, Integer musicScoreId){
         Map<String, String> result = new HashMap<>();
 
+        LOGGER.info("文件名：{}，伴奏编号：{}", record.getOriginalFilename(), musicScoreId);
+
         DecimalFormat df   = new DecimalFormat("######0.00");
 
         result.put("score", df.format(Math.random()*100));
+        result.put("intonation", df.format(Math.random()*100));
         result.put("cadence", df.format(Math.random()*100));
         result.put("integrity", df.format(Math.random()*100));
 
+        soundService.compare(record, musicScoreId);
+
         return succeed(result);
     }