feat:小节评分

mec-student/pom.xml

@@ -32,6 +32,11 @@
 			<artifactId>spring-cloud-sleuth-zipkin</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-websocket</artifactId>
+		</dependency>
+
 		<!-- swagger-spring-boot -->
 		<dependency>
 			<groupId>com.spring4all</groupId>

mec-student/src/main/java/com/ym/mec/student/config/WebSocketConfig.java

@@ -0,0 +1,40 @@
+package com.ym.mec.student.config;
+
+import com.ym.mec.student.handler.WebSocketHandler;
+import com.ym.mec.student.interceptor.WebSocketHandshakeInterceptor;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.web.socket.config.annotation.EnableWebSocket;
+import org.springframework.web.socket.config.annotation.WebSocketConfigurer;
+import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry;
+import org.springframework.web.socket.server.standard.ServletServerContainerFactoryBean;
+
+/**
+ * @Author Joburgess
+ * @Date 2021/6/8 0008
+ */
+@Configuration
+@EnableWebSocket
+public class WebSocketConfig implements WebSocketConfigurer {
+
+    @Autowired
+    private WebSocketHandler webSocketHandler;
+    @Autowired
+    private WebSocketHandshakeInterceptor webSocketHandshakeInterceptor;
+
+    @Override
+    public void registerWebSocketHandlers(WebSocketHandlerRegistry webSocketHandlerRegistry) {
+        webSocketHandlerRegistry.addHandler(webSocketHandler, "/ws")
+                .addInterceptors(webSocketHandshakeInterceptor);
+    }
+
+    @Bean
+    public ServletServerContainerFactoryBean createWebSocketContainer() {
+        ServletServerContainerFactoryBean container = new ServletServerContainerFactoryBean();
+        container.setMaxTextMessageBufferSize(8192*4);
+        container.setMaxBinaryMessageBufferSize(8192*4);
+        return container;
+    }
+
+}

mec-student/src/main/java/com/ym/mec/student/handler/WebSocketHandler.java

@@ -0,0 +1,323 @@
+package com.ym.mec.student.handler;
+
+import be.tarsos.dsp.AudioDispatcher;
+import be.tarsos.dsp.io.jvm.AudioDispatcherFactory;
+import be.tarsos.dsp.pitch.PitchProcessor;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.ym.mec.biz.dal.dto.MusicPitchDetailDto;
+import com.ym.mec.biz.dal.dto.SoundCompareHelper;
+import com.ym.mec.biz.dal.dto.WavHeader;
+import com.ym.mec.biz.dal.dto.WebSocketInfo;
+import com.ym.mec.biz.service.SoundSocketService;
+import com.ym.mec.common.constant.CommonConstants;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+import org.springframework.web.socket.*;
+import org.springframework.web.socket.handler.AbstractWebSocketHandler;
+
+import javax.sound.sampled.AudioFormat;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.math.BigDecimal;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+/**
+ * @Author Joburgess
+ * @Date 2021/6/9 0009
+ */
+@Service
+public class WebSocketHandler extends AbstractWebSocketHandler {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(WebSocketHandler.class);
+
+    public static final Map<String, WebSocketSession> WS_CLIENTS = new ConcurrentHashMap<>();
+
+    private BigDecimal oneHundred = new BigDecimal(100);
+
+    private static final AudioFormat audioFormat = new AudioFormat(44100, 16, 1, true, false);
+    private static final PitchProcessor.PitchEstimationAlgorithm algo = PitchProcessor.PitchEstimationAlgorithm.FFT_YIN;
+
+    private static final String tmpDir = FileUtils.getTempDirectoryPath() + "/soundCompare/";
+
+    private Map<String, SoundCompareHelper> userSoundInfoMap = new ConcurrentHashMap<>();
+
+    public WebSocketHandler() {
+        super();
+        File soundDir = new File(tmpDir);
+        if(!soundDir.exists()){
+            soundDir.mkdir();
+        }
+    }
+
+    @Override
+    public void afterConnectionEstablished(WebSocketSession session) throws Exception {
+        String phone = session.getPrincipal().getName().split(":")[1];
+        LOGGER.info("{}上线", phone);
+        WS_CLIENTS.put(phone, session);
+        super.afterConnectionEstablished(session);
+    }
+
+    @Override
+    public void handleMessage(WebSocketSession session, WebSocketMessage<?> message) throws Exception {
+        super.handleMessage(session, message);
+    }
+
+    @Override
+    protected void handleTextMessage(WebSocketSession session, TextMessage message) throws Exception {
+        super.handleTextMessage(session, message);
+        String phone = session.getPrincipal().getName().split(":")[1];
+        LOGGER.info("{}: {}", phone, message.getPayload());
+        WebSocketInfo webSocketInfo = JSON.parseObject(message.getPayload(), WebSocketInfo.class);
+        JSONObject bodyObject = (JSONObject) webSocketInfo.getBody();
+
+        String commond = "";
+        if(webSocketInfo.getHeader().containsKey(SoundSocketService.COMMOND)){
+            commond = webSocketInfo.getHeader().get(SoundSocketService.COMMOND);
+        }
+        switch (commond){
+            case SoundSocketService.MUSIC_XML:
+                userSoundInfoMap.put(phone, new SoundCompareHelper());
+                List<MusicPitchDetailDto> musicXmlInfos = JSON.parseArray(bodyObject.getString("musicXmlInfos"), MusicPitchDetailDto.class);
+                userSoundInfoMap.get(phone).setMusicScoreId(bodyObject.getInteger("id"));
+                userSoundInfoMap.get(phone).setMeasureXmlInfoMap(musicXmlInfos.stream().collect(Collectors.groupingBy(MusicPitchDetailDto::getMeasureIndex)));
+                for (Map.Entry<Integer, List<MusicPitchDetailDto>> userMeasureXmlInfoEntry : userSoundInfoMap.get(phone).getMeasureXmlInfoMap().entrySet()) {
+                    MusicPitchDetailDto musicPitchDetailDto = userMeasureXmlInfoEntry.getValue().stream().max(Comparator.comparing(MusicPitchDetailDto::getTimeStamp)).get();
+                    userSoundInfoMap.get(phone).getMeasureEndTime().put(userMeasureXmlInfoEntry.getKey(), musicPitchDetailDto.getTimeStamp()+musicPitchDetailDto.getDuration());
+                }
+                break;
+            case SoundSocketService.RECORD_START:
+                File file = new File(tmpDir+phone + "_"+ userSoundInfoMap.get(phone).getMusicScoreId() +"_"+ LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")) +".wav");
+                userSoundInfoMap.get(phone).setAccessFile(new RandomAccessFile(file, "rw"));
+                break;
+            case SoundSocketService.RECORD_END:
+                if(!CollectionUtils.isEmpty(userSoundInfoMap.get(phone).getMeasureEndTime())){
+                    measureCompare(phone, userSoundInfoMap.get(phone).getMeasureEndTime().keySet().stream().max(Integer::compareTo).get());
+                }
+                calTotalScore(phone);
+                createHeader(phone);
+                break;
+            default:
+                break;
+        }
+    }
+
+    @Override
+    protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) throws Exception {
+        super.handleBinaryMessage(session, message);
+        String phone = session.getPrincipal().getName().split(":")[1];
+        if(Objects.nonNull(userSoundInfoMap.get(phone).getAccessFile())){
+            userSoundInfoMap.get(phone).getAccessFile().write(message.getPayload().array());
+        }
+        List<MusicPitchDetailDto> recordInfo = new ArrayList<>();
+        AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), audioFormat, 256, 128);
+        dispatcher.addAudioProcessor(new PitchProcessor(algo, 44100, 256, (pitchDetectionResult, audioEvent) -> {
+            int timeStamp = (int) (userSoundInfoMap.get(phone).getMeasureStartTime() + audioEvent.getTimeStamp()*1000);
+            float pitch = pitchDetectionResult.getPitch();
+            recordInfo.add(new MusicPitchDetailDto(timeStamp, pitch));
+        }));
+        dispatcher.run();
+        double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(audioFormat.getFrameSize()*audioFormat.getFrameRate())*1000;
+        userSoundInfoMap.get(phone).setMeasureStartTime(recordTime);
+        userSoundInfoMap.get(phone).getRecordMeasurePithInfo().addAll(recordInfo);
+        for (Map.Entry<Integer, Integer> userMeasureEndTimeMapEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) {
+            if(recordTime>userMeasureEndTimeMapEntry.getValue()){
+                measureCompare(phone, userMeasureEndTimeMapEntry.getKey());
+                userSoundInfoMap.get(phone).getMeasureEndTime().remove(userMeasureEndTimeMapEntry.getKey());
+                break;
+            }
+        }
+
+    }
+
+    @Override
+    protected void handlePongMessage(WebSocketSession session, PongMessage message) throws Exception {
+        super.handlePongMessage(session, message);
+        LOGGER.info("心跳信息：{}", new String(message.getPayload().array(), "utf-8"));
+    }
+
+    @Override
+    public void handleTransportError(WebSocketSession session, Throwable exception) throws Exception {
+        super.handleTransportError(session, exception);
+        exception.printStackTrace();
+        String phone = session.getPrincipal().getName().split(":")[1];
+        LOGGER.info("发生了错误,移除客户端: {}", phone);
+        session.close();
+        WS_CLIENTS.remove(phone);
+        createHeader(phone);
+    }
+
+    @Override
+    public void afterConnectionClosed(WebSocketSession session, CloseStatus status) throws Exception {
+        super.afterConnectionClosed(session, status);
+        String phone = session.getPrincipal().getName().split(":")[1];
+        LOGGER.info("{}离线", phone);
+        WS_CLIENTS.remove(phone);
+        createHeader(phone);
+    }
+
+    @Override
+    public boolean supportsPartialMessages() {
+        return super.supportsPartialMessages();
+    }
+
+    private void createHeader(String phone) throws IOException {
+        if(Objects.nonNull(userSoundInfoMap.get(phone).getAccessFile())){
+            return;
+        }
+        RandomAccessFile randomAccessFile = userSoundInfoMap.get(phone).getAccessFile();
+        LOGGER.info("音频时长：{}", randomAccessFile.length()/(audioFormat.getFrameSize()*audioFormat.getFrameRate())*1000);
+        randomAccessFile.seek(0);
+        randomAccessFile.write(WavHeader.getWaveHeader(randomAccessFile.length(), (long) audioFormat.getFrameRate(), audioFormat.getSampleSizeInBits()));
+        randomAccessFile.close();
+        userSoundInfoMap.get(phone).setAccessFile(null);
+        userSoundInfoMap.remove(phone);
+    }
+
+    private void measureCompare(String phone, int measureIndex) throws IOException {
+        //总分
+        BigDecimal score = BigDecimal.ZERO;
+        //相似度
+        BigDecimal intonation = BigDecimal.ZERO;
+        //节奏
+        BigDecimal cadence = BigDecimal.ZERO;
+        //完整度
+        BigDecimal integrity = BigDecimal.ZERO;
+
+        try {
+            //最低有效频率
+            float minValidFrequency = 20;
+
+            //有效音频数量
+            float validNum = 0;
+            //音频有效阈值
+            float validDuty = 0.7f;
+
+            //音准匹配数量
+            float intonationNum = 0;
+            //音准匹配误差范围
+            float intonationErrRange = 70;
+            //音准有效阈值
+            float intonationValidDuty = 0.7f;
+
+            //节奏匹配数量
+            float cadenceNum = 0;
+            //节奏匹配误差范围
+            float cadenceErrRange = 130;
+            //节奏有效阈值
+            float cadenceValidDuty = 0.5f;
+
+            int totalCompareNum = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size();
+
+            for (MusicPitchDetailDto musicXmlInfo : userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex)) {
+                int startTimeStamp = musicXmlInfo.getTimeStamp();
+                int endTimeStamp = musicXmlInfo.getTimeStamp()+musicXmlInfo.getDuration();
+
+                //时间范围内有效音准数量
+                float recordValidIntonationNum = 0;
+                //时间范围内有效节奏数量
+                float cadenceValidNum = 0;
+                //时间范围内有效音频数量
+                float recordValidNum = 0;
+                //时间范围内匹配次数
+                float compareNum = 0;
+                for (MusicPitchDetailDto recordInfo : userSoundInfoMap.get(phone).getRecordMeasurePithInfo()) {
+                    //如果在时间范围之外直接跳过
+                    if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
+                        continue;
+                    }
+                    compareNum++;
+                    //如果在最低有效频率以下则跳过
+                    if(recordInfo.getFrequency()<minValidFrequency){
+                        continue;
+                    }
+                    recordValidNum++;
+                    //如果频率差值在节奏误差范围内
+                    if(recordInfo.getFrequency()-musicXmlInfo.getFrequency()<=cadenceErrRange){
+                        cadenceValidNum++;
+                    }
+                    //如果频率差值在音准误差范围内
+                    if(recordInfo.getFrequency()-musicXmlInfo.getFrequency()<=intonationErrRange){
+                        recordValidIntonationNum++;
+                    }
+                }
+                //有效音频占比
+                float recordValidDuty = recordValidNum/compareNum;
+                if(recordValidDuty<validDuty){
+                    continue;
+                }
+                validNum++;
+                //有效音高占比
+                float intonationDuty = recordValidIntonationNum/compareNum;
+                //有效节奏占比
+                float cadenceDuty = cadenceValidNum/compareNum;
+                if(intonationDuty>=intonationValidDuty){
+                    intonationNum++;
+                }
+                if(cadenceDuty>=cadenceValidDuty){
+                    cadenceNum++;
+                }
+            }
+
+            intonation = new BigDecimal(intonationNum).divide(new BigDecimal(totalCompareNum), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
+            cadence = new BigDecimal(cadenceNum).divide(new BigDecimal(totalCompareNum), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
+            integrity = new BigDecimal(validNum).divide(new BigDecimal(totalCompareNum), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
+        } catch (ArithmeticException e){
+            LOGGER.info("无musicXml信息");
+        }
+
+        if(userSoundInfoMap.get(phone).getUserScoreMap().containsKey("intonation")){
+            userSoundInfoMap.get(phone).getUserScoreMap().put("intonation", intonation.add(userSoundInfoMap.get(phone).getUserScoreMap().get("intonation")));
+        }else{
+            userSoundInfoMap.get(phone).getUserScoreMap().put("intonation", intonation);
+        }
+
+        if(userSoundInfoMap.get(phone).getUserScoreMap().containsKey("cadence")){
+            userSoundInfoMap.get(phone).getUserScoreMap().put("cadence", cadence.add(userSoundInfoMap.get(phone).getUserScoreMap().get("cadence")));
+        }else{
+            userSoundInfoMap.get(phone).getUserScoreMap().put("cadence", cadence);
+        }
+
+        if(userSoundInfoMap.get(phone).getUserScoreMap().containsKey("integrity")){
+            userSoundInfoMap.get(phone).getUserScoreMap().put("integrity", integrity.add(userSoundInfoMap.get(phone).getUserScoreMap().get("integrity")));
+        }else{
+            userSoundInfoMap.get(phone).getUserScoreMap().put("integrity", integrity);
+        }
+        WS_CLIENTS.get(phone).sendMessage(new TextMessage(JSON.toJSONString(createPushInfo("measureScore", measureIndex, intonation, cadence, integrity))));
+//                LOGGER.info("推送评分结果：{}", phone);
+    }
+
+    private void calTotalScore(String phone) throws IOException {
+        int totalCompareNum = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().keySet().size();
+        BigDecimal intonation = userSoundInfoMap.get(phone).getUserScoreMap().get("intonation").divide(new BigDecimal(totalCompareNum), 0, BigDecimal.ROUND_DOWN);
+        BigDecimal cadence = userSoundInfoMap.get(phone).getUserScoreMap().get("cadence").divide(new BigDecimal(totalCompareNum), 0, BigDecimal.ROUND_DOWN);
+        BigDecimal integrity = userSoundInfoMap.get(phone).getUserScoreMap().get("integrity").divide(new BigDecimal(totalCompareNum), 0, BigDecimal.ROUND_DOWN);
+
+        WS_CLIENTS.get(phone).sendMessage(new TextMessage(JSON.toJSONString(createPushInfo("overall", -1, intonation, cadence, integrity))));
+    }
+
+    private WebSocketInfo createPushInfo(String commond, Integer measureIndex,
+                                         BigDecimal intonation, BigDecimal cadence, BigDecimal integrity){
+        WebSocketInfo webSocketInfo = new WebSocketInfo();
+        HashMap<String, String> header = new HashMap<>();
+        header.put("commond", commond);
+        webSocketInfo.setHeader(header);
+        Map<String, Object> result = new HashMap<>();
+        BigDecimal score = intonation.multiply(new BigDecimal(0.45)).add(cadence.multiply(new BigDecimal(0.45))).add(integrity.multiply(new BigDecimal(0.1))).setScale(0, BigDecimal.ROUND_HALF_UP);
+        result.put("score", score);
+        result.put("intonation", intonation);
+        result.put("cadence", cadence);
+        result.put("integrity", integrity);
+        result.put("measureIndex", measureIndex);
+        webSocketInfo.setBody(result);
+        return webSocketInfo;
+    }
+}

mec-student/src/main/java/com/ym/mec/student/interceptor/WebSocketHandshakeInterceptor.java

@@ -0,0 +1,28 @@
+package com.ym.mec.student.interceptor;
+
+import org.springframework.http.server.ServerHttpRequest;
+import org.springframework.http.server.ServerHttpResponse;
+import org.springframework.stereotype.Component;
+import org.springframework.web.socket.WebSocketHandler;
+import org.springframework.web.socket.server.HandshakeInterceptor;
+
+import java.util.Map;
+
+/**
+ * @Author Joburgess
+ * @Date 2021/6/9 0009
+ */
+@Component
+public class WebSocketHandshakeInterceptor implements HandshakeInterceptor {
+
+    @Override
+    public boolean beforeHandshake(ServerHttpRequest serverHttpRequest, ServerHttpResponse serverHttpResponse, WebSocketHandler webSocketHandler, Map<String, Object> map) throws Exception {
+        return true;
+    }
+
+    @Override
+    public void afterHandshake(ServerHttpRequest serverHttpRequest, ServerHttpResponse serverHttpResponse, WebSocketHandler webSocketHandler, Exception e) {
+
+    }
+
+}