package com.ym.mec.biz.handler; import be.tarsos.dsp.AudioDispatcher; import be.tarsos.dsp.io.jvm.AudioDispatcherFactory; import be.tarsos.dsp.pitch.PitchProcessor; import be.tarsos.dsp.util.PitchConverter; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.ym.mec.biz.dal.config.SoundCompareConfig; import com.ym.mec.biz.dal.dto.*; import com.ym.mec.biz.service.SoundSocketService; import com.ym.mec.biz.service.SysMusicCompareRecordService; import com.ym.mec.common.constant.CommonConstants; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import org.springframework.web.socket.*; import org.springframework.web.socket.handler.AbstractWebSocketHandler; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.math.BigDecimal; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; /** * @Author Joburgess * @Date 2021/6/9 0009 */ @Service public class WebSocketHandler extends AbstractWebSocketHandler { private static final Logger LOGGER = LoggerFactory.getLogger(WebSocketHandler.class); /** * @describe 存储客户端链接 */ public static final Map WS_CLIENTS = new ConcurrentHashMap<>(); private BigDecimal oneHundred = new BigDecimal(100); private final String tmpDir = FileUtils.getTempDirectoryPath() + "/soundCompare/"; /** * @describe 用户对应评分信息 */ private Map userSoundInfoMap = new ConcurrentHashMap<>(); /** * @describe 音频处理参数 */ private SoundCompareConfig soundCompareConfig = new SoundCompareConfig(); @Autowired private SysMusicCompareRecordService sysMusicCompareRecordService; public WebSocketHandler() { super(); File soundDir = new File(tmpDir); if(!soundDir.exists()){ soundDir.mkdir(); } } @Override public void afterConnectionEstablished(WebSocketSession session) throws Exception { String phone = session.getPrincipal().getName().split(":")[1]; LOGGER.info("{}上线", phone); WS_CLIENTS.put(phone, new WebSocketClientDetail(session, new Date())); super.afterConnectionEstablished(session); } @Override public void handleMessage(WebSocketSession session, WebSocketMessage message) throws Exception { super.handleMessage(session, message); } @Override protected void handleTextMessage(WebSocketSession session, TextMessage message) throws Exception { String phone = session.getPrincipal().getName().split(":")[1]; LOGGER.info("{}: {}", phone, message.getPayload()); WebSocketInfo webSocketInfo = JSON.parseObject(message.getPayload(), WebSocketInfo.class); JSONObject bodyObject = (JSONObject) webSocketInfo.getBody(); String commond = ""; if(webSocketInfo.getHeader().containsKey(SoundSocketService.COMMOND)){ commond = webSocketInfo.getHeader().get(SoundSocketService.COMMOND); } switch (commond){ case SoundSocketService.MUSIC_XML: userSoundInfoMap.put(phone, new SoundCompareHelper()); List musicXmlInfos = JSON.parseArray(bodyObject.getString("musicXmlInfos"), MusicPitchDetailDto.class); userSoundInfoMap.get(phone).setMusicXmlInfos(musicXmlInfos); musicXmlInfos = musicXmlInfos.stream().filter(m->!m.getDontEvaluating()).collect(Collectors.toList()); userSoundInfoMap.get(phone).setMusicScoreId(bodyObject.getInteger("id")); userSoundInfoMap.get(phone).setMeasureXmlInfoMap(musicXmlInfos.stream().collect(Collectors.groupingBy(MusicPitchDetailDto::getMeasureIndex))); musicXmlInfos.forEach(e->userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(e.getMusicalNotesIndex(), e.getFrequency())); for (Map.Entry> userMeasureXmlInfoEntry : userSoundInfoMap.get(phone).getMeasureXmlInfoMap().entrySet()) { MusicPitchDetailDto firstPitch = userMeasureXmlInfoEntry.getValue().stream().min(Comparator.comparing(MusicPitchDetailDto::getTimeStamp)).get(); MusicPitchDetailDto lastPitch = userMeasureXmlInfoEntry.getValue().stream().max(Comparator.comparing(MusicPitchDetailDto::getTimeStamp)).get(); long dc = userMeasureXmlInfoEntry.getValue().stream().filter(m -> m.getDontEvaluating()).count(); MusicPitchDetailDto musicPitchDetailDto = new MusicPitchDetailDto(firstPitch.getTimeStamp(), lastPitch.getTimeStamp() + lastPitch.getDuration()); musicPitchDetailDto.setDuration(musicPitchDetailDto.getEndTimeStamp()-musicPitchDetailDto.getTimeStamp()); musicPitchDetailDto.setDontEvaluating(dc == userMeasureXmlInfoEntry.getValue().size()); userSoundInfoMap.get(phone).getMeasureEndTime().put(userMeasureXmlInfoEntry.getKey(), musicPitchDetailDto); } break; case SoundSocketService.RECORD_START: if(!userSoundInfoMap.containsKey(phone)){ break; } File file = new File(tmpDir+phone + "_"+ userSoundInfoMap.get(phone).getMusicScoreId() +"_"+ LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")) +".wav"); userSoundInfoMap.get(phone).setAccessFile(new RandomAccessFile(file, "rw")); userSoundInfoMap.get(phone).setRecordFilePath(file.getAbsolutePath()); break; case SoundSocketService.RECORD_END: if(!userSoundInfoMap.containsKey(phone)){ break; } if(!CollectionUtils.isEmpty(userSoundInfoMap.get(phone).getMeasureEndTime())){ Integer lastMeasureIndex = userSoundInfoMap.get(phone).getMeasureEndTime().keySet().stream().min(Integer::compareTo).get(); double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(soundCompareConfig.audioFormat.getFrameSize()*soundCompareConfig.audioFormat.getFrameRate())*1000; //如果结束时时长大于某小节，则此小节需要评分 if(recordTime>userSoundInfoMap.get(phone).getMeasureEndTime().get(lastMeasureIndex).getEndTimeStamp()){ measureCompare(phone, lastMeasureIndex); userSoundInfoMap.get(phone).getMeasureEndTime().remove(lastMeasureIndex); } } calTotalScore(phone); createHeader(phone); break; case SoundSocketService.RECORD_CANCEL: createHeader(phone); break; case SoundSocketService.PROXY_MESSAGE: // if(bodyObject.containsKey(SoundSocketService.OFFSET_TIME)){ // int offsetTime = bodyObject.getIntValue(SoundSocketService.OFFSET_TIME); // calOffsetTime(phone, offsetTime); // } break; default: break; } } @Override protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) throws Exception { String phone = session.getPrincipal().getName().split(":")[1]; if(!userSoundInfoMap.containsKey(phone)){ return; } if(Objects.nonNull(userSoundInfoMap.get(phone).getAccessFile())){ userSoundInfoMap.get(phone).getAccessFile().write(message.getPayload().array()); } AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), soundCompareConfig.audioFormat, soundCompareConfig.simpleSize, soundCompareConfig.overlap); dispatcher.addAudioProcessor(soundCompareConfig.silenceDetector); dispatcher.addAudioProcessor(new PitchProcessor(soundCompareConfig.algo, soundCompareConfig.simpleRate, soundCompareConfig.simpleSize, (pitchDetectionResult, audioEvent) -> { int timeStamp = (int) (userSoundInfoMap.get(phone).getMeasureStartTime() + audioEvent.getTimeStamp()*1000); float pitch = pitchDetectionResult.getPitch(); if(pitch>0 && userSoundInfoMap.get(phone).getOffsetTime() == -1){ int preTimeStamp = CollectionUtils.isEmpty(userSoundInfoMap.get(phone).getRecordMeasurePithInfo())?0:userSoundInfoMap.get(phone).getRecordMeasurePithInfo().get(userSoundInfoMap.get(phone).getRecordMeasurePithInfo().size()-1).getTimeStamp(); calOffsetTime(phone, timeStamp - (timeStamp - preTimeStamp)/2); } if(soundCompareConfig.silenceDetector.currentSPL() userMeasureEndTimeMapEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) { if(recordTime>(userMeasureEndTimeMapEntry.getValue().getEndTimeStamp())){ if(userMeasureEndTimeMapEntry.getValue().getDontEvaluating()){ continue; }else{ measureCompare(phone, userMeasureEndTimeMapEntry.getKey()); } userSoundInfoMap.get(phone).getMeasureEndTime().remove(userMeasureEndTimeMapEntry.getKey()); break; } } } @Override protected void handlePongMessage(WebSocketSession session, PongMessage message) throws Exception { super.handlePongMessage(session, message); LOGGER.info("心跳信息：{}", new String(message.getPayload().array(), "utf-8")); } @Override public void handleTransportError(WebSocketSession session, Throwable exception) throws Exception { String phone = session.getPrincipal().getName().split(":")[1]; session.close(); if(!WS_CLIENTS.containsKey(phone)){ return; } exception.printStackTrace(); LOGGER.info("发生了错误,移除客户端: {}", phone); WS_CLIENTS.remove(phone); userSoundInfoMap.remove(phone); createHeader(phone); } @Override public void afterConnectionClosed(WebSocketSession session, CloseStatus status) throws Exception { super.afterConnectionClosed(session, status); String phone = session.getPrincipal().getName().split(":")[1]; LOGGER.info("{}离线", phone); createHeader(phone); WS_CLIENTS.remove(phone); userSoundInfoMap.remove(phone); } @Override public boolean supportsPartialMessages() { return super.supportsPartialMessages(); } /** * @describe 处理时间偏移 * @author Joburgess * @date 2021/7/5 0005 * @param phone: * @param offsetTime: * @return void */ private void calOffsetTime(String phone, int offsetTime){ userSoundInfoMap.get(phone).setOffsetTime(offsetTime); for (Map.Entry musicPitchDetailDtoEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) { musicPitchDetailDtoEntry.getValue().setTimeStamp(musicPitchDetailDtoEntry.getValue().getTimeStamp() + offsetTime); musicPitchDetailDtoEntry.getValue().setEndTimeStamp(musicPitchDetailDtoEntry.getValue().getEndTimeStamp() + offsetTime); } } /** * @describe 保存录音数据，并生成wav头信息 * @author Joburgess * @date 2021/6/25 0025 * @param phone: * @return void */ private void createHeader(String phone) throws IOException { if(!userSoundInfoMap.containsKey(phone)){ return; } if(Objects.nonNull(userSoundInfoMap.get(phone).getAccessFile())){ RandomAccessFile randomAccessFile = userSoundInfoMap.get(phone).getAccessFile(); LOGGER.info("音频时长：{}", randomAccessFile.length()/(soundCompareConfig.audioFormat.getFrameSize()*soundCompareConfig.audioFormat.getFrameRate())*1000); randomAccessFile.seek(0); randomAccessFile.write(WavHeader.getWaveHeader(randomAccessFile.length(), (long) soundCompareConfig.audioFormat.getFrameRate(), soundCompareConfig.audioFormat.getSampleSizeInBits())); randomAccessFile.close(); userSoundInfoMap.get(phone).setAccessFile(null); } // userSoundInfoMap.get(phone).setRecordMeasurePithInfo(null); userSoundInfoMap.remove(phone); } /** * @describe 数据比对，生成分数 * @author Joburgess * @date 2021/6/25 0025 * @param phone: * @param measureIndex: * @return void */ private void measureCompare(String phone, int measureIndex) throws IOException { if (userSoundInfoMap.get(phone).getOffsetTime() == -1){ userSoundInfoMap.get(phone).setOffsetTime(0); } //相似度 BigDecimal intonation = BigDecimal.ZERO; //节奏 BigDecimal cadence = BigDecimal.ZERO; //完整度 BigDecimal integrity = BigDecimal.ZERO; try { //音准分数 float intonationScore = 0; //节奏匹配数量 float cadenceNum = 0; //完整性分数 float integrityScore = 0; int totalCompareNum = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size(); for (int i = 0; i < userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size(); i++) { MusicPitchDetailDto musicXmlInfo = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).get(i); int ot5 = (int) (musicXmlInfo.getDuration()*0.1); int startTimeStamp = musicXmlInfo.getTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime() + ot5; int endTimeStamp = musicXmlInfo.getTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime() + musicXmlInfo.getDuration() - ot5; //时间范围内有效节奏数量 float cadenceValidNum = 0; //时间范围内有效音频数量 float integrityValidNum = 0; //时间范围内匹配次数 float compareNum = 0; List measureSoundPitchInfos = new ArrayList<>(); for (int j = 0; j < userSoundInfoMap.get(phone).getRecordMeasurePithInfo().size(); j++) { MusicPitchDetailDto recordInfo = userSoundInfoMap.get(phone).getRecordMeasurePithInfo().get(j); //如果在时间范围之外直接跳过 if(recordInfo.getTimeStamp()endTimeStamp){ continue; } measureSoundPitchInfos.add(recordInfo); compareNum++; //如果在最低有效频率以下则跳过 if(recordInfo.getFrequency() collect = measureSoundPitchInfos.stream().map(pitch -> (int)pitch.getFrequency()).collect(Collectors.groupingBy(Integer::intValue, Collectors.counting())); //出现次数最多的频率 Integer pitch = collect.entrySet().stream().max(Comparator.comparing(e -> e.getValue())).get().getKey(); //当前频率 double cf = -1; //频率持续数量 int fnum = 0; //是否演奏中 boolean ing = false; //当前分贝 double cd = 0; //分贝变化方向，-1变小，1变大 int dcd = -1; //分贝持续数量 int dnum = 0; for (MusicPitchDetailDto musicalNotesPitch : measureSoundPitchInfos) { //计算频率断层次数 if (Math.abs(musicalNotesPitch.getFrequency() - cf) > 20){ fnum ++; } if (fnum>=5){ cf = musicalNotesPitch.getFrequency(); fnum = 0; if (cf != -1){ errPitchNum ++; ing = true; cd = musicalNotesPitch.getDecibel(); } } //计算声音大小断层册数 if(ing && Math.abs(musicalNotesPitch.getDecibel() - cd) > 10){ dnum ++; } if (dnum > 2){ int tdcd = cd > musicalNotesPitch.getDecibel() ? -1 : 1; cd = musicalNotesPitch.getDecibel(); dnum = 0; if (tdcd != dcd) { decibelChangeNum++; } dcd = tdcd; } } userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(musicXmlInfo.getMusicalNotesIndex(), (float) pitch); } //有效节奏占比 float cadenceDuty = cadenceValidNum/compareNum; //如果频率出现断层或这个音量出现断层，则当前音符节奏无效 if(errPitchNum>=2 || decibelChangeNum>1){ cadenceDuty = 0; } //节奏 if(cadenceDuty>=soundCompareConfig.cadenceValidDuty){ cadenceNum++; } //音准 if (!CollectionUtils.isEmpty(measureSoundPitchInfos)){ Double avgPitch = measureSoundPitchInfos.stream().filter(pitch -> Math.abs((pitch.getFrequency()-musicXmlInfo.getFrequency()))<5).collect(Collectors.averagingDouble(pitch -> pitch.getFrequency())); //音分 double recordCents = 0; if (avgPitch > 0){ recordCents = PitchConverter.hertzToAbsoluteCent(avgPitch); } double cents = PitchConverter.hertzToAbsoluteCent(musicXmlInfo.getFrequency()); double score = 100 - Math.round(Math.abs(cents - recordCents)) + soundCompareConfig.intonationCentsRange; if (score < 0){ score = 0; }else if(score > 100){ score = 100; } intonationScore += score; musicXmlInfo.setAvgFrequency(avgPitch.floatValue()); } //完成度 if(integrityValidNum>0){ integrityValidNum = integrityValidNum; } if(integrityValidNum > compareNum){ integrityValidNum = compareNum; } float integrityDuty = integrityValidNum/compareNum; integrityScore += integrityDuty; } BigDecimal measureNum = new BigDecimal(totalCompareNum); intonation = new BigDecimal(intonationScore).divide(measureNum, CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).setScale(0, BigDecimal.ROUND_UP); cadence = new BigDecimal(cadenceNum).divide(measureNum, CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_UP); integrity = new BigDecimal(integrityScore).divide(measureNum, CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_UP); } catch (ArithmeticException e){ LOGGER.info("无musicXml信息"); } if(userSoundInfoMap.get(phone).getUserScoreMap().containsKey("intonation")){ userSoundInfoMap.get(phone).getUserScoreMap().put("intonation", intonation.add(userSoundInfoMap.get(phone).getUserScoreMap().get("intonation"))); }else{ userSoundInfoMap.get(phone).getUserScoreMap().put("intonation", intonation); } if(userSoundInfoMap.get(phone).getUserScoreMap().containsKey("cadence")){ userSoundInfoMap.get(phone).getUserScoreMap().put("cadence", cadence.add(userSoundInfoMap.get(phone).getUserScoreMap().get("cadence"))); }else{ userSoundInfoMap.get(phone).getUserScoreMap().put("cadence", cadence); } if(userSoundInfoMap.get(phone).getUserScoreMap().containsKey("integrity")){ userSoundInfoMap.get(phone).getUserScoreMap().put("integrity", integrity.add(userSoundInfoMap.get(phone).getUserScoreMap().get("integrity"))); }else{ userSoundInfoMap.get(phone).getUserScoreMap().put("integrity", integrity); } //计算分数并推送 createPushInfo(phone, "measureScore", measureIndex, intonation, cadence, integrity); } /** * @describe 计算最终评分 * @author Joburgess * @date 2021/6/25 0025 * @param phone: * @return void */ private void calTotalScore(String phone) throws IOException { int totalCompareNum = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().keySet().size(); int currentCompareNum = totalCompareNum-userSoundInfoMap.get(phone).getMeasureEndTime().keySet().size(); BigDecimal intonation = BigDecimal.ZERO; BigDecimal cadence = BigDecimal.ZERO; BigDecimal integrity = BigDecimal.ZERO; if(currentCompareNum>0){ intonation = userSoundInfoMap.get(phone).getUserScoreMap().get("intonation").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_DOWN); cadence = userSoundInfoMap.get(phone).getUserScoreMap().get("cadence").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_DOWN); integrity = userSoundInfoMap.get(phone).getUserScoreMap().get("integrity").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_DOWN); } //计算分数并推送 createPushInfo(phone, "overall", -1, intonation, cadence, integrity); //存储评分数据 sysMusicCompareRecordService.saveMusicCompareData(phone, userSoundInfoMap.get(phone)); LOGGER.info("评分数据：{}", JSON.toJSONString(userSoundInfoMap.get(phone))); } /** * @describe 生成评分结果 * @author Joburgess * @date 2021/6/25 0025 * @param command: * @param measureIndex: * @param intonation: * @param cadence: * @param integrity: * @return com.ym.mec.biz.dal.dto.WebSocketInfo */ private WebSocketInfo createPushInfo(String phone, String command, Integer measureIndex, BigDecimal intonation, BigDecimal cadence, BigDecimal integrity) throws IOException { WebSocketInfo webSocketInfo = new WebSocketInfo(); HashMap header = new HashMap<>(); header.put("commond", command); webSocketInfo.setHeader(header); Map result = new HashMap<>(); // BigDecimal score = intonation.multiply(new BigDecimal(0.5)).add(cadence.multiply(new BigDecimal(0.5))).setScale(0, BigDecimal.ROUND_HALF_UP); BigDecimal score = intonation.add(cadence).add(integrity).divide(new BigDecimal(3), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).setScale(0, BigDecimal.ROUND_UP); // BigDecimal score = integrity.setScale(0, BigDecimal.ROUND_HALF_UP); result.put("score", score); result.put("intonation", intonation); result.put("cadence", cadence); result.put("integrity", integrity); result.put("measureIndex", measureIndex); webSocketInfo.setBody(result); userSoundInfoMap.get(phone).getUserMeasureScoreMap().put(measureIndex, result); LOGGER.info("小节频分：{}", JSON.toJSONString(webSocketInfo)); //推送结果 WS_CLIENTS.get(phone).getSession().sendMessage(new TextMessage(JSON.toJSONString(webSocketInfo))); return webSocketInfo; } }