|
@@ -1,16 +1,13 @@
|
|
|
package com.ym.mec.biz.handler;
|
|
|
|
|
|
import be.tarsos.dsp.AudioDispatcher;
|
|
|
-import be.tarsos.dsp.SilenceDetector;
|
|
|
import be.tarsos.dsp.io.jvm.AudioDispatcherFactory;
|
|
|
import be.tarsos.dsp.pitch.PitchProcessor;
|
|
|
import be.tarsos.dsp.util.PitchConverter;
|
|
|
import com.alibaba.fastjson.JSON;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
-import com.ym.mec.biz.dal.dto.MusicPitchDetailDto;
|
|
|
-import com.ym.mec.biz.dal.dto.SoundCompareHelper;
|
|
|
-import com.ym.mec.biz.dal.dto.WavHeader;
|
|
|
-import com.ym.mec.biz.dal.dto.WebSocketInfo;
|
|
|
+import com.ym.mec.biz.dal.config.SoundCompareConfig;
|
|
|
+import com.ym.mec.biz.dal.dto.*;
|
|
|
import com.ym.mec.biz.service.SoundSocketService;
|
|
|
import com.ym.mec.biz.service.SysMusicCompareRecordService;
|
|
|
import com.ym.mec.common.constant.CommonConstants;
|
|
@@ -23,8 +20,6 @@ import org.springframework.util.CollectionUtils;
|
|
|
import org.springframework.web.socket.*;
|
|
|
import org.springframework.web.socket.handler.AbstractWebSocketHandler;
|
|
|
|
|
|
-import javax.sound.sampled.AudioFormat;
|
|
|
-import javax.sound.sampled.UnsupportedAudioFileException;
|
|
|
import java.io.File;
|
|
|
import java.io.IOException;
|
|
|
import java.io.RandomAccessFile;
|
|
@@ -44,22 +39,23 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
|
|
|
private static final Logger LOGGER = LoggerFactory.getLogger(WebSocketHandler.class);
|
|
|
|
|
|
- //存储客户端链接
|
|
|
- public static final Map<String, WebSocketSession> WS_CLIENTS = new ConcurrentHashMap<>();
|
|
|
-
|
|
|
- private final BigDecimal oneHundred = new BigDecimal(100);
|
|
|
- private final float simpleRate = 44100;
|
|
|
- private final int simpleSize = 1024;
|
|
|
- private final int overlap = 256;
|
|
|
+ /**
|
|
|
+ * @describe 存储客户端链接
|
|
|
+ */
|
|
|
+ public static final Map<String, WebSocketClientDetail> WS_CLIENTS = new ConcurrentHashMap<>();
|
|
|
|
|
|
- private final AudioFormat audioFormat = new AudioFormat(simpleRate, 16, 1, true, false);
|
|
|
- private static final PitchProcessor.PitchEstimationAlgorithm algo = PitchProcessor.PitchEstimationAlgorithm.FFT_YIN;
|
|
|
- private final SilenceDetector silenceDetecor = new SilenceDetector();
|
|
|
+ private BigDecimal oneHundred = new BigDecimal(100);
|
|
|
|
|
|
private final String tmpDir = FileUtils.getTempDirectoryPath() + "/soundCompare/";
|
|
|
|
|
|
- //用户对应评分信息
|
|
|
+ /**
|
|
|
+ * @describe 用户对应评分信息
|
|
|
+ */
|
|
|
private Map<String, SoundCompareHelper> userSoundInfoMap = new ConcurrentHashMap<>();
|
|
|
+ /**
|
|
|
+ * @describe 音频处理参数
|
|
|
+ */
|
|
|
+ private SoundCompareConfig soundCompareConfig = new SoundCompareConfig();
|
|
|
|
|
|
@Autowired
|
|
|
private SysMusicCompareRecordService sysMusicCompareRecordService;
|
|
@@ -76,7 +72,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
public void afterConnectionEstablished(WebSocketSession session) throws Exception {
|
|
|
String phone = session.getPrincipal().getName().split(":")[1];
|
|
|
LOGGER.info("{}上线", phone);
|
|
|
- WS_CLIENTS.put(phone, session);
|
|
|
+ WS_CLIENTS.put(phone, new WebSocketClientDetail(session, new Date()));
|
|
|
super.afterConnectionEstablished(session);
|
|
|
}
|
|
|
|
|
@@ -121,6 +117,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
}
|
|
|
File file = new File(tmpDir+phone + "_"+ userSoundInfoMap.get(phone).getMusicScoreId() +"_"+ LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")) +".wav");
|
|
|
userSoundInfoMap.get(phone).setAccessFile(new RandomAccessFile(file, "rw"));
|
|
|
+ userSoundInfoMap.get(phone).setRecordFilePath(file.getAbsolutePath());
|
|
|
break;
|
|
|
case SoundSocketService.RECORD_END:
|
|
|
if(!userSoundInfoMap.containsKey(phone)){
|
|
@@ -128,7 +125,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
}
|
|
|
if(!CollectionUtils.isEmpty(userSoundInfoMap.get(phone).getMeasureEndTime())){
|
|
|
Integer lastMeasureIndex = userSoundInfoMap.get(phone).getMeasureEndTime().keySet().stream().min(Integer::compareTo).get();
|
|
|
- double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(audioFormat.getFrameSize()*audioFormat.getFrameRate())*1000;
|
|
|
+ double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(soundCompareConfig.audioFormat.getFrameSize()*soundCompareConfig.audioFormat.getFrameRate())*1000;
|
|
|
//如果结束时时长大于某小节,则此小节需要评分
|
|
|
if(recordTime>userSoundInfoMap.get(phone).getMeasureEndTime().get(lastMeasureIndex).getEndTimeStamp()){
|
|
|
measureCompare(phone, lastMeasureIndex);
|
|
@@ -162,31 +159,30 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
userSoundInfoMap.get(phone).getAccessFile().write(message.getPayload().array());
|
|
|
}
|
|
|
|
|
|
- AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), audioFormat, simpleSize, overlap);
|
|
|
- dispatcher.addAudioProcessor(silenceDetecor);
|
|
|
- dispatcher.addAudioProcessor(new PitchProcessor(algo, simpleRate, simpleSize, (pitchDetectionResult, audioEvent) -> {
|
|
|
+ AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), soundCompareConfig.audioFormat, soundCompareConfig.simpleSize, soundCompareConfig.overlap);
|
|
|
+ dispatcher.addAudioProcessor(soundCompareConfig.silenceDetector);
|
|
|
+ dispatcher.addAudioProcessor(new PitchProcessor(soundCompareConfig.algo, soundCompareConfig.simpleRate, soundCompareConfig.simpleSize, (pitchDetectionResult, audioEvent) -> {
|
|
|
int timeStamp = (int) (userSoundInfoMap.get(phone).getMeasureStartTime() + audioEvent.getTimeStamp()*1000);
|
|
|
float pitch = pitchDetectionResult.getPitch();
|
|
|
if(pitch>0 && userSoundInfoMap.get(phone).getOffsetTime() == -1){
|
|
|
int preTimeStamp = CollectionUtils.isEmpty(userSoundInfoMap.get(phone).getRecordMeasurePithInfo())?0:userSoundInfoMap.get(phone).getRecordMeasurePithInfo().get(userSoundInfoMap.get(phone).getRecordMeasurePithInfo().size()-1).getTimeStamp();
|
|
|
calOffsetTime(phone, timeStamp - (timeStamp - preTimeStamp)/2);
|
|
|
}
|
|
|
- if(silenceDetecor.currentSPL()<-70){
|
|
|
+ if(soundCompareConfig.silenceDetector.currentSPL()<soundCompareConfig.validDb){
|
|
|
pitch = -1;
|
|
|
}
|
|
|
-// LOGGER.info("时间:{}, 频率:{}, 分贝:{}, 音分:{}", timeStamp, pitch, silenceDetecor.currentSPL(), cents);
|
|
|
- userSoundInfoMap.get(phone).getRecordMeasurePithInfo().add(new MusicPitchDetailDto(timeStamp, pitch, silenceDetecor.currentSPL()));
|
|
|
+// LOGGER.info("时间:{}, 频率:{}, 分贝:{}", timeStamp, pitch, silenceDetecor.currentSPL());
|
|
|
+ userSoundInfoMap.get(phone).getRecordMeasurePithInfo().add(new MusicPitchDetailDto(timeStamp, pitch, soundCompareConfig.silenceDetector.currentSPL()));
|
|
|
}));
|
|
|
dispatcher.run();
|
|
|
if(Objects.isNull(userSoundInfoMap.get(phone).getAccessFile())){
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
- double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(audioFormat.getFrameSize()*audioFormat.getFrameRate())*1000;
|
|
|
+ double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(soundCompareConfig.audioFormat.getFrameSize()*soundCompareConfig.audioFormat.getFrameRate())*1000;
|
|
|
userSoundInfoMap.get(phone).setMeasureStartTime(recordTime);
|
|
|
for (Map.Entry<Integer, MusicPitchDetailDto> userMeasureEndTimeMapEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) {
|
|
|
- int ot = (int) (userMeasureEndTimeMapEntry.getValue().getDuration()*0.1);
|
|
|
- if(recordTime>(userMeasureEndTimeMapEntry.getValue().getEndTimeStamp()+ot)){
|
|
|
+ if(recordTime>(userMeasureEndTimeMapEntry.getValue().getEndTimeStamp())){
|
|
|
if(userMeasureEndTimeMapEntry.getValue().getDontEvaluating()){
|
|
|
continue;
|
|
|
}else{
|
|
@@ -262,9 +258,9 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
}
|
|
|
if(Objects.nonNull(userSoundInfoMap.get(phone).getAccessFile())){
|
|
|
RandomAccessFile randomAccessFile = userSoundInfoMap.get(phone).getAccessFile();
|
|
|
- LOGGER.info("音频时长:{}", randomAccessFile.length()/(audioFormat.getFrameSize()*audioFormat.getFrameRate())*1000);
|
|
|
+ LOGGER.info("音频时长:{}", randomAccessFile.length()/(soundCompareConfig.audioFormat.getFrameSize()*soundCompareConfig.audioFormat.getFrameRate())*1000);
|
|
|
randomAccessFile.seek(0);
|
|
|
- randomAccessFile.write(WavHeader.getWaveHeader(randomAccessFile.length(), (long) audioFormat.getFrameRate(), audioFormat.getSampleSizeInBits()));
|
|
|
+ randomAccessFile.write(WavHeader.getWaveHeader(randomAccessFile.length(), (long) soundCompareConfig.audioFormat.getFrameRate(), soundCompareConfig.audioFormat.getSampleSizeInBits()));
|
|
|
randomAccessFile.close();
|
|
|
userSoundInfoMap.get(phone).setAccessFile(null);
|
|
|
}
|
|
@@ -293,23 +289,15 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
BigDecimal integrity = BigDecimal.ZERO;
|
|
|
|
|
|
try {
|
|
|
- //最低有效频率
|
|
|
- float minValidFrequency = 20;
|
|
|
-
|
|
|
//音准分数
|
|
|
float intonationScore = 0;
|
|
|
|
|
|
//节奏匹配数量
|
|
|
float cadenceNum = 0;
|
|
|
- //节奏有效阈值
|
|
|
- float cadenceValidDuty = 0.09f;
|
|
|
|
|
|
- //完整性误差范围
|
|
|
- float integrityRange = 30;
|
|
|
//完整性分数
|
|
|
float integrityScore = 0;
|
|
|
|
|
|
-
|
|
|
int totalCompareNum = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size();
|
|
|
|
|
|
for (int i = 0; i < userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size(); i++) {
|
|
@@ -319,12 +307,6 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
int startTimeStamp = musicXmlInfo.getTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime() + ot5;
|
|
|
int endTimeStamp = musicXmlInfo.getTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime() + musicXmlInfo.getDuration() - ot5;
|
|
|
|
|
|
- int preMeasureEndTimeStamp = startTimeStamp;
|
|
|
- List<MusicPitchDetailDto> ms = userSoundInfoMap.get(phone).getMusicXmlInfos().stream().filter(m -> m.getMusicalNotesIndex() == musicXmlInfo.getMusicalNotesIndex() - 1).collect(Collectors.toList());
|
|
|
- if(!CollectionUtils.isEmpty(ms)){
|
|
|
- preMeasureEndTimeStamp = ms.get(0).getEndTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime();
|
|
|
- }
|
|
|
-
|
|
|
//时间范围内有效节奏数量
|
|
|
float cadenceValidNum = 0;
|
|
|
//时间范围内有效音频数量
|
|
@@ -332,38 +314,6 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
//时间范围内匹配次数
|
|
|
float compareNum = 0;
|
|
|
|
|
|
-// boolean newMeasure = false;
|
|
|
-// float preMusicalNotesPitch = 0;
|
|
|
-// if(userSoundInfoMap.get(phone).getMusicalNotePitchMap().containsKey(musicXmlInfo.getMusicalNotesIndex()-1)){
|
|
|
-// preMusicalNotesPitch = userSoundInfoMap.get(phone).getMusicalNotePitchMap().get(musicXmlInfo.getMusicalNotesIndex()-1);
|
|
|
-// }
|
|
|
-// if(userSoundInfoMap.get(phone).getMusicalNotePitchMap().get(musicXmlInfo.getMusicalNotesIndex())==-1){
|
|
|
-// newMeasure = true;
|
|
|
-// }
|
|
|
-// int newNum = 0;
|
|
|
-//
|
|
|
-// for (MusicPitchDetailDto recordInfo : userSoundInfoMap.get(phone).getRecordMeasurePithInfo()) {
|
|
|
-// if(musicXmlInfo.getMusicalNotesIndex()==0){
|
|
|
-// newMeasure = true;
|
|
|
-// }
|
|
|
-// if(newMeasure){
|
|
|
-// break;
|
|
|
-// }
|
|
|
-// if(recordInfo.getTimeStamp()<preMeasureEndTimeStamp||recordInfo.getTimeStamp()>startTimeStamp){
|
|
|
-// continue;
|
|
|
-// }
|
|
|
-// if(Math.abs(recordInfo.getFrequency()-preMusicalNotesPitch)>10){
|
|
|
-// newNum++;
|
|
|
-// }else{
|
|
|
-// newNum = 0;
|
|
|
-// }
|
|
|
-// if(newNum>=2){
|
|
|
-// newMeasure = true;
|
|
|
-// }
|
|
|
-// }
|
|
|
-
|
|
|
-// List<Float> musicalNotesPitchs = new ArrayList<>();
|
|
|
-// List<Float> decibels = new ArrayList<>();
|
|
|
List<MusicPitchDetailDto> measureSoundPitchInfos = new ArrayList<>();
|
|
|
|
|
|
for (int j = 0; j < userSoundInfoMap.get(phone).getRecordMeasurePithInfo().size(); j++) {
|
|
@@ -372,21 +322,15 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
|
|
|
continue;
|
|
|
}
|
|
|
-// musicalNotesPitchs.add(recordInfo.getFrequency());
|
|
|
-// decibels.add(recordInfo.getDecibel());
|
|
|
measureSoundPitchInfos.add(recordInfo);
|
|
|
compareNum++;
|
|
|
-// LOGGER.info("{}频率({}-{}):{}, {}", recordInfo.getTimeStamp(), startTimeStamp, endTimeStamp, musicXmlInfo.getFrequency(), recordInfo.getFrequency());
|
|
|
//如果在最低有效频率以下则跳过
|
|
|
- if(recordInfo.getFrequency()<minValidFrequency&&musicXmlInfo.getFrequency()!=-1){
|
|
|
+ if(recordInfo.getFrequency()<soundCompareConfig.validFrequency&&musicXmlInfo.getFrequency()!=-1){
|
|
|
continue;
|
|
|
}
|
|
|
cadenceValidNum++;
|
|
|
- if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
|
|
|
- continue;
|
|
|
- }
|
|
|
//如果频率差值在节奏误差范围内
|
|
|
- if(Math.abs(recordInfo.getFrequency()-musicXmlInfo.getFrequency())<=integrityRange){
|
|
|
+ if(Math.abs(recordInfo.getFrequency()-musicXmlInfo.getFrequency())<=soundCompareConfig.integrityFrequencyRange){
|
|
|
integrityValidNum++;
|
|
|
}
|
|
|
}
|
|
@@ -452,7 +396,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
cadenceDuty = 0;
|
|
|
}
|
|
|
//节奏
|
|
|
- if(cadenceDuty>=cadenceValidDuty){
|
|
|
+ if(cadenceDuty>=soundCompareConfig.cadenceValidDuty){
|
|
|
cadenceNum++;
|
|
|
}
|
|
|
//音准
|
|
@@ -464,7 +408,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
recordCents = PitchConverter.hertzToAbsoluteCent(avgPitch);
|
|
|
}
|
|
|
double cents = PitchConverter.hertzToAbsoluteCent(musicXmlInfo.getFrequency());
|
|
|
- double score = 100 - Math.round(Math.abs(cents - recordCents)) + 3;
|
|
|
+ double score = 100 - Math.round(Math.abs(cents - recordCents)) + soundCompareConfig.intonationCentsRange;
|
|
|
if (score < 0){
|
|
|
score = 0;
|
|
|
}else if(score > 100){
|
|
@@ -475,7 +419,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
}
|
|
|
//完成度
|
|
|
if(integrityValidNum>0){
|
|
|
- integrityValidNum = integrityValidNum + (float) (compareNum * 0.05);
|
|
|
+ integrityValidNum = integrityValidNum;
|
|
|
}
|
|
|
if(integrityValidNum > compareNum){
|
|
|
integrityValidNum = compareNum;
|
|
@@ -540,7 +484,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
createPushInfo(phone, "overall", -1, intonation, cadence, integrity);
|
|
|
|
|
|
//存储评分数据
|
|
|
- sysMusicCompareRecordService.saveMusicCompareData(phone, userSoundInfoMap.get(phone).getMusicScoreId(), userSoundInfoMap.get(phone).getUserMeasureScoreMap());
|
|
|
+ sysMusicCompareRecordService.saveMusicCompareData(phone, userSoundInfoMap.get(phone));
|
|
|
|
|
|
LOGGER.info("评分数据:{}", JSON.toJSONString(userSoundInfoMap.get(phone)));
|
|
|
}
|
|
@@ -578,7 +522,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
|
|
|
LOGGER.info("小节频分:{}", JSON.toJSONString(webSocketInfo));
|
|
|
|
|
|
//推送结果
|
|
|
- WS_CLIENTS.get(phone).sendMessage(new TextMessage(JSON.toJSONString(webSocketInfo)));
|
|
|
+ WS_CLIENTS.get(phone).getSession().sendMessage(new TextMessage(JSON.toJSONString(webSocketInfo)));
|
|
|
return webSocketInfo;
|
|
|
}
|
|
|
}
|