|
@@ -6,6 +6,7 @@ import be.tarsos.dsp.pitch.PitchProcessor;
|
|
|
import be.tarsos.dsp.util.PitchConverter;
|
|
|
import com.alibaba.fastjson.JSON;
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.alibaba.fastjson.serializer.SerializerFeature;
|
|
|
import com.ym.mec.biz.dal.config.SoundCompareConfig;
|
|
|
import com.ym.mec.biz.dal.dao.SysMusicScoreAccompanimentDao;
|
|
|
import com.ym.mec.biz.dal.dto.*;
|
|
@@ -31,7 +32,6 @@ import org.springframework.web.socket.WebSocketSession;
|
|
|
|
|
|
import javax.sound.sampled.UnsupportedAudioFileException;
|
|
|
import java.io.File;
|
|
|
-import java.io.FileNotFoundException;
|
|
|
import java.io.IOException;
|
|
|
import java.io.RandomAccessFile;
|
|
|
import java.math.BigDecimal;
|
|
@@ -50,7 +50,7 @@ import static com.ym.mec.biz.service.SoundSocketService.VIDEO_UPDATE;
|
|
|
@Service
|
|
|
public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
|
|
|
- private final Logger LOGGER = LoggerFactory.getLogger(SoundCompareHandler.class);
|
|
|
+ public static final Logger LOGGER = LoggerFactory.getLogger(SoundCompareHandler.class);
|
|
|
|
|
|
private BigDecimal oneHundred = new BigDecimal(100);
|
|
|
|
|
@@ -99,6 +99,9 @@ public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
userSoundInfoMap.get(phone).setMusicXmlInfos(musicXmlInfos);
|
|
|
musicXmlInfos = musicXmlInfos.stream().filter(m->!m.getDontEvaluating()).collect(Collectors.toList());
|
|
|
userSoundInfoMap.get(phone).setMusicScoreId(bodyObject.getInteger("id"));
|
|
|
+ if(bodyObject.containsKey("beatLength")){
|
|
|
+ userSoundInfoMap.get(phone).setFirstMeasureStartBytes((long) (bodyObject.getLong("beatLength")/1000f*(soundCompareConfig.audioFormat.getFrameSize()*soundCompareConfig.audioFormat.getFrameRate())));
|
|
|
+ }
|
|
|
if(bodyObject.containsKey("platform")){
|
|
|
userSoundInfoMap.get(phone).setDeviceType(DeviceTypeEnum.valueOf(bodyObject.getString("platform")));
|
|
|
}
|
|
@@ -148,8 +151,9 @@ public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
File file = new File(tmpDir+phone + "_"+ userSoundInfoMap.get(phone).getMusicScoreId() +"_"+ LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")) +".wav");
|
|
|
userSoundInfoMap.get(phone).setFile(file);
|
|
|
userSoundInfoMap.get(phone).setAccessFile(new RandomAccessFile(file, "rw"));
|
|
|
+ userSoundInfoMap.get(phone).getAccessFile().seek(44);
|
|
|
userSoundInfoMap.get(phone).setRecordFilePath(file.getAbsolutePath());
|
|
|
- } catch (FileNotFoundException e) {
|
|
|
+ } catch (IOException e) {
|
|
|
throw new BizException("文件创建失败:", e);
|
|
|
}
|
|
|
break;
|
|
@@ -178,10 +182,10 @@ public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
createHeader(phone, false);
|
|
|
break;
|
|
|
case SoundSocketService.PROXY_MESSAGE:
|
|
|
- if(DeviceTypeEnum.IOS.equals(userSoundInfoMap.get(phone).getDeviceType())&&bodyObject.containsKey(SoundSocketService.OFFSET_TIME)){
|
|
|
- int offsetTime = bodyObject.getIntValue(SoundSocketService.OFFSET_TIME);
|
|
|
- calOffsetTime(phone, offsetTime);
|
|
|
- }
|
|
|
+// if(DeviceTypeEnum.IOS.equals(userSoundInfoMap.get(phone).getDeviceType())&&bodyObject.containsKey(SoundSocketService.OFFSET_TIME)){
|
|
|
+// int offsetTime = bodyObject.getIntValue(SoundSocketService.OFFSET_TIME);
|
|
|
+// calOffsetTime(phone, offsetTime);
|
|
|
+// }
|
|
|
break;
|
|
|
case VIDEO_UPDATE:
|
|
|
SysMusicCompareRecord update = null;
|
|
@@ -207,24 +211,27 @@ public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
if(!userSoundInfoMap.containsKey(phone)){
|
|
|
return;
|
|
|
}
|
|
|
+ userSoundInfoMap.get(phone).setRecordBytes(userSoundInfoMap.get(phone).getRecordBytes()+message.getPayloadLength());
|
|
|
+ if(userSoundInfoMap.get(phone).getRecordBytes()<userSoundInfoMap.get(phone).getFirstMeasureStartBytes()){
|
|
|
+ return;
|
|
|
+ }
|
|
|
try {
|
|
|
if(Objects.nonNull(userSoundInfoMap.get(phone).getAccessFile())){
|
|
|
userSoundInfoMap.get(phone).getAccessFile().write(message.getPayload().array());
|
|
|
+ }else{
|
|
|
+ return;
|
|
|
}
|
|
|
|
|
|
AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), soundCompareConfig.audioFormat, soundCompareConfig.simpleSize, soundCompareConfig.overlap);
|
|
|
-
|
|
|
dispatcher.addAudioProcessor(userSoundInfoMap.get(phone).silenceDetector);
|
|
|
dispatcher.addAudioProcessor(new PitchProcessor(soundCompareConfig.algo, soundCompareConfig.simpleRate, soundCompareConfig.simpleSize, userSoundInfoMap.get(phone)));
|
|
|
dispatcher.run();
|
|
|
- if(Objects.isNull(userSoundInfoMap.get(phone).getAccessFile())){
|
|
|
- return;
|
|
|
- }
|
|
|
|
|
|
double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(soundCompareConfig.audioFormat.getFrameSize()*soundCompareConfig.audioFormat.getFrameRate())*1000;
|
|
|
userSoundInfoMap.get(phone).setMeasureStartTime(recordTime);
|
|
|
+
|
|
|
for (Map.Entry<Integer, MusicPitchDetailDto> userMeasureEndTimeMapEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) {
|
|
|
- if(recordTime>(userMeasureEndTimeMapEntry.getValue().getEndTimeStamp())){
|
|
|
+ if(recordTime>(userMeasureEndTimeMapEntry.getValue().getEndTimeStamp()+100)){
|
|
|
if(userMeasureEndTimeMapEntry.getValue().getDontEvaluating()){
|
|
|
continue;
|
|
|
}else{
|
|
@@ -261,6 +268,7 @@ public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
musicPitchDetailDtoEntry.getValue().setTimeStamp(musicPitchDetailDtoEntry.getValue().getTimeStamp() + offsetTime);
|
|
|
musicPitchDetailDtoEntry.getValue().setEndTimeStamp(musicPitchDetailDtoEntry.getValue().getEndTimeStamp() + offsetTime);
|
|
|
}
|
|
|
+ userSoundInfoMap.get(phone).getMusicXmlInfos().forEach(e->e.setTimeStamp(e.getTimeStamp()+offsetTime));
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -347,111 +355,32 @@ public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
for (int i = 0; i < userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size(); i++) {
|
|
|
MusicPitchDetailDto musicXmlInfo = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).get(i);
|
|
|
|
|
|
- int ot5 = (int) (musicXmlInfo.getDuration()*0.1);
|
|
|
- int startTimeStamp = musicXmlInfo.getTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime() + ot5;
|
|
|
- int endTimeStamp = musicXmlInfo.getTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime() + musicXmlInfo.getDuration() - ot5;
|
|
|
-
|
|
|
- //时间范围内有效节奏数量
|
|
|
- float cadenceValidNum = 0;
|
|
|
- //时间范围内有效音频数量
|
|
|
- float integrityValidNum = 0;
|
|
|
- //时间范围内匹配次数
|
|
|
- float compareNum = 0;
|
|
|
+ int startTimeStamp = musicXmlInfo.getTimeStamp();
|
|
|
+ int endTimeStamp = musicXmlInfo.getTimeStamp() + musicXmlInfo.getDuration();
|
|
|
|
|
|
- List<MusicPitchDetailDto> measureSoundPitchInfos = new ArrayList<>();
|
|
|
+ int ot5 = (int) (musicXmlInfo.getDuration()*0.22<70?70:musicXmlInfo.getDuration()*0.22);
|
|
|
+ int rightTimeRange = ot5>200?200:ot5;
|
|
|
|
|
|
- for (int j = 0; j < userSoundInfoMap.get(phone).getRecordMeasurePithInfo().size(); j++) {
|
|
|
- MusicPitchDetailDto recordInfo = userSoundInfoMap.get(phone).getRecordMeasurePithInfo().get(j);
|
|
|
- //如果在时间范围之外直接跳过
|
|
|
- if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
|
|
|
- continue;
|
|
|
- }
|
|
|
- measureSoundPitchInfos.add(recordInfo);
|
|
|
- compareNum++;
|
|
|
- //如果在最低有效频率以下则跳过
|
|
|
- if(recordInfo.getFrequency()<soundCompareConfig.validFrequency&&musicXmlInfo.getFrequency()!=-1){
|
|
|
- continue;
|
|
|
- }
|
|
|
- cadenceValidNum++;
|
|
|
- //如果频率差值在节奏误差范围内
|
|
|
-// if(Math.abs(recordInfo.getFrequency()-musicXmlInfo.getFrequency())<=soundCompareConfig.integrityFrequencyRange){
|
|
|
-// integrityValidNum++;
|
|
|
-// }
|
|
|
- }
|
|
|
-
|
|
|
- //非正常频率次数
|
|
|
- int errPitchNum = 0;
|
|
|
- //分贝变化次数
|
|
|
- int decibelChangeNum = 0;
|
|
|
-
|
|
|
- if(CollectionUtils.isEmpty(measureSoundPitchInfos)){
|
|
|
- userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(musicXmlInfo.getMusicalNotesIndex(), (float) 0);
|
|
|
- }else{
|
|
|
- Map<Integer, Long> collect = measureSoundPitchInfos.stream().map(pitch -> (int)pitch.getFrequency()).collect(Collectors.groupingBy(Integer::intValue, Collectors.counting()));
|
|
|
- //出现次数最多的频率
|
|
|
- Integer pitch = collect.entrySet().stream().max(Comparator.comparing(e -> e.getValue())).get().getKey();
|
|
|
- //当前频率
|
|
|
- double cf = -1;
|
|
|
- //频率持续数量
|
|
|
- int fnum = 0;
|
|
|
- //是否演奏中
|
|
|
- boolean ing = false;
|
|
|
- //当前分贝
|
|
|
- double cd = 0;
|
|
|
- //分贝变化方向,-1变小,1变大
|
|
|
- int dcd = -1;
|
|
|
- //分贝持续数量
|
|
|
- int dnum = 0;
|
|
|
- for (MusicPitchDetailDto musicalNotesPitch : measureSoundPitchInfos) {
|
|
|
- //计算频率断层次数
|
|
|
- if (Math.abs(musicalNotesPitch.getFrequency() - cf) > 20){
|
|
|
- fnum ++;
|
|
|
- }
|
|
|
- if (fnum>=5){
|
|
|
- cf = musicalNotesPitch.getFrequency();
|
|
|
- fnum = 0;
|
|
|
- if (cf != -1){
|
|
|
- errPitchNum ++;
|
|
|
- ing = true;
|
|
|
- cd = musicalNotesPitch.getDecibel();
|
|
|
- }
|
|
|
- }
|
|
|
- //计算声音大小断层册数
|
|
|
- if(ing && Math.abs(musicalNotesPitch.getDecibel() - cd) > 10){
|
|
|
- dnum ++;
|
|
|
- }
|
|
|
- if (dnum > 2){
|
|
|
- int tdcd = cd > musicalNotesPitch.getDecibel() ? -1 : 1;
|
|
|
- cd = musicalNotesPitch.getDecibel();
|
|
|
- dnum = 0;
|
|
|
- if (tdcd != dcd) {
|
|
|
- decibelChangeNum++;
|
|
|
- }
|
|
|
- dcd = tdcd;
|
|
|
- }
|
|
|
- }
|
|
|
- userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(musicXmlInfo.getMusicalNotesIndex(), (float) pitch);
|
|
|
- }
|
|
|
+ List<MusicPitchDetailDto> recordPitchs = userSoundInfoMap.get(phone).getRecordMeasurePitchInfos().stream().filter(m -> m.getTimeStamp()>=startTimeStamp-rightTimeRange && m.getTimeStamp() < endTimeStamp-rightTimeRange).collect(Collectors.toList());
|
|
|
|
|
|
boolean cadenceRight = false;
|
|
|
boolean intonationRight = false;
|
|
|
boolean integrityRight = false;
|
|
|
|
|
|
- //有效节奏占比
|
|
|
- float cadenceDuty = cadenceValidNum/compareNum;
|
|
|
- //如果频率出现断层或这个音量出现断层,则当前音符节奏无效
|
|
|
- if(errPitchNum>=2 || decibelChangeNum>1){
|
|
|
- cadenceDuty = 0;
|
|
|
+ float integrityDuty = 0;
|
|
|
+ if(recordPitchs.size()>0){
|
|
|
+ integrityDuty = recordPitchs.get(0).getDuration()/(float)musicXmlInfo.getDuration();
|
|
|
}
|
|
|
+ integrityDuty = scoreMapping(integrityDuty, userSoundInfoMap.get(phone).getHeardLevel().getIntegrityRange(), 1);
|
|
|
//节奏
|
|
|
- if(cadenceDuty>=userSoundInfoMap.get(phone).getHeardLevel().getCadenceRange()){
|
|
|
+ if(recordPitchs.size()==1){
|
|
|
cadenceNum++;
|
|
|
cadenceRight = true;
|
|
|
}
|
|
|
//音准、完成度
|
|
|
- if (cadenceRight && !CollectionUtils.isEmpty(measureSoundPitchInfos)){
|
|
|
+ if (cadenceRight){
|
|
|
//音准
|
|
|
- Double avgPitch = measureSoundPitchInfos.stream().filter(pitch -> Math.abs((pitch.getFrequency()-musicXmlInfo.getFrequency()))<15).collect(Collectors.averagingDouble(pitch -> pitch.getFrequency()));
|
|
|
+ float avgPitch = recordPitchs.get(0).getFrequency();
|
|
|
//音分
|
|
|
double recordCents = 0;
|
|
|
if (avgPitch > 0){
|
|
@@ -461,29 +390,40 @@ public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
if(musicXmlInfo.getFrequency()>0){
|
|
|
cents = PitchConverter.hertzToAbsoluteCent(musicXmlInfo.getFrequency());
|
|
|
}
|
|
|
- double score = 100 - Math.round(Math.abs(cents - recordCents)) + userSoundInfoMap.get(phone).getHeardLevel().getIntonationCentsRange();
|
|
|
+ double score = 100 - Math.round(Math.abs(cents - recordCents)) + 3;
|
|
|
if (score < 0){
|
|
|
score = 0;
|
|
|
}else if(score > 100){
|
|
|
score = 100;
|
|
|
}
|
|
|
+
|
|
|
+ score = Math.pow(score/100f, userSoundInfoMap.get(phone).getHeardLevel().getIntonationCentsRange())*100;
|
|
|
+
|
|
|
+ if(Objects.nonNull(userSoundInfoMap.get(phone).getSubjectId())&&userSoundInfoMap.get(phone).getSubjectId()==23){
|
|
|
+ score = 100;
|
|
|
+ integrityDuty = 1;
|
|
|
+ }
|
|
|
+
|
|
|
intonationScore += score;
|
|
|
- musicXmlInfo.setAvgFrequency(avgPitch.floatValue());
|
|
|
+ musicXmlInfo.setAvgFrequency(avgPitch);
|
|
|
intonationRight = score>70;
|
|
|
|
|
|
- if(score>0){
|
|
|
- integrityValidNum = measureSoundPitchInfos.stream().filter(pitch -> Math.abs((pitch.getFrequency()-musicXmlInfo.getFrequency()))<15).count();
|
|
|
- }else{
|
|
|
- integrityValidNum = 0;
|
|
|
- }
|
|
|
+ integrityScore += integrityDuty;
|
|
|
+ integrityRight = integrityDuty>0.7;
|
|
|
}
|
|
|
- //完成度
|
|
|
- if(integrityValidNum > compareNum){
|
|
|
- integrityValidNum = compareNum;
|
|
|
+
|
|
|
+ //如果当前音符不需要演奏
|
|
|
+ if (musicXmlInfo.getFrequency()<0&&recordPitchs.size()<=0){
|
|
|
+ cadenceNum++;
|
|
|
+ cadenceRight = true;
|
|
|
+
|
|
|
+ intonationScore += 100;
|
|
|
+ musicXmlInfo.setAvgFrequency(-1);
|
|
|
+ intonationRight = true;
|
|
|
+
|
|
|
+ integrityScore += 1;
|
|
|
+ integrityRight = true;
|
|
|
}
|
|
|
- float integrityDuty = integrityValidNum/compareNum;
|
|
|
- integrityScore += integrityDuty;
|
|
|
- integrityRight = integrityDuty>0.7;
|
|
|
|
|
|
if(!cadenceRight){
|
|
|
userSoundInfoMap.get(phone).getMusicalNotesPlayStats().add(new MusicalNotesPlayStatDto(musicXmlInfo.getMusicalNotesIndex(), MusicalErrorTypeEnum.CADENCE_WRONG));
|
|
@@ -528,6 +468,11 @@ public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
createPushInfo(phone, "measureScore", measureIndex, intonation, cadence, integrity);
|
|
|
}
|
|
|
|
|
|
+ private float scoreMapping(float score, float divisor, float maxValue){
|
|
|
+ score = score*divisor;
|
|
|
+ return score>maxValue?maxValue:score;
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* @describe 计算最终评分
|
|
|
* @author Joburgess
|
|
@@ -543,15 +488,15 @@ public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
BigDecimal integrity = BigDecimal.ZERO;
|
|
|
|
|
|
if(currentCompareNum>0){
|
|
|
- intonation = userSoundInfoMap.get(phone).getUserScoreMap().get("intonation").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_DOWN);
|
|
|
- cadence = userSoundInfoMap.get(phone).getUserScoreMap().get("cadence").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_DOWN);
|
|
|
- integrity = userSoundInfoMap.get(phone).getUserScoreMap().get("integrity").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_DOWN);
|
|
|
+ intonation = userSoundInfoMap.get(phone).getUserScoreMap().get("intonation").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_UP);
|
|
|
+ cadence = userSoundInfoMap.get(phone).getUserScoreMap().get("cadence").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_UP);
|
|
|
+ integrity = userSoundInfoMap.get(phone).getUserScoreMap().get("integrity").divide(new BigDecimal(currentCompareNum), 0, BigDecimal.ROUND_UP);
|
|
|
}
|
|
|
|
|
|
//计算分数并推送
|
|
|
createPushInfo(phone, "overall", -1, intonation, cadence, integrity);
|
|
|
|
|
|
- LOGGER.info("评分数据:{}", JSON.toJSONString(userSoundInfoMap.get(phone)));
|
|
|
+ LOGGER.info("评分数据:{}", JSON.toJSONString(userSoundInfoMap.get(phone), SerializerFeature.DisableCircularReferenceDetect));
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -570,12 +515,8 @@ public class SoundCompareHandler implements WebSocketEventHandler {
|
|
|
WebSocketInfo webSocketInfo = new WebSocketInfo();
|
|
|
webSocketInfo.setHeader(new WebSocketInfo.Head(command));
|
|
|
Map<String, Object> result = new HashMap<>(5);
|
|
|
- //打击乐只看节奏分
|
|
|
- BigDecimal score = cadence;
|
|
|
- //非打击乐总分为平均分
|
|
|
- if(Objects.isNull(userSoundInfoMap.get(phone).getSubjectId())||userSoundInfoMap.get(phone).getSubjectId()!=23){
|
|
|
- score = intonation.add(cadence).add(integrity).divide(new BigDecimal(3), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).setScale(0, BigDecimal.ROUND_UP);
|
|
|
- }
|
|
|
+ BigDecimal score = intonation.add(cadence).add(integrity).divide(new BigDecimal(3), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).setScale(0, BigDecimal.ROUND_UP);
|
|
|
+
|
|
|
result.put("score", score);
|
|
|
result.put("intonation", intonation);
|
|
|
result.put("cadence", cadence);
|