|
|
@@ -0,0 +1,314 @@
|
|
|
+package com.yonge.netty.server.service;
|
|
|
+
|
|
|
+import io.netty.channel.Channel;
|
|
|
+
|
|
|
+import java.io.File;
|
|
|
+import java.math.BigDecimal;
|
|
|
+import java.text.SimpleDateFormat;
|
|
|
+import java.util.Comparator;
|
|
|
+import java.util.Date;
|
|
|
+import java.util.HashMap;
|
|
|
+import java.util.List;
|
|
|
+import java.util.Map;
|
|
|
+import java.util.Map.Entry;
|
|
|
+import java.util.Objects;
|
|
|
+import java.util.stream.Collectors;
|
|
|
+
|
|
|
+import javax.sound.sampled.AudioFormat;
|
|
|
+
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
+import org.slf4j.Logger;
|
|
|
+import org.slf4j.LoggerFactory;
|
|
|
+import org.springframework.beans.factory.annotation.Autowired;
|
|
|
+import org.springframework.stereotype.Component;
|
|
|
+
|
|
|
+import com.alibaba.fastjson.JSON;
|
|
|
+import com.alibaba.fastjson.JSONObject;
|
|
|
+import com.alibaba.fastjson.JSONPath;
|
|
|
+import com.ym.mec.biz.dal.entity.SysMusicCompareRecord;
|
|
|
+import com.ym.mec.biz.dal.enums.DeviceTypeEnum;
|
|
|
+import com.ym.mec.biz.dal.enums.FeatureType;
|
|
|
+import com.ym.mec.biz.service.SysMusicCompareRecordService;
|
|
|
+import com.ym.mec.thirdparty.storage.StoragePluginContext;
|
|
|
+import com.ym.mec.thirdparty.storage.provider.KS3StoragePlugin;
|
|
|
+import com.ym.mec.util.upload.UploadUtil;
|
|
|
+import com.yonge.audio.analysis.AudioFloatConverter;
|
|
|
+import com.yonge.audio.utils.ArrayUtil;
|
|
|
+import com.yonge.nettty.dto.SectionAnalysis;
|
|
|
+import com.yonge.nettty.dto.UserChannelContext;
|
|
|
+import com.yonge.nettty.dto.WebSocketResponse;
|
|
|
+import com.yonge.nettty.entity.MusicXmlBasicInfo;
|
|
|
+import com.yonge.nettty.entity.MusicXmlNote;
|
|
|
+import com.yonge.netty.server.handler.NettyChannelManager;
|
|
|
+import com.yonge.netty.server.handler.message.MessageHandler;
|
|
|
+import com.yonge.netty.server.processor.WaveformWriter;
|
|
|
+
|
|
|
+@Component
|
|
|
+public class AudioCompareHandler implements MessageHandler {
|
|
|
+
|
|
|
+ private static final Logger LOGGER = LoggerFactory.getLogger(AudioCompareHandler.class);
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private UserChannelContextService userChannelContextService;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private NettyChannelManager nettyChannelManager;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private SysMusicCompareRecordService sysMusicCompareRecordService;
|
|
|
+
|
|
|
+ @Autowired
|
|
|
+ private StoragePluginContext storagePluginContext;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * @describe 采样率
|
|
|
+ */
|
|
|
+ private float sampleRate = 44100;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 每个采样大小（Bit）
|
|
|
+ */
|
|
|
+ private int bitsPerSample = 16;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * 通道数
|
|
|
+ */
|
|
|
+ private int channels = 1;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * @describe 采样大小
|
|
|
+ */
|
|
|
+ private int bufferSize = 1024 * 4;
|
|
|
+
|
|
|
+ private boolean signed = true;
|
|
|
+
|
|
|
+ private boolean bigEndian = false;
|
|
|
+
|
|
|
+ private AudioFormat audioFormat = new AudioFormat(sampleRate, bitsPerSample, channels, signed, bigEndian);
|
|
|
+
|
|
|
+ private AudioFloatConverter converter = AudioFloatConverter.getConverter(audioFormat);
|
|
|
+
|
|
|
+ private String tmpFileDir = "/mdata/soundCompare/";
|
|
|
+
|
|
|
+ private SimpleDateFormat sdf = new SimpleDateFormat("yyMMddHHmmSS");
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public String getAction() {
|
|
|
+ return "SOUND_COMPARE";
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public boolean handleTextMessage(String user, Channel channel, String jsonMsg) {
|
|
|
+
|
|
|
+ String command = (String) JSONPath.extract(jsonMsg, "$.header.commond");
|
|
|
+
|
|
|
+ JSONObject dataObj = (JSONObject) JSONPath.extract(jsonMsg, "$.body");
|
|
|
+
|
|
|
+ UserChannelContext channelContext = userChannelContextService.getChannelContext(channel);
|
|
|
+
|
|
|
+ MusicXmlBasicInfo musicXmlBasicInfo = null;
|
|
|
+
|
|
|
+ switch (command) {
|
|
|
+ case "musicXml": // 同步music xml信息
|
|
|
+
|
|
|
+ musicXmlBasicInfo = JSONObject.toJavaObject(dataObj, MusicXmlBasicInfo.class);
|
|
|
+
|
|
|
+ userChannelContextService.remove(channel);
|
|
|
+
|
|
|
+ if (channelContext == null) {
|
|
|
+ channelContext = new UserChannelContext();
|
|
|
+ }
|
|
|
+
|
|
|
+ channelContext.getSongMusicXmlMap().put(musicXmlBasicInfo.getExamSongId(), musicXmlBasicInfo);
|
|
|
+ channelContext.init(musicXmlBasicInfo.getHeardLevel(), musicXmlBasicInfo.getSubjectId(), musicXmlBasicInfo.getBeatLength());
|
|
|
+
|
|
|
+ userChannelContextService.register(channel, channelContext);
|
|
|
+
|
|
|
+ break;
|
|
|
+ case "recordStart": // 开始评测
|
|
|
+
|
|
|
+ // 清空缓存信息
|
|
|
+ channelContext.resetUserInfo();
|
|
|
+
|
|
|
+ musicXmlBasicInfo = channelContext.getMusicXmlBasicInfo(null);
|
|
|
+
|
|
|
+ if (musicXmlBasicInfo != null) {
|
|
|
+ Date date = new Date();
|
|
|
+ SysMusicCompareRecord sysMusicCompareRecord = new SysMusicCompareRecord(FeatureType.CLOUD_STUDY_EVALUATION);
|
|
|
+ sysMusicCompareRecord.setCreateTime(date);
|
|
|
+ sysMusicCompareRecord.setUserId(Integer.parseInt(nettyChannelManager.getUser(channel)));
|
|
|
+ sysMusicCompareRecord.setSysMusicScoreId(musicXmlBasicInfo.getExamSongId());
|
|
|
+ sysMusicCompareRecord.setBehaviorId(musicXmlBasicInfo.getBehaviorId());
|
|
|
+ //sysMusicCompareRecord.setClientId();
|
|
|
+ sysMusicCompareRecord.setDeviceType(DeviceTypeEnum.valueOf(musicXmlBasicInfo.getPlatform()));
|
|
|
+ sysMusicCompareRecord.setSpeed(musicXmlBasicInfo.getSpeed());
|
|
|
+
|
|
|
+ MusicXmlNote musicXmlNote = musicXmlBasicInfo.getMusicXmlInfos().stream().max(Comparator.comparing(MusicXmlNote::getTimeStamp)).get();
|
|
|
+ sysMusicCompareRecord.setSourceTime((float) ((musicXmlNote.getTimeStamp()+musicXmlNote.getDuration())/1000));
|
|
|
+ sysMusicCompareRecordService.insert(sysMusicCompareRecord);
|
|
|
+ channelContext.setRecordId(sysMusicCompareRecord.getId());
|
|
|
+ }
|
|
|
+ break;
|
|
|
+ case "recordEnd": // 结束评测
|
|
|
+ case "recordCancel": // 取消评测
|
|
|
+ if (channelContext == null) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ WaveformWriter waveFileProcessor = channelContext.getWaveFileProcessor();
|
|
|
+ if (waveFileProcessor != null) {
|
|
|
+ // 写文件头
|
|
|
+ waveFileProcessor.processingFinished();
|
|
|
+ }
|
|
|
+
|
|
|
+ if (StringUtils.equals(command, "recordEnd")) {
|
|
|
+ // 生成评测报告
|
|
|
+ Map<String, Object> params = new HashMap<String, Object>();
|
|
|
+
|
|
|
+ Map<String, Integer> scoreMap = channelContext.evaluateForMusic();
|
|
|
+ for (Entry<String, Integer> entry : scoreMap.entrySet()) {
|
|
|
+ params.put(entry.getKey(), entry.getValue());
|
|
|
+ }
|
|
|
+
|
|
|
+ //保存评测结果
|
|
|
+ Long recordId = channelContext.getRecordId();
|
|
|
+ SysMusicCompareRecord sysMusicCompareRecord = sysMusicCompareRecordService.get(recordId);
|
|
|
+ if(sysMusicCompareRecord != null){
|
|
|
+ musicXmlBasicInfo = channelContext.getMusicXmlBasicInfo(null);
|
|
|
+
|
|
|
+ if (scoreMap != null && scoreMap.size() > 1) {
|
|
|
+ sysMusicCompareRecord.setScore(new BigDecimal(scoreMap.get("score")));
|
|
|
+ sysMusicCompareRecord.setIntonation(new BigDecimal(scoreMap.get("intonation")));
|
|
|
+ sysMusicCompareRecord.setIntegrity(new BigDecimal(scoreMap.get("integrity")));
|
|
|
+ sysMusicCompareRecord.setCadence(new BigDecimal(scoreMap.get("cadence")));
|
|
|
+ sysMusicCompareRecord.setPlayTime(scoreMap.get("playTime") / 1000);
|
|
|
+ }
|
|
|
+ sysMusicCompareRecord.setFeature(FeatureType.CLOUD_STUDY_EVALUATION);
|
|
|
+
|
|
|
+ String url = null;
|
|
|
+ try {
|
|
|
+ String folder = UploadUtil.getFileFloder();
|
|
|
+ url = storagePluginContext.asyncUploadFile(KS3StoragePlugin.PLUGIN_NAME,"soundCompare/" + folder, waveFileProcessor.getFile(), true);
|
|
|
+ } catch (Exception e) {
|
|
|
+ LOGGER.error("录音文件上传失败：{}", e);
|
|
|
+ }
|
|
|
+ sysMusicCompareRecord.setRecordFilePath(url);
|
|
|
+ //sysMusicCompareRecord.setVideoFilePath(videoFilePath);
|
|
|
+
|
|
|
+ Map<String, Object> scoreData = new HashMap<>();
|
|
|
+ List<SectionAnalysis> sectionAnalysisList = channelContext.getDoneSectionAnalysisList();
|
|
|
+ sectionAnalysisList = sectionAnalysisList.stream().filter(t -> t.isIngore() == false).collect(Collectors.toList());
|
|
|
+ scoreData.put("userMeasureScore", sectionAnalysisList.stream().collect(Collectors.toMap(SectionAnalysis :: getIndex, t -> t)));
|
|
|
+
|
|
|
+ Map<String, Object> musicalNotesPlayStats = new HashMap<>();
|
|
|
+ musicalNotesPlayStats.put("detailId", musicXmlBasicInfo.getDetailId());
|
|
|
+ musicalNotesPlayStats.put("examSongId", musicXmlBasicInfo.getExamSongId());
|
|
|
+ musicalNotesPlayStats.put("xmlUrl", musicXmlBasicInfo.getXmlUrl());
|
|
|
+
|
|
|
+ musicalNotesPlayStats.put("notesData", channelContext.getDoneNoteAnalysisList().stream().filter(t -> t.isIgnore() == false).collect(Collectors.toList()));
|
|
|
+ scoreData.put("musicalNotesPlayStats", musicalNotesPlayStats);
|
|
|
+ sysMusicCompareRecord.setScoreData(JSON.toJSONString(scoreData));
|
|
|
+
|
|
|
+ sysMusicCompareRecordService.saveMusicCompareData(sysMusicCompareRecord);
|
|
|
+ }
|
|
|
+
|
|
|
+ WebSocketResponse<Map<String, Object>> resp = new WebSocketResponse<Map<String, Object>>("overall", params);
|
|
|
+
|
|
|
+ nettyChannelManager.sendTextMessage(nettyChannelManager.getUser(channel), resp);
|
|
|
+ }
|
|
|
+
|
|
|
+ // 清空缓存信息
|
|
|
+ channelContext.resetUserInfo();
|
|
|
+
|
|
|
+ break;
|
|
|
+ case "proxyMessage": // ???
|
|
|
+
|
|
|
+ break;
|
|
|
+ case "videoUpload": // 上传音频
|
|
|
+ SysMusicCompareRecord musicCompareRecord = null;
|
|
|
+ if (dataObj.containsKey("recordId")) {
|
|
|
+ musicCompareRecord = sysMusicCompareRecordService.get(dataObj.getLong("recordId"));
|
|
|
+ }
|
|
|
+ if (Objects.nonNull(musicCompareRecord) && dataObj.containsKey("filePath")) {
|
|
|
+ musicCompareRecord.setVideoFilePath(dataObj.getString("filePath"));
|
|
|
+ sysMusicCompareRecordService.update(musicCompareRecord);
|
|
|
+ } else {
|
|
|
+ musicCompareRecord.setVideoFilePath(musicCompareRecord.getRecordFilePath());
|
|
|
+ sysMusicCompareRecordService.update(musicCompareRecord);
|
|
|
+ }
|
|
|
+
|
|
|
+ break;
|
|
|
+
|
|
|
+ default:
|
|
|
+ // 非法请求
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public boolean handleBinaryMessage(String user, Channel channel, byte[] datas) {
|
|
|
+
|
|
|
+ UserChannelContext channelContext = userChannelContextService.getChannelContext(channel);
|
|
|
+
|
|
|
+ if (channelContext == null) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 写录音文件
|
|
|
+ WaveformWriter waveFileProcessor = channelContext.getWaveFileProcessor();
|
|
|
+ if (waveFileProcessor == null) {
|
|
|
+ File file = new File(tmpFileDir + user + "_" + sdf.format(new Date()) + ".wav");
|
|
|
+ waveFileProcessor = new WaveformWriter(file.getAbsolutePath());
|
|
|
+ channelContext.setWaveFileProcessor(waveFileProcessor);
|
|
|
+ }
|
|
|
+ waveFileProcessor.process(datas);
|
|
|
+
|
|
|
+ datas = channelContext.skipHeader(datas);
|
|
|
+
|
|
|
+ if (datas.length == 0) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ channelContext.setChannelBufferBytes(ArrayUtil.mergeByte(channelContext.getChannelBufferBytes(), datas));
|
|
|
+
|
|
|
+ int totalLength = channelContext.getChannelBufferBytes().length;
|
|
|
+
|
|
|
+ while (totalLength >= bufferSize) {
|
|
|
+ byte[] bufferData = ArrayUtil.extractByte(channelContext.getChannelBufferBytes(), 0, bufferSize - 1);
|
|
|
+
|
|
|
+ if (bufferSize != totalLength) {
|
|
|
+ channelContext.setChannelBufferBytes(ArrayUtil.extractByte(channelContext.getChannelBufferBytes(), bufferSize, totalLength - 1));
|
|
|
+ } else {
|
|
|
+ channelContext.setChannelBufferBytes(new byte[0]);
|
|
|
+ }
|
|
|
+
|
|
|
+ float[] sampleFloats = new float[bufferSize / 2];
|
|
|
+
|
|
|
+ converter.toFloatArray(bufferData, sampleFloats);
|
|
|
+
|
|
|
+ channelContext.handle(sampleFloats, audioFormat);
|
|
|
+
|
|
|
+ MusicXmlBasicInfo musicXmlBasicInfo = channelContext.getMusicXmlBasicInfo(null);
|
|
|
+ int sectionIndex = channelContext.getEvaluatingSectionIndex().get();
|
|
|
+
|
|
|
+ // 评分
|
|
|
+ int score = channelContext.evaluateForSection(sectionIndex, musicXmlBasicInfo.getSubjectId());
|
|
|
+ if (score >= 0) {
|
|
|
+
|
|
|
+ Map<String, Object> params = new HashMap<String, Object>();
|
|
|
+ params.put("score", score);
|
|
|
+ params.put("measureIndex", sectionIndex);
|
|
|
+
|
|
|
+ WebSocketResponse<Map<String, Object>> resp = new WebSocketResponse<Map<String, Object>>("measureScore", params);
|
|
|
+
|
|
|
+ nettyChannelManager.sendTextMessage(user, resp);
|
|
|
+ }
|
|
|
+
|
|
|
+ totalLength = channelContext.getChannelBufferBytes().length;
|
|
|
+ }
|
|
|
+
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+
|
|
|
+}
|
yonge