fix iOS playback by playing dummy sound on play button (#38, PR #101)

no unmute button needed for now

squashed commits:

* basic iOS playback fix with unmute button (always shown)

* show unmute button only on iOS (detect iOS)

* fix iOS playback without unmute button (!!), remove unnecessary code for unmute button and audio initialization

turns out all you need is to play a dummy sound on a play button click.

---------

Co-authored-by: sschmidTU <s.schmid@phonicscore.com>

demo/index.html

@@ -220,6 +220,10 @@
         <div class="mdc-fab__ripple"></div>
         <span class="mdc-fab__icon material-icons">settings</span>
     </button>
+    <!-- <button class="mdc-fab mdc-fab--mini playback-settings-button" id="unmute-button" aria-label="Unmute">
+        <div class="mdc-fab__ripple"></div>
+        <span class="mdc-fab__icon material-icons">volume_off</span>
+    </button> -->
 </div>
 </body>
 </html>

demo/index.js

@@ -11,6 +11,7 @@ import { TransposeCalculator } from '../src/Plugins/Transpose/TransposeCalculato
     var openSheetMusicDisplay;
     var sampleFolder = "",
         samples = {
+            "playerdemo.musicxml": "playerdemo.musicxml",
             "Beethoven, L.v. - An die ferne Geliebte": "Beethoven_AnDieFerneGeliebte.xml",
             "Clementi, M. - Sonatina Op.36 No.1 Pt.1": "MuzioClementi_SonatinaOpus36No1_Part1.xml",
             "Clementi, M. - Sonatina Op.36 No.1 Pt.2": "MuzioClementi_SonatinaOpus36No1_Part2.xml",
@@ -110,6 +111,7 @@ import { TransposeCalculator } from '../src/Plugins/Transpose/TransposeCalculato
         performanceModeBtn,
         playbackControlsButton,
         playbackControl;
+        //unmuteButton;
     
     // manage option setting and resetting for specific samples, e.g. in the autobeam sample autobeam is set to true, otherwise reset to previous state
     // TODO design a more elegant option state saving & restoring system, though that requires saving the options state in OSMD
@@ -136,7 +138,25 @@ import { TransposeCalculator } from '../src/Plugins/Transpose/TransposeCalculato
     var showHeader = true;
     var showDebugControls = false;
 
-    document.title = "OpenSheetMusicDisplay Demo";
+    document.title = "OSMD Audio Player Demo";
+
+    function iOSDetected() {
+        // according to https://stackoverflow.com/a/9039885/10295942
+        return [
+          'iPad Simulator',
+          'iPhone Simulator',
+          'iPod Simulator',
+          'iPad',
+          'iPhone',
+          'iPod'
+        ].includes(navigator.platform)
+        // note that .platform was apparently only accidentally marked deprecated,
+        //   see https://stackoverflow.com/a/47599911/10295942
+        // iPad on iOS 13 detection
+        || (navigator.userAgent.includes("Mac") && "ontouchend" in document)
+        // note that some people recommend just checking userAgent instead of platform, but
+        //   "userAgent gives false positives because some vendors fake it to mimic Apple devices for whatever reasons"
+      }
 
     // Initialization code
     function init() {
@@ -262,7 +282,8 @@ import { TransposeCalculator } from '../src/Plugins/Transpose/TransposeCalculato
         printPdfBtns.push(document.getElementById("print-pdf-btn-optional"));
         transpose = document.getElementById('transpose');
         transposeBtn = document.getElementById('transpose-btn');
-        playbackControlsButton = document.getElementById("playback-settings-button")
+        playbackControlsButton = document.getElementById("playback-settings-button");
+        // unmuteButton = document.getElementById("unmute-button");
 
         //var defaultDisplayVisibleValue = "block"; // TODO in some browsers flow could be the better/default value
         var defaultVisibilityValue = "visible";
@@ -801,7 +822,11 @@ import { TransposeCalculator } from '../src/Plugins/Transpose/TransposeCalculato
             selectSample.removeChild(custom);
         }
 
-        playbackControl.initialize();
+        if (openSheetMusicDisplay.Sheet) {
+            playbackControl.initialize();
+        } else {
+            console.warn("Sheet couldn't be loaded, so playback control can't be initialized");
+        }
 
         // Enable controls again
         enable();

src/Common/Interfaces/IAudioPlayer.ts

@@ -1,6 +1,8 @@
 import { MidiInstrument } from "../../MusicalScore/VoiceData/Instructions/ClefInstruction";
 
 export interface IAudioPlayer<TSoundFont> {
+  ac: AudioContext;
+
   open(uniqueInstruments: number[], numberOfinstruments: number): void;
 
   close(): void;

src/Playback/BasicAudioPlayer.ts

@@ -2,13 +2,13 @@
 
 import { MidiInstrument } from "../MusicalScore/VoiceData/Instructions/ClefInstruction";
 import { IAudioPlayer } from "../Common/Interfaces/IAudioPlayer";
-import { AudioContext as SAudioContext } from "standardized-audio-context";
+// import { AudioContext as SAudioContext } from "standardized-audio-context";
 import * as SoundfontPlayer from "soundfont-player";
 import midiNames from "./midiNames";
 
 export class BasicAudioPlayer implements IAudioPlayer<SoundfontPlayer.Player> {
 
-  private ac: SAudioContext = new SAudioContext();
+  public ac: AudioContext = new AudioContext();
   // private mainTuningRatio: number = 1.0;
   private channelVolumes: number[] = [];
   // private activeSamples: Map<number, any> = new Map();

src/Playback/PlaybackManager.ts

@@ -102,7 +102,8 @@ export class PlaybackManager implements IPlaybackParametersListener {
     private isPlaying: boolean = false;
     private metronome: MetronomeInstrument;
     private metronomeSoundPlayed: boolean = false;
-
+    /** Whether a dummy sound was played to initialize the audio context / enable sound (on iOS). */
+    public DummySoundPlayed: boolean = false;
 
     private tempoImpactFactor: number = 1.0;
     private sheetStartBPM: number;
@@ -187,6 +188,23 @@ export class PlaybackManager implements IPlaybackParametersListener {
     // Only used for debug and scheduling precision measurements
     //private wantedNextIteratorTimestampMs: number = 0;
 
+    /** Play dummy sound to initialize audio context (e.g. on user click for iOS) */
+    public playDummySound(): void {
+        const context: AudioContext = this.audioPlayer.ac;
+        // create empty buffer and play it (to initialize context on user click)
+        const buffer: AudioBuffer = context.createBuffer(1, 1, 22050);
+        const source: AudioBufferSourceNode = context.createBufferSource();
+        source.buffer = buffer;
+        source.connect(context.destination);
+
+        // play the buffer. noteOn is the older version of start()
+        if (source.start) {
+            source.start(0);
+        } else {
+            (source as any).noteOn(0); // this was the old way to start a sound
+        }
+    }
+
     public playVoiceEntry(voiceEntry: VoiceEntry): void {
         const ve: VoiceEntry = voiceEntry;
         if (ve !== undefined) {

src/Playback/TimingSources/LinearTimingSource.ts

@@ -1,5 +1,5 @@
 import { AbstractTimingSource } from "./AbstractTimingSource";
-import { AudioContext } from "standardized-audio-context";
+// import { AudioContext } from "standardized-audio-context";
 import { Fraction } from "../../Common/DataObjects";
 
 export class LinearTimingSource extends AbstractTimingSource {

src/Playback/UIComponents/ControlPanel/ControlPanel.ts

@@ -12,6 +12,7 @@ import { Dictionary } from "typescript-collections";
 import { CursorPosChangedData } from "../../../Common/DataObjects/CursorPosChangedData";
 import { Fraction } from "../../../Common/DataObjects/Fraction";
 import { IPlaybackListener } from "../../../Common/Interfaces/IPlaybackListener";
+import { PlaybackManager } from "../../PlaybackManager";
 
 export class ControlPanel extends AUIController<IPlaybackParametersListener>
 implements IPlaybackParametersListener, IPlaybackListener {
@@ -110,6 +111,11 @@ implements IPlaybackParametersListener, IPlaybackListener {
         this.playPauseButton.listen((state) => {
             if (state === "playing") {
                 for (const listener of self.eventListeners) {
+                    if (listener instanceof PlaybackManager) {
+                        if (!listener.DummySoundPlayed) {
+                            listener.playDummySound();
+                        }
+                    }
                     listener.play();
                 }
             } else {

test/data/playerdemo.musicxml

@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE score-partwise PUBLIC "-//Recordare//DTD MusicXML 3.1 Partwise//EN" "http://www.musicxml.org/dtds/partwise.dtd">
+<score-partwise version="3.1">
+  <work>
+    <work-title>OSMD Audio Player Demo</work-title>
+    </work>
+  <identification>
+    <encoding>
+      <software>MuseScore 3.6.2</software>
+      <encoding-date>2023-04-29</encoding-date>
+      <supports element="accidental" type="yes"/>
+      <supports element="beam" type="yes"/>
+      <supports element="print" attribute="new-page" type="yes" value="yes"/>
+      <supports element="print" attribute="new-system" type="yes" value="yes"/>
+      <supports element="stem" type="yes"/>
+      </encoding>
+    </identification>
+  <defaults>
+    <scaling>
+      <millimeters>6.99911</millimeters>
+      <tenths>40</tenths>
+      </scaling>
+    <page-layout>
+      <page-height>1596.77</page-height>
+      <page-width>1233.87</page-width>
+      <page-margins type="even">
+        <left-margin>85.7252</left-margin>
+        <right-margin>85.7252</right-margin>
+        <top-margin>85.7252</top-margin>
+        <bottom-margin>85.7252</bottom-margin>
+        </page-margins>
+      <page-margins type="odd">
+        <left-margin>85.7252</left-margin>
+        <right-margin>85.7252</right-margin>
+        <top-margin>85.7252</top-margin>
+        <bottom-margin>85.7252</bottom-margin>
+        </page-margins>
+      </page-layout>
+    <word-font font-family="Edwin" font-size="10"/>
+    <lyric-font font-family="Edwin" font-size="10"/>
+    </defaults>
+  <part-list>
+    <score-part id="P1">
+      <part-name>Piano</part-name>
+      <part-abbreviation>Pno.</part-abbreviation>
+      <score-instrument id="P1-I1">
+        <instrument-name>Piano</instrument-name>
+        </score-instrument>
+      <midi-device id="P1-I1" port="1"></midi-device>
+      <midi-instrument id="P1-I1">
+        <midi-channel>1</midi-channel>
+        <midi-program>1</midi-program>
+        <volume>78.7402</volume>
+        <pan>0</pan>
+        </midi-instrument>
+      </score-part>
+    </part-list>
+  <part id="P1">
+    <measure number="1" width="407.91">
+      <print>
+        <system-layout>
+          <system-margins>
+            <left-margin>50.00</left-margin>
+            <right-margin>570.79</right-margin>
+            </system-margins>
+          <top-system-distance>170.00</top-system-distance>
+          </system-layout>
+        </print>
+      <attributes>
+        <divisions>1</divisions>
+        <key>
+          <fifths>0</fifths>
+          </key>
+        <time>
+          <beats>4</beats>
+          <beat-type>4</beat-type>
+          </time>
+        <clef>
+          <sign>G</sign>
+          <line>2</line>
+          </clef>
+        </attributes>
+      <note default-x="80.72" default-y="-30.00">
+        <pitch>
+          <step>G</step>
+          <octave>4</octave>
+          </pitch>
+        <duration>1</duration>
+        <voice>1</voice>
+        <type>quarter</type>
+        <stem>up</stem>
+        </note>
+      <note default-x="159.86" default-y="-25.00">
+        <pitch>
+          <step>A</step>
+          <octave>4</octave>
+          </pitch>
+        <duration>1</duration>
+        <voice>1</voice>
+        <type>quarter</type>
+        <stem>up</stem>
+        </note>
+      <note default-x="238.99" default-y="-20.00">
+        <pitch>
+          <step>B</step>
+          <octave>4</octave>
+          </pitch>
+        <duration>1</duration>
+        <voice>1</voice>
+        <type>quarter</type>
+        <stem>down</stem>
+        </note>
+      <note default-x="318.13" default-y="-15.00">
+        <pitch>
+          <step>C</step>
+          <octave>5</octave>
+          </pitch>
+        <duration>1</duration>
+        <voice>1</voice>
+        <type>quarter</type>
+        <stem>down</stem>
+        </note>
+      <barline location="right">
+        <bar-style>light-heavy</bar-style>
+        </barline>
+      </measure>
+    </part>
+  </score-partwise>