Phaser3：带麦克风输入的着色器（Web）

Question

我想用 Phaser 3 做一个音频输入可视化器，我\xe2\x80\x99m 试图将麦克风输入到着色器，但我\xe2\x80\x99t 找不到一种方法让它工作。

\n

我对着色器有基本的了解，我可以使用图像纹理，但我真的不了解如何提供声音。我检查了在 Three.js 中制作的工作示例：Three.js webaudio - 可视化工具，并且我已经设法以 1024 个数字的 Uint8Array 形式从麦克风获取声音输入。

\n

这里 \xe2\x80\x99s 着色器 I\xe2\x80\x99m 使用：

\n

// simplesound.gsl.js\n#ifdef GL_ES\nprecision highp float;\n#endif\n\nprecision mediump float;\nuniform vec2 resolution;\nuniform sampler2D iChannel0;\n\nvarying vec2 fragCoord;\n\nvoid main() {\n  vec2 uv = fragCoord.xy / resolution.xy;\n  vec2 mu = texture2D(iChannel0, uv).rg;\n\n  float y = uv.y - mu.x;\n  y = smoothstep(0., 0.02, abs(y - 0.1));\n\n  gl_FragColor = vec4(y);\n}\n

\n

这是我的场景代码，试图使其工作：

\n

import Phaser from \'phaser\';\n// This will provide the array mentioned above with code that will use `navigator.getUserMedia`.\nimport { setupAudioContext } from \'../audiostream\';\n\nexport default class MainScene2 extends Phaser.Scene {\n  constructor() {\n    super({ key: \'MainScene2\' });\n  }\n\n  preload() {\n    this.load.glsl(\'simplesound\', \'/static/simplesound.glsl.js\');\n  }\n\n  create() {\n    this.shader = this.add.shader(\'simplesound\', 400, 300, 800, 600);\n\n    // When the user presses the \'g\' key we will start listening for mic input\n    const GKey = this.input.keyboard.addKey(\'G\');\n\n    GKey.on(\'down\', () => {\n      setupAudioContext((array) => {\n        // this array is the array mentioned above, in the three.js example they do something like creating\n        // a texture from this input and providing that texture to the shader uniform. I tried different things but\n        // nothing worked :(\n        //\n        // I tried using this.shader.setChannel0 and this.shader.setUniform but nothing seems to work as well.\n      });\n    });\n  }\n}\n

\n

我已经尝试让这项工作有一段时间了，但什么也没得到:(

\n

Answer 1

对于没有着色器的可能解决方案，仅使用移相器和javascript可能看起来像这样（实际上没有着色器，但我也对着色器版本的外观非常感兴趣）。

在此演示中，我使用音频文件中的数据。为了使其适用于您的用例，您只需将麦克风数据插入变量即可data。

演示：（
代码中的注释是为了突出主要思想）
单击并等待几秒钟。顺便说一句：我添加了一些屏幕震动，以使演示更加生动。

document.body.style = 'margin:0;';
    
    var data = [];
    var playing =  -1;
    var audioContext = new (window.AudioContext || window.webkitAudioContext)();
    var analyser = audioContext.createAnalyser();
    var buffer;
    var source;
    var url = 'https://labs.phaser.io/assets/audio/Rossini - William Tell Overture (8 Bits Version)/left.ogg'

    // START Audio part for Demo
    function loadAudio() {
        var request = new XMLHttpRequest();
        request.open('GET', url, true);
        request.responseType = 'arraybuffer';
        request.onload = function() {
            audioContext.decodeAudioData(request.response, function(buf) {
                buffer = buf;
                playAudio();
            });
        };
        request.send();
    }
    
    function playAudio() {
        source = audioContext.createBufferSource();
        source.buffer = buffer;
        source.connect(audioContext.destination);
        source.connect(analyser);
        source.start(0);
    }

    // END Audio part for Demo

    var config = {
        type: Phaser.AUTO,
        width: 536,
        height: 183,
        scene: {
            create,
            update
        },
        banner: false
    }; 

    var game = new Phaser.Game(config);

    // this would be the varibale that should be updated from the audio source
    var markers;
    var createRandomData = true;

    function create () {
        // Start create Marker texture 
        // this could be remove if you want to load an actual image
        let g = this.make.graphics({x: 0, y: 0, add: false});
        
        g.lineStyle(10, 0xffffff);
        g.beginPath();
        g.moveTo(0, 0);
        g.lineTo(50, 0);

        g.strokePath();

        g.generateTexture('marker', 30, 10);
        // End create Marker texture 
    
        // Create the markers
        // the repeat property sets how many markers you want to display, if you want all 1024 => that would be your value
        markers = this.add.group({ key: 'marker', repeat: 50,
            setXY: { x: 10, y: 10, stepX: 35 }, setOrigin: { x: 0, y: 0}});
        
        this.add.rectangle(10, 10, 180, 20, 0).setOrigin(0);
        let label = this.add.text( 10, 10, 'Click to start music', {color: 'red', fontSize:'20px', fontStyle:'bold'} )
        
        // start and stop the playback of music     
        this.input.on('pointerdown', function () {
            switch (playing) {
                case -1:
                    loadAudio();
                    playing = 1;
                    label.setText('Click to stop music');
                    break;
                case 0:
                    playAudio();
                    playing = 1;
                    label.setText('Click to stop music');
                    break;
                case 1:
                    source.stop();
                    playing = 0;
                    label.setText('Click to start music');
                    break;
            }   
        }); 

    }

    function update(){
        if (markers){
            // here we update the y-position of the marker in depending on the value of the data. 
            // ( min y = 10 and max y ~ 245)
            markers.children.iterate(function (child, idx) {
                child.y = 10 + (config.height - 20) / 255 * data[idx];
                
                // you could even add some camera shake, for more effect
                if(idx < 3 && data[idx] > 253){
                    this.cameras.main.shake(30);
                }
            }, this);
            
            // if the analyser is valid and updates the data variable
            // this part could some where else, I just wanted to keep the code concise
            if(analyser){
              var spectrums = new Uint8Array(analyser.frequencyBinCount);
              analyser.getByteFrequencyData(spectrums);

              // convert data to a plain array and updating the data variable
              data = [].slice.call(spectrums);
            }
        }
    }

<script src="https://cdn.jsdelivr.net/npm/phaser@3.55.2/dist/phaser.js"></script>

基本上，该应用程序“仅”根据从音频文件加载的音频数组返回的值来更改每个标记的 Y 位置。
免责声明：这是粗略的演示代码，如果应该在生产中使用，可以使用一些清理/改进。