Snippets

Depthkit Depthkit Unity Render Streaming Support Scripts

Created by Tim Scaffidi last modified
using Unity.WebRTC;
using UnityEngine;
using UnityEngine.Experimental.Rendering;
using Depthkit;
using Unity.RenderStreaming;
using System.Collections;
using System;

namespace Depthkit
{
    public class DepthkitMetadataChannel : DataChannelBase
    {
        [SerializeField] private Clip m_clip = null;

        private float lastMetadataRequestTime = 0.0f;
        private bool received = false;
        private bool isStarted = false;

        internal enum MessageCode
        {
            Received,
            Request
        }

        void Awake()
        {
            OnStartedChannel += channelStarted;
            OnStoppedChannel += channelStopped;
        }

        public override void SetChannel(string connectionId, RTCDataChannel channel)
        {
            Debug.Log("DepthkitMetadataChannel::SetChannel()");

            base.SetChannel(connectionId, channel);
        }

        public void SendMetadata()
        {
            Debug.Log("DepthkitMetadataChannel::SendMetadata()");

            // Load metadata from clip
            Metadata metadata = parseJsonMetadata(m_clip.metadataFile.text);

            // Modify metadata with the current clip bounds from the MeshSource component
            Bounds bounds = m_clip.gameObject.GetComponent<MeshSource>().GetLocalBounds();

            metadata.boundsSize = bounds.size;
            metadata.boundsCenter = new Vector3(bounds.center.x, bounds.center.y * -1.0f, bounds.center.z * -1.0f);

            // Modify metadata texture size with the video resolution to be sent over the network
            VideoStreamSender videoSender = GetComponent<VideoStreamSender>();
            if (videoSender)
            {
                metadata.textureWidth = (int)videoSender.width;
                metadata.textureHeight = (int)videoSender.height;
            }

            Channel.Send(JsonUtility.ToJson(metadata));
        }

        protected override void OnMessage(byte[] bytes)
        {
            Debug.Log("DepthkitMetadataChannel::OnMessage()");
            if (IsLocal)
            {
                MessageCode msg;
                if (Enum.TryParse<MessageCode>(System.Text.Encoding.UTF8.GetString(bytes, 0, bytes.Length), out msg))
                {
                    switch (msg)
                    {
                        case MessageCode.Request:
                            Debug.Log("Remote requested metadata, sending.");
                            received = false;
                            SendMetadata();
                            break;
                        case MessageCode.Received:
                            Debug.Log("Remote acknowledged receipt of metadata.");
                            received = true;
                            break;
                        default:
                            break;
                    }
                }
            }
            else
            {
                m_clip.LoadMetadata(System.Text.Encoding.UTF8.GetString(bytes, 0, bytes.Length));

                StudioMeshSource studioMeshSource = m_clip.gameObject.GetComponent<StudioMeshSource>();
                if (studioMeshSource)
                {
                    studioMeshSource.ResetVolumeBounds();
                }

                Debug.Log("Remote metadata texture size: " + m_clip.metadata.textureWidth + "x"+ m_clip.metadata.textureHeight);

                received = true;
                Channel.Send(MessageCode.Received.ToString());
            }
        }

        void channelStarted(string connectionId)
        {
            Debug.Log("DepthkitMetadataChannel::channelStarted(" + connectionId + ")");

            isStarted = true;

            if (IsLocal)
            {
                SendMetadata();
            }
            else
            {
                lastMetadataRequestTime = Time.timeSinceLevelLoad;
            }
        }

        void channelStopped(string connectionId)
        {
            Debug.Log("DepthkitMetadataChannel::channelStopped(" + connectionId + ")");
            received = false;
            isStarted = false;
        }

        protected void Update()
        {
            if (isStarted && !IsLocal && !received && Time.timeSinceLevelLoad - lastMetadataRequestTime > 2.5f)
            {
                lastMetadataRequestTime = Time.timeSinceLevelLoad;
                Debug.Log("Requesting metadata");
                Channel.Send(MessageCode.Request.ToString());
            }
        }

        private Metadata FromSinglePerspective(Metadata.MetadataSinglePerspective md)
        {
            return new Depthkit.Metadata
            {
                _versionMajor = 0,
                _versionMinor = 4,
                format = md.format,
                boundsCenter = md.boundsCenter,
                boundsSize = md.boundsSize,
                textureWidth = md.textureWidth,
                textureHeight = md.textureHeight,
                perspectives = new Metadata.Perspective[] {
                    new Metadata.Perspective {
                        depthImageSize = md.depthImageSize,
                        depthPrincipalPoint = md.depthPrincipalPoint,
                        depthFocalLength = md.depthFocalLength,
                        farClip = md.farClip,
                        nearClip = md.nearClip,
                        extrinsics = md.extrinsics,
                        crop = md.crop,
                        clipEpsilon = md.clipEpsilon
                    }
                }
            };
        }

        private Metadata parseJsonMetadata(string jsonString)
        {
            const float eps = 0.00000001f;

            Metadata metadata;
            Metadata.MetadataVersion mdVersion = JsonUtility.FromJson<Metadata.MetadataVersion>(jsonString);

            // Read and upgrade old single perspective format.
            if (mdVersion._versionMajor == 0 && mdVersion._versionMinor < 3)
            {
                Metadata.MetadataSinglePerspective md = JsonUtility.FromJson<Metadata.MetadataSinglePerspective>(jsonString);

                //for version 1.0 (from Depthkit Visualize) fill in defaults for missing parameters
                if (mdVersion.format == "perpixel" && mdVersion._versionMinor == 1)
                {
                    md.numAngles = 1;

                    // check if we have a zero'd crop (is this possible?), if so default to full window
                    if (md.crop.x <= eps && md.crop.y <= eps && md.crop.z <= eps && md.crop.w <= eps)
                    {
                        md.crop = new Vector4(0.0f, 0.0f, 1.0f, 1.0f);
                    }

                    md.extrinsics = Matrix4x4.identity;
                }

                if (md.clipEpsilon < eps)
                {
                    md.clipEpsilon = 0.005f; // default depth clipping epsilon, set for older versions of metadata
                }

                metadata = FromSinglePerspective(md);
                metadata.numRows = 1;
                metadata.numColumns = 1;
            }
            else
            {
                // Read multiperspective format.
                metadata = JsonUtility.FromJson<Metadata>(jsonString);
                if (mdVersion._versionMinor == 3)
                {
                    metadata.numRows = 1;
                    metadata.numColumns = metadata.numAngles;
                }
            }

            metadata.perspectivesCount = metadata.perspectives.Length;

            return metadata;
        }
    }
}
using System.Linq;
using UnityEngine;
using UnityEngine.UI;

namespace Unity.RenderStreaming.Samples
{
    class SimpleConnection : MonoBehaviour
    {
#pragma warning disable 0649
        [SerializeField] private SignalingManager signalingManager;
        [SerializeField] private SingleConnection singleConnection;
        [SerializeField] private string connectionId;
#pragma warning restore 0649

        internal enum ConnectionState
        {
            Disconnected,
            Connecting,
            Connected,
            Disconnecting
        }

        private ConnectionState connectionState = ConnectionState.Disconnected;

        void Awake()
        {
            foreach (var audio in singleConnection.Streams.OfType<AudioStreamReceiver>())
            {
                audio.OnUpdateReceiveAudioSource += source =>
                {
                    Debug.Log("Audio source updated, configuring");
                    source.loop = true;
                    source.Play();
                };
            }

        }

        void Start()
        {
            connectionState = ConnectionState.Disconnected;
            for(int i = 0; i < Microphone.devices.Length; i++)
            {
                Debug.Log("Audio Device " + i + ": " + Microphone.devices[i]);
            }

            if (!signalingManager.runOnAwake)
            {
                signalingManager.Run();
            }
        }

        private float connectionCreatedTime = 0.0f;

        protected void Update()
        {
            // Note: this delay of 1 second is intended to allow time for establishing a connection with the signaling
            // server in leu of a proper API for detecting that the signaling server is connected.
            // If modifying this script to be powered by a GUI where the user initiates the CreateConnection call below,
            // there will be a built in delay by virtue of the user interaction and this delay can likely be removed
            if (Time.timeSinceLevelLoad < 2.5f) return;

            if (singleConnection.IsConnected(connectionId))
            {
                if (connectionState == ConnectionState.Connecting)
                {
                    Debug.Log("Connected " + connectionId);
                    connectionState = ConnectionState.Connected;
                }
            }
            else
            {
                if (connectionState == ConnectionState.Disconnected)
                {
                    Debug.Log("Creating connection " + connectionId);
                    singleConnection.CreateConnection(connectionId);
                    connectionState = ConnectionState.Connecting;
                    connectionCreatedTime = Time.timeSinceLevelLoad;
                }
                else if (connectionState == ConnectionState.Connected)
                {
                    Debug.Log("Deleting connection " + connectionId);
                    singleConnection.DeleteConnection(connectionId);
                    connectionState = ConnectionState.Disconnected;
                }
                else if (connectionState == ConnectionState.Connecting && Time.timeSinceLevelLoad - connectionCreatedTime > 5.0f)
                {
                    if (singleConnection.ExistConnection(connectionId))
                        singleConnection.DeleteConnection(connectionId);
                    Debug.Log("Timed out creating connection, deleting " + connectionId);
                    connectionState = ConnectionState.Disconnected;
                }
            }

            //Debug.Log("Connection Exists: " + singleConnection.ExistConnection(connectionId) + ". IsConnected: " + singleConnection.IsConnected(connectionId) + ". ConnectionState: " + connectionState);

        }

        void OnDestroy()
        {
            if (singleConnection.IsConnected(connectionId))
            {
                singleConnection.DeleteConnection(connectionId);
            }
        }

        void OnApplicationQuit()
        {
            if (singleConnection.IsConnected(connectionId))
            {
                singleConnection.DeleteConnection(connectionId);
            }
        }
    }
}
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Unity.RenderStreaming;

public class VideoSenderBitrateOverride : MonoBehaviour
{
    public VideoStreamSender videoStreamSender;

    public uint minBitrate = 5000;
    public uint maxBirate = 30000;

    // Start is called before the first frame update
    void Start()
    {
        videoStreamSender.SetBitrate(minBitrate, maxBirate);
    }

    // Update is called once per frame
    void Update()
    {
    }
}
/************************************************************************************

Depthkit Unity SDK License v1
Copyright 2016-2019 Scatter All Rights reserved.

Licensed under the Scatter Software Development Kit License Agreement (the "License");
you may not use this SDK except in compliance with the License,
which is provided at the time of installation or download,
or which otherwise accompanies this software in either electronic or hard copy form.

You may obtain a copy of the License at http://www.depthkit.tv/license-agreement-v1

Unless required by applicable law or agreed to in writing,
the SDK distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and limitations under the License.

************************************************************************************/

using UnityEngine;
using System;
using System.Collections;
using Depthkit;
using Unity.RenderStreaming;

namespace Depthkit
{
    /// <summary>
    /// Implementation of the Depthkit player with the Unity VideoPlayer-based backend.
    /// </summary>
    [AddComponentMenu("Depthkit/Players/Depthkit WebRTC Player")]
    public class WebrtcPlayer : Depthkit.ClipPlayer
    {
        [SerializeField]
        protected VideoStreamReceiver m_videoReceiver;

        [SerializeField, HideInInspector]
        protected AudioSource m_audioSource;

        protected int m_frame = 0;

        void Start()
        {
            m_frame = 0;
        }

        public override void CreatePlayer()
        {
            m_audioSource = gameObject.GetComponent<AudioSource>();

            if (m_audioSource == null)
            {
                m_audioSource = gameObject.AddComponent<AudioSource>();
                // Note this just creates the audio source, it must be manually hooked up as the target of the AudioStreamReceiver
            }
        }

        public override bool IsPlayerCreated()
        {
            return m_videoReceiver != null;
        }

        public override bool IsPlayerSetup()
        {
            return IsPlayerCreated();
        }

        /// <summary>
        /// Sets the video from a path. Assumed relative to data foldder file path.</summary>
        public override void SetVideoPath(string path)
        {
        }

        /// <summary>
        /// Get the absolute path to the video.</summary>
        public override string GetVideoPath()
        {
            return "";
        }

        public override void StartVideoLoad()
        {
        }

        public override IEnumerator Load()
        {
            yield return null;
        }

        public override void OnMetadataUpdated(Depthkit.Metadata metadata) { /* do nothing */ }

        public override IEnumerator LoadAndPlay()
        {
            yield return null;
        }

        public override void Play()
        {
        }

        public override void Pause()
        {
        }

        public override void Stop()
        {
        }

        public override int GetCurrentFrame()
        {
            return m_frame;
        }
        public override double GetCurrentTime()
        {
            return 0.0;
        }

        public override double GetDuration()
        {
            return Double.PositiveInfinity;
        }

        public override Texture GetTexture()
        {
            ++m_frame;

            return m_videoReceiver?.texture;
        }
        public override bool IsTextureFlipped()
        {
            return false;
        }
        public override GammaCorrection GammaCorrectDepth()
        {
            if (QualitySettings.activeColorSpace == ColorSpace.Linear)
            {
#if UNITY_2018_2_OR_NEWER
                return GammaCorrection.LinearToGammaSpace;
#elif UNITY_2017_1_OR_NEWER
                //https://issuetracker.unity3d.com/issues/regression-videos-are-dark-when-using-linear-color-space?page=1
                Debug.LogWarning("Unity Video Player does not display correct color on Windows between version 2017.1 and 2018.2. Use AVPro, switch to Gamma Color Space, or upgrade Unity to use Depthkit with this project.");
                return GammaCorrection.LinearToGammaSpace2x;
#else
                return GammaCorrection.LinearToGammaSpace;
#endif
            }
            else
            {
                return GammaCorrection.None;
            }
        }
        public override GammaCorrection GammaCorrectColor()
        {
            if (QualitySettings.activeColorSpace == ColorSpace.Linear)
            {
#if UNITY_2018_2_OR_NEWER
                return GammaCorrection.None;
#elif UNITY_2017_1_OR_NEWER
                return GammaCorrection.LinearToGammaSpace;
#else
                return GammaCorrection.None;
#endif
            }
            else
            {
                return GammaCorrection.None;
            }
        }
        public override bool IsPlaying()
        {
            return m_videoReceiver != null && m_videoReceiver.texture != null;
        }

        public override void RemoveComponents()
        {
        }

        public override string GetPlayerTypeName()
        {
            return typeof(Depthkit.WebrtcPlayer).Name;
        }

        public new static string GetPlayerPrettyName()
        {
            return "WebRTC Player";
        }

        public override void Seek(float toTime)
        {
        }

        public override uint GetVideoWidth()
        {
            return m_videoReceiver != null && m_videoReceiver.texture != null ? (uint)m_videoReceiver.texture.width : 0u;
        }
        public override uint GetVideoHeight()
        {
            return m_videoReceiver != null && m_videoReceiver.texture != null ? (uint)m_videoReceiver.texture.height : 0u;
        }

        public override bool SupportsPosterFrame()
        {
            return true;
        }
    }
}

Comments (0)

HTTPS SSH

You can clone a snippet to your computer for local editing. Learn more.