using UnityEngine; using UnityEngine.Rendering; #if UNITY_EDITOR_WIN || UNITY_STANDALONE_WIN public class OVRSandwichComposition : OVRCameraComposition { public float frameRealtime; public Camera fgCamera; public Camera bgCamera; public class HistoryRecord { public float timestamp = float.MinValue; public RenderTexture fgRenderTexture; public RenderTexture bgRenderTexture; public RenderTexture boundaryMeshMaskTexture; } public readonly int historyRecordCount = 8; // enough to compensate 88ms latency @ 90 fps public readonly HistoryRecord[] historyRecordArray; public int historyRecordCursorIndex = 0; public GameObject cameraProxyPlane; public Camera compositionCamera; public OVRSandwichCompositionManager compositionManager; private int _cameraFramePlaneLayer = -1; // find an unnamed layer between 24..29 public int cameraFramePlaneLayer { get { if (_cameraFramePlaneLayer < 0) { for (int i=24; i<=29; ++i) { string layerName = LayerMask.LayerToName(i); if (layerName == null || layerName.Length == 0) { _cameraFramePlaneLayer = i; break; } } if (_cameraFramePlaneLayer == -1) { Debug.LogWarning("Unable to find an unnamed layer between 24 and 29."); _cameraFramePlaneLayer = 25; } Debug.LogFormat("Set the CameraFramePlaneLayer in SandwichComposition to {0}. Please do NOT put any other gameobject in this layer.", _cameraFramePlaneLayer); } return _cameraFramePlaneLayer; } } public override OVRManager.CompositionMethod CompositionMethod() { return OVRManager.CompositionMethod.Sandwich; } public OVRSandwichComposition(GameObject parentObject, Camera mainCamera, OVRManager.CameraDevice cameraDevice, bool useDynamicLighting, OVRManager.DepthQuality depthQuality) : base(cameraDevice, useDynamicLighting, depthQuality) { frameRealtime = Time.realtimeSinceStartup; historyRecordCount = OVRManager.instance.sandwichCompositionBufferedFrames; if (historyRecordCount < 1) { Debug.LogWarning("Invalid sandwichCompositionBufferedFrames in OVRManager. It should be at least 1"); historyRecordCount = 1; } if (historyRecordCount > 16) { Debug.LogWarning("The value of sandwichCompositionBufferedFrames in OVRManager is too big. It would consume a lot of memory. It has been override to 16"); historyRecordCount = 16; } historyRecordArray = new HistoryRecord[historyRecordCount]; for (int i=0; i(); fgCamera.depth = 200; fgCamera.clearFlags = CameraClearFlags.SolidColor; fgCamera.backgroundColor = Color.clear; fgCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers); fgCamera.nearClipPlane = mainCamera.nearClipPlane; fgCamera.farClipPlane = mainCamera.farClipPlane; GameObject bgObject = new GameObject("MRSandwichBackgroundCamera"); bgObject.transform.parent = parentObject.transform; bgCamera = bgObject.AddComponent(); bgCamera.depth = 100; bgCamera.clearFlags = mainCamera.clearFlags; bgCamera.backgroundColor = mainCamera.backgroundColor; bgCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers); bgCamera.nearClipPlane = mainCamera.nearClipPlane; bgCamera.farClipPlane = mainCamera.farClipPlane; // Create cameraProxyPlane for clipping Debug.Assert(cameraProxyPlane == null); cameraProxyPlane = GameObject.CreatePrimitive(PrimitiveType.Quad); cameraProxyPlane.name = "MRProxyClipPlane"; cameraProxyPlane.transform.parent = parentObject.transform; cameraProxyPlane.GetComponent().enabled = false; cameraProxyPlane.GetComponent().shadowCastingMode = UnityEngine.Rendering.ShadowCastingMode.Off; Material clipMaterial = new Material(Shader.Find("Oculus/OVRMRClipPlane")); cameraProxyPlane.GetComponent().material = clipMaterial; clipMaterial.SetColor("_Color", Color.clear); clipMaterial.SetFloat("_Visible", 0.0f); cameraProxyPlane.transform.localScale = new Vector3(1000, 1000, 1000); cameraProxyPlane.SetActive(true); OVRMRForegroundCameraManager foregroundCameraManager = fgCamera.gameObject.AddComponent(); foregroundCameraManager.clipPlaneGameObj = cameraProxyPlane; GameObject compositionCameraObject = new GameObject("MRSandwichCaptureCamera"); compositionCameraObject.transform.parent = parentObject.transform; compositionCamera = compositionCameraObject.AddComponent(); compositionCamera.stereoTargetEye = StereoTargetEyeMask.None; compositionCamera.depth = float.MaxValue; compositionCamera.rect = new Rect(0.0f, 0.0f, 1.0f, 1.0f); compositionCamera.clearFlags = CameraClearFlags.Depth; compositionCamera.backgroundColor = mainCamera.backgroundColor; compositionCamera.cullingMask = 1 << cameraFramePlaneLayer; compositionCamera.nearClipPlane = mainCamera.nearClipPlane; compositionCamera.farClipPlane = mainCamera.farClipPlane; if (!hasCameraDeviceOpened) { Debug.LogError("Unable to open camera device " + cameraDevice); } else { Debug.Log("SandwichComposition activated : useDynamicLighting " + (useDynamicLighting ? "ON" : "OFF")); CreateCameraFramePlaneObject(parentObject, compositionCamera, useDynamicLighting); cameraFramePlaneObject.layer = cameraFramePlaneLayer; RefreshRenderTextures(mainCamera); compositionManager = compositionCamera.gameObject.AddComponent(); compositionManager.fgTexture = historyRecordArray[historyRecordCursorIndex].fgRenderTexture; compositionManager.bgTexture = historyRecordArray[historyRecordCursorIndex].bgRenderTexture; } } public override void Update(Camera mainCamera) { if (!hasCameraDeviceOpened) { return; } frameRealtime = Time.realtimeSinceStartup; ++historyRecordCursorIndex; if (historyRecordCursorIndex >= historyRecordCount) { historyRecordCursorIndex = 0; } if (!OVRPlugin.SetHandNodePoseStateLatency(OVRManager.instance.handPoseStateLatency)) { Debug.LogWarning("HandPoseStateLatency is invalid. Expect a value between 0.0 to 0.5, get " + OVRManager.instance.handPoseStateLatency); } RefreshRenderTextures(mainCamera); bgCamera.clearFlags = mainCamera.clearFlags; bgCamera.backgroundColor = mainCamera.backgroundColor; bgCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers); fgCamera.cullingMask = mainCamera.cullingMask & (~OVRManager.instance.extraHiddenLayers); if (OVRMixedReality.useFakeExternalCamera || OVRPlugin.GetExternalCameraCount() == 0) { OVRPose worldSpacePose = new OVRPose(); OVRPose trackingSpacePose = new OVRPose(); trackingSpacePose.position = OVRMixedReality.fakeCameraPositon; trackingSpacePose.orientation = OVRMixedReality.fakeCameraRotation; worldSpacePose = OVRExtensions.ToWorldSpacePose(trackingSpacePose); RefreshCameraPoses(OVRMixedReality.fakeCameraFov, OVRMixedReality.fakeCameraAspect, worldSpacePose); } else { OVRPlugin.CameraExtrinsics extrinsics; OVRPlugin.CameraIntrinsics intrinsics; // So far, only support 1 camera for MR and always use camera index 0 if (OVRPlugin.GetMixedRealityCameraInfo(0, out extrinsics, out intrinsics)) { OVRPose worldSpacePose = ComputeCameraWorldSpacePose(extrinsics); float fovY = Mathf.Atan(intrinsics.FOVPort.UpTan) * Mathf.Rad2Deg * 2; float aspect = intrinsics.FOVPort.LeftTan / intrinsics.FOVPort.UpTan; RefreshCameraPoses(fovY, aspect, worldSpacePose); } else { Debug.LogWarning("Failed to get external camera information"); } } compositionCamera.GetComponent().boundaryMeshMaskTexture = historyRecordArray[historyRecordCursorIndex].boundaryMeshMaskTexture; HistoryRecord record = GetHistoryRecordForComposition(); UpdateCameraFramePlaneObject(mainCamera, compositionCamera, record.boundaryMeshMaskTexture); OVRSandwichCompositionManager compositionManager = compositionCamera.gameObject.GetComponent(); compositionManager.fgTexture = record.fgRenderTexture; compositionManager.bgTexture = record.bgRenderTexture; cameraProxyPlane.transform.position = fgCamera.transform.position + fgCamera.transform.forward * cameraFramePlaneDistance; cameraProxyPlane.transform.LookAt(cameraProxyPlane.transform.position + fgCamera.transform.forward); } public override void Cleanup() { base.Cleanup(); Camera[] cameras = { fgCamera, bgCamera, compositionCamera }; foreach (Camera c in cameras) { OVRCompositionUtil.SafeDestroy(c.gameObject); } fgCamera = null; bgCamera = null; compositionCamera = null; Debug.Log("SandwichComposition deactivated"); } private RenderTextureFormat DesiredRenderTextureFormat(RenderTextureFormat originalFormat) { if (originalFormat == RenderTextureFormat.RGB565) { return RenderTextureFormat.ARGB1555; } else if (originalFormat == RenderTextureFormat.RGB111110Float) { return RenderTextureFormat.ARGBHalf; } else { return originalFormat; } } protected void RefreshRenderTextures(Camera mainCamera) { int width = Screen.width; int height = Screen.height; RenderTextureFormat format = mainCamera.targetTexture ? DesiredRenderTextureFormat(mainCamera.targetTexture.format) : RenderTextureFormat.ARGB32; int depth = mainCamera.targetTexture ? mainCamera.targetTexture.depth : 24; Debug.Assert(fgCamera != null && bgCamera != null); HistoryRecord record = historyRecordArray[historyRecordCursorIndex]; record.timestamp = frameRealtime; if (record.fgRenderTexture == null || record.fgRenderTexture.width != width || record.fgRenderTexture.height != height || record.fgRenderTexture.format != format || record.fgRenderTexture.depth != depth) { record.fgRenderTexture = new RenderTexture(width, height, depth, format); record.fgRenderTexture.name = "Sandwich FG " + historyRecordCursorIndex.ToString(); } fgCamera.targetTexture = record.fgRenderTexture; if (record.bgRenderTexture == null || record.bgRenderTexture.width != width || record.bgRenderTexture.height != height || record.bgRenderTexture.format != format || record.bgRenderTexture.depth != depth) { record.bgRenderTexture = new RenderTexture(width, height, depth, format); record.bgRenderTexture.name = "Sandwich BG " + historyRecordCursorIndex.ToString(); } bgCamera.targetTexture = record.bgRenderTexture; if (OVRManager.instance.virtualGreenScreenType != OVRManager.VirtualGreenScreenType.Off) { if (record.boundaryMeshMaskTexture == null || record.boundaryMeshMaskTexture.width != width || record.boundaryMeshMaskTexture.height != height) { record.boundaryMeshMaskTexture = new RenderTexture(width, height, 0, RenderTextureFormat.R8); record.boundaryMeshMaskTexture.name = "Boundary Mask " + historyRecordCursorIndex.ToString(); record.boundaryMeshMaskTexture.Create(); } } else { record.boundaryMeshMaskTexture = null; } Debug.Assert(fgCamera.targetTexture != null && bgCamera.targetTexture != null && (OVRManager.instance.virtualGreenScreenType == OVRManager.VirtualGreenScreenType.Off || record.boundaryMeshMaskTexture != null)); } protected HistoryRecord GetHistoryRecordForComposition() { float expectedTime = frameRealtime - OVRManager.instance.sandwichCompositionRenderLatency; int currIndex = historyRecordCursorIndex; int prevIndex = currIndex - 1; if (prevIndex < 0) { prevIndex = historyRecordCount - 1; } while (prevIndex != historyRecordCursorIndex) { if (historyRecordArray[prevIndex].timestamp <= expectedTime) { float timeToCurrIndex = historyRecordArray[currIndex].timestamp - expectedTime; float timeToPrevIndex = expectedTime - historyRecordArray[prevIndex].timestamp; return timeToCurrIndex <= timeToPrevIndex ? historyRecordArray[currIndex] : historyRecordArray[prevIndex]; } currIndex = prevIndex; prevIndex = currIndex - 1; if (prevIndex < 0) prevIndex = historyRecordCount - 1; } // return the earliest frame return historyRecordArray[currIndex]; } protected void RefreshCameraPoses(float fovY, float aspect, OVRPose pose) { Camera[] cameras = { fgCamera, bgCamera, compositionCamera }; foreach (Camera c in cameras) { c.fieldOfView = fovY; c.aspect = aspect; c.transform.FromOVRPose(pose); } } public class OVRSandwichCompositionManager : MonoBehaviour { public RenderTexture fgTexture; public RenderTexture bgTexture; public Material alphaBlendMaterial; void Start() { Shader alphaBlendShader = Shader.Find("Oculus/UnlitTransparent"); if (alphaBlendShader == null) { Debug.LogError("Unable to create transparent shader"); return; } alphaBlendMaterial = new Material(alphaBlendShader); } private void OnPreRender() { if (fgTexture == null || bgTexture == null || alphaBlendMaterial == null) { Debug.LogError("OVRSandwichCompositionManager has not setup properly"); return; } Graphics.Blit(bgTexture, RenderTexture.active); } void OnPostRender() { if (fgTexture == null || bgTexture == null || alphaBlendMaterial == null) { Debug.LogError("OVRSandwichCompositionManager has not setup properly"); return; } Graphics.Blit(fgTexture, RenderTexture.active, alphaBlendMaterial); } } } #endif