次世代AI × XR（VR / AR / MR / WebXR）× AIプログラミング × Google Cloud Vertex AI 完全統合アーキテクチャ：
空間コンピューティングとマルチモーダル基盤による自律インタラクション

using System;
using System.Collections;
using System.Collections.Generic;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using UnityEngine;

[Serializable]
public class SpatialTransformPayload
{
    public string target_object_id;
    public float translation_x;
    public float translation_y;
    public float translation_z;
    public float rotation_qx;
    public float rotation_qy;
    public float rotation_qz;
    public float rotation_qw;
    public string interaction_mode;
    public string spatial_speech_text;
    public float confidence_score;
}

public class VertexAISpatialClient : MonoBehaviour
{
    [Header("Network Endpoint")]
    [SerializeField] private string gatewayUrl = "wss://spatial-gateway.ok.st/v1/stream";

    [Header("Target Objects")]
    [SerializeField] private Transform guideAvatar;
    [SerializeField] private AudioSource spatialAudioSource;

    private ClientWebSocket _webSocket;
    private CancellationTokenSource _cts;
    private Vector3 _targetPosition;
    private Quaternion _targetRotation;
    private readonly float _slerpSpeed = 8.0f;

    private void Start()
    {
        _targetPosition = guideAvatar.position;
        _targetRotation = guideAvatar.rotation;
        _cts = new CancellationTokenSource();
        _ = ConnectToVertexAIGatewayAsync();
    }

    private async Task ConnectToVertexAIGatewayAsync()
    {
        try
        {
            _webSocket = new ClientWebSocket();
            _webSocket.Options.SetRequestHeader("X-Client-Platform", "Unity-MetaQuest3");
            Uri serverUri = new Uri(gatewayUrl);
            await _webSocket.ConnectAsync(serverUri, _cts.Token);
            Debug.Log("[VertexAI] WebSocket Connected to Spatial Intelligence Gateway.");

            // Start bidirectional listener loop
            _ = ReceiveStreamingActionsAsync();
        }
        catch (Exception ex)
        {
            Debug.LogError($"[VertexAI] Connection failed: {ex.Message}");
        }
    }

    private async Task ReceiveStreamingActionsAsync()
    {
        byte[] buffer = new byte[4096];
        while (_webSocket.State == WebSocketState.Open && !_cts.IsCancellationRequested)
        {
            var result = await _webSocket.ReceiveAsync(new ArraySegment<byte>(buffer), _cts.Token);
            if (result.MessageType == WebSocketMessageType.Close)
            {
                await _webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closing", CancellationToken.None);
                break;
            }

            string jsonMessage = Encoding.UTF8.GetString(buffer, 0, result.Count);
            ProcessSpatialPayload(jsonMessage);
        }
    }

    private void ProcessSpatialPayload(string json)
    {
        try
        {
            var payload = JsonUtility.FromJson<SpatialTransformPayload>(json);
            if (payload == null) return;

            // Update Target Positions for Slerp interpolation in Update()
            _targetPosition = new Vector3(payload.translation_x, payload.translation_y, payload.translation_z);
            _targetRotation = new Quaternion(payload.rotation_qx, payload.rotation_qy, payload.rotation_qz, payload.rotation_qw);

            if (!string.IsNullOrEmpty(payload.spatial_speech_text))
            {
                Debug.Log($"[VertexAI Speech] {payload.spatial_speech_text} (Score: {payload.confidence_score})");
                // Trigger 3D spatial voice synthesizer
                if (spatialAudioSource != null && !spatialAudioSource.isPlaying)
                {
                    spatialAudioSource.Play();
                }
            }
        }
        catch (Exception ex)
        {
            Debug.LogWarning($"[VertexAI] Payload parsing error: {ex.Message}");
        }
    }

    private void Update()
    {
        if (guideAvatar == null) return;

        // Smooth physical dampening (Interpolation without camera jitter)
        guideAvatar.position = Vector3.Lerp(guideAvatar.position, _targetPosition, Time.deltaTime * _slerpSpeed);
        guideAvatar.rotation = Quaternion.Slerp(guideAvatar.rotation, _targetRotation, Time.deltaTime * _slerpSpeed);
    }

    private void OnDestroy()
    {
        _cts?.Cancel();
        _webSocket?.Dispose();
    }
}

import * as THREE from 'three';
import initWasm, { downsample_point_cloud, compute_se3_transform } from './pkg/wasm_spatial_math.js';

interface SpatialActionResponse {
  target_object_id: string;
  translation: [number, number, number];
  rotation_quat: [number, number, number, number];
  instruction_text: string;
}

export class WebXRSpatialAgent {
  private renderer: THREE.WebGLRenderer;
  private scene: THREE.Scene;
  private camera: THREE.PerspectiveCamera;
  private socket: WebSocket | null = null;
  private targetMesh: THREE.Mesh;

  constructor() {
    this.scene = new THREE.Scene();
    this.camera = new THREE.PerspectiveCamera(70, window.innerWidth / window.innerHeight, 0.01, 20);
    this.renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
    this.renderer.xr.enabled = true;

    // Create 3D Holographic Spatial Marker
    const geometry = new THREE.BoxGeometry(0.1, 0.1, 0.1);
    const material = new THREE.MeshStandardMaterial({ color: 0x0f766e, metalness: 0.6, roughness: 0.2 });
    this.targetMesh = new THREE.Mesh(geometry, material);
    this.scene.add(this.targetMesh);
  }

  public async initialize(): Promise<void> {
    // 1. Initialize Rust WASM SIMD Math Engine
    await initWasm();
    console.log('[WebXR] Rust WASM Math Module Initialized.');

    // 2. Connect to Vertex AI Streaming Gateway
    this.socket = new WebSocket('wss://spatial-gateway.ok.st/v1/stream');
    this.socket.onmessage = (event) => {
      const data: SpatialActionResponse = JSON.parse(event.data);
      this.applySpatialTransform(data);
    };

    // 3. Request Immersive AR WebXR Session
    if ('xr' in navigator) {
      const session = await (navigator as any).xr.requestSession('immersive-ar', {
        requiredFeatures: ['local-floor', 'hit-test'],
        optionalFeatures: ['depth-sensing', 'hand-tracking']
      });
      await this.renderer.xr.setSession(session);
      this.renderer.setAnimationLoop(this.render.bind(this));
    }
  }

  private applySpatialTransform(action: SpatialActionResponse): void {
    // Apply smooth interpolation to avoid abrupt visual shifts
    const [x, y, z] = action.translation;
    const [qx, qy, qz, qw] = action.rotation_quat;

    // Call Rust WASM for SIMD Coordinate Bounds Verification
    const isValid = compute_se3_transform(x, y, z, qx, qy, qz, qw);
    if (isValid) {
      this.targetMesh.position.set(x, y, z);
      this.targetMesh.quaternion.set(qx, qy, qz, qw);
    }
  }

  private render(timestamp: number, frame?: any): void {
    // Real-time Motion-to-Photon rendering loop (60Hz / 90Hz)
    this.renderer.render(this.scene, this.camera);
  }
}

import os
import json
import asyncio
from typing import List
from pydantic import BaseModel, Field
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from google import genai
from google.genai import types

# 1. Pydantic Strict Schema for Controlled Generation (CFG Enforcement)
class SpatialTransform(BaseModel):
    translation_x: float = Field(..., ge=-10.0, le=10.0, description="X coordinate in meters")
    translation_y: float = Field(..., ge=-2.0, le=5.0, description="Y height in meters")
    translation_z: float = Field(..., ge=-10.0, le=10.0, description="Z depth in meters")
    rotation_qx: float = Field(..., ge=-1.0, le=1.0)
    rotation_qy: float = Field(..., ge=-1.0, le=1.0)
    rotation_qz: float = Field(..., ge=-1.0, le=1.0)
    rotation_qw: float = Field(..., ge=-1.0, le=1.0)

class SpatialActionDirective(BaseModel):
    target_object_id: string = Field(..., description="Unique ID of the physical entity")
    transform: SpatialTransform
    interaction_mode: str = Field(..., description="gesture | inspection | warning")
    spatial_speech_text: str = Field(..., description="Spoken instruction for user")
    confidence_score: float = Field(..., ge=0.0, le=1.0)

app = FastAPI(title="Open Knowledge Vertex AI Spatial Gateway")

# 2. Initialize Official Google GenAI SDK for Vertex AI
client = genai.Client(
    vertexai=True,
    project="ok-sys-spatial-prod",
    location="asia-northeast1"
)

# 3. Create or Retrieve Context Cache for Static 3D Facility Map
cached_facility_map = client.cached_contents.create(
    model="gemini-2.5-flash",
    config=types.CreateCachedContentConfig(
        contents=[
            types.Content(
                role="user",
                parts=[
                    types.Part.from_text(
                        "You are an enterprise XR Spatial Copilot. "
                        "Facility Cadastral Matrix: Room A-101 has floor origin (0,0,0). "
                        "High voltage panel is located at (1.5, 1.2, -3.0). Valve B3 is at (0.4, 1.1, -1.8)."
                    )
                ]
            )
        ],
        display_name="facility_room_a101_v3",
        ttl="86400s"
    )
)

@app.websocket("/v1/stream")
async def websocket_spatial_endpoint(websocket: WebSocket):
    await websocket.accept()
    try:
        while True:
            # Receive real-time multimodal sensor packet from headset
            raw_data = await websocket.receive_text()
            sensor_packet = json.loads(raw_data)

            # Generate Controlled Action with CFG JSON schema
            response_stream = client.models.generate_content_stream(
                model="gemini-2.5-flash",
                contents=[
                    types.Content(
                        role="user",
                        parts=[
                            types.Part.from_text(
                                f"User gaze vector: {sensor_packet.get('gaze')}. "
                                f"Detected voice prompt: '{sensor_packet.get('speech')}'. "
                                "Compute safe 3D guidance coordinate and voice response."
                            )
                        ]
                    )
                ],
                config=types.GenerateContentConfig(
                    cached_content=cached_facility_map.name,
                    response_mime_type="application/json",
                    response_schema=SpatialActionDirective,
                    temperature=0.1
                )
            )

            # Stream chunks directly back to the spatial frontend
            for chunk in response_stream:
                if chunk.text:
                    await websocket.send_text(chunk.text)

    except WebSocketDisconnect:
        print("[Gateway] XR client disconnected normally.")