Chapter 10: Sensor Simulation in Unity

Overview

This chapter explores sensor simulation in Unity for robotics applications. You'll learn how to implement LIDAR, IMU, and camera sensors in Unity environments, providing realistic sensor data for robot perception testing and training.

Learning Objectives

By the end of this chapter, you will be able to:

Implement LIDAR sensor simulation using raycasting
Create IMU sensors with realistic noise models
Generate procedural physics-based environments
Optimize sensor simulation for real-time performance

Unity Sensor Simulation

Unity can simulate various sensor types used in robotics, providing realistic sensor data for testing and training. The main sensor types include cameras, LIDAR, and IMU sensors.

Camera Simulation

Unity's camera components can simulate various camera types:

RGB cameras for visual perception
Depth cameras for 3D reconstruction
Semantic segmentation cameras for object recognition
Stereo cameras for depth perception

LIDAR Simulation

LIDAR sensors can be simulated using raycasting techniques, providing accurate distance measurements and point cloud data similar to real LIDAR sensors.

IMU Simulation

Inertial measurement units can be simulated by tracking the acceleration and rotation of objects in the Unity scene, providing realistic IMU data for navigation algorithms.

Code Examples

Unity C# Script for LIDAR Simulation

using System.Collections.Generic;
using UnityEngine;

public class LIDARSensor : MonoBehaviour
{
    [Header("LIDAR Configuration")]
    public int numberOfRays = 360;
    public float minAngle = -90f;
    public float maxAngle = 90f;
    public float maxDistance = 20f;
    public LayerMask detectionMask = -1;
    public string outputTopic = "/laser_scan";

    [Header("Performance")]
    public float updateRate = 10f;
    public bool visualizeRays = true;

    private LineRenderer lineRenderer;
    private float updateInterval;
    private float lastUpdateTime;

    [System.Serializable]
    public class LaserScanData
    {
        public float[] ranges;
        public float angle_min;
        public float angle_max;
        public float angle_increment;
        public float time_increment;
        public float scan_time;
        public float range_min;
        public float range_max;
    }

    void Start()
    {
        updateInterval = 1f / updateRate;
        lastUpdateTime = 0f;

        if (visualizeRays)
        {
            SetupLineRenderer();
        }
    }

    void SetupLineRenderer()
    {
        lineRenderer = gameObject.AddComponent<LineRenderer>();
        lineRenderer.material = new Material(Shader.Find("Sprites/Default"));
        lineRenderer.widthMultiplier = 0.02f;
        lineRenderer.positionCount = numberOfRays + 1;
        lineRenderer.useWorldSpace = false;
    }

    void Update()
    {
        if (Time.time - lastUpdateTime >= updateInterval)
        {
            PerformLIDARScan();
            lastUpdateTime = Time.time;
        }
    }

    public LaserScanData PerformLIDARScan()
    {
        LaserScanData scanData = new LaserScanData();
        scanData.ranges = new float[numberOfRays];
        scanData.angle_min = minAngle * Mathf.Deg2Rad;
        scanData.angle_max = maxAngle * Mathf.Deg2Rad;
        scanData.angle_increment = (maxAngle - minAngle) * Mathf.Deg2Rad / numberOfRays;
        scanData.time_increment = 0f;
        scanData.scan_time = updateInterval;
        scanData.range_min = 0.1f;
        scanData.range_max = maxDistance;

        float angleStep = (maxAngle - minAngle) / numberOfRays;

        for (int i = 0; i < numberOfRays; i++)
        {
            float angle = minAngle + i * angleStep;
            float radians = angle * Mathf.Deg2Rad;

            Vector3 rayDirection = new Vector3(
                Mathf.Cos(radians),
                0f,
                Mathf.Sin(radians)
            );

            Vector3 worldRayDirection = transform.TransformDirection(rayDirection);

            RaycastHit hit;
            if (Physics.Raycast(transform.position, worldRayDirection, out hit, maxDistance, detectionMask))
            {
                scanData.ranges[i] = hit.distance;

                if (visualizeRays && lineRenderer != null)
                {
                    lineRenderer.SetPosition(i, Vector3.zero);
                    lineRenderer.SetPosition(i + 1, transform.InverseTransformPoint(hit.point));
                }
            }
            else
            {
                scanData.ranges[i] = float.PositiveInfinity;

                if (visualizeRays && lineRenderer != null)
                {
                    Vector3 maxPoint = transform.InverseTransformPoint(
                        transform.position + worldRayDirection * maxDistance
                    );
                    lineRenderer.SetPosition(i, Vector3.zero);
                    lineRenderer.SetPosition(i + 1, maxPoint);
                }
            }
        }

        return scanData;
    }

    public List<Vector3> GetPointCloud(LaserScanData scanData)
    {
        List<Vector3> pointCloud = new List<Vector3>();
        Vector3 sensorPosition = transform.position;

        for (int i = 0; i < scanData.ranges.Length; i++)
        {
            float range = scanData.ranges[i];
            if (range > scanData.range_min && range < scanData.range_max)
            {
                float angle = scanData.angle_min + i * scanData.angle_increment;

                Vector3 point = new Vector3(
                    range * Mathf.Cos(angle),
                    0f,
                    range * Mathf.Sin(angle)
                );

                point = transform.TransformPoint(point);
                pointCloud.Add(point);
            }
        }

        return pointCloud;
    }

    void OnValidate()
    {
        numberOfRays = Mathf.Clamp(numberOfRays, 1, 10000);
        maxDistance = Mathf.Clamp(maxDistance, 0.1f, 1000f);
        updateRate = Mathf.Clamp(updateRate, 0.1f, 100f);
    }
}

Unity C# Script for IMU Simulation

using System.Collections;
using UnityEngine;

public class IMUSensor : MonoBehaviour
{
    [Header("IMU Configuration")]
    public float updateRate = 100f;
    public bool includeGyroscope = true;
    public bool includeAccelerometer = true;
    public bool includeMagnetometer = false;

    [Header("Noise Parameters")]
    public float accelerometerNoise = 0.01f;
    public float gyroscopeNoise = 0.01f;
    public float magnetometerNoise = 0.1f;

    [Header("Gravity")]
    public Vector3 gravity = new Vector3(0, -9.81f, 0);

    private float updateInterval;
    private float lastUpdateTime;

    [System.Serializable]
    public class IMUData
    {
        public Vector3 linear_acceleration;
        public Vector3 angular_velocity;
        public Vector3 magnetic_field;
        public Quaternion orientation;
    }

    private Vector3 previousPosition;
    private Quaternion previousRotation;
    private float previousTime;

    void Start()
    {
        updateInterval = 1f / updateRate;
        lastUpdateTime = Time.time;

        previousPosition = transform.position;
        previousRotation = transform.rotation;
        previousTime = Time.time;
    }

    void Update()
    {
        if (Time.time - lastUpdateTime >= updateInterval)
        {
            PublishIMUData();
            lastUpdateTime = Time.time;
        }
    }

    public IMUData GetIMUData()
    {
        IMUData imuData = new IMUData();

        if (includeAccelerometer)
        {
            Vector3 currentPosition = transform.position;
            float currentTime = Time.time;

            Vector3 velocity = (currentPosition - previousPosition) / (currentTime - previousTime);
            Vector3 previousVelocity = velocity;

            Vector3 acceleration = (velocity - previousVelocity) / (currentTime - previousTime);

            imuData.linear_acceleration = acceleration - transform.InverseTransformDirection(gravity);

            imuData.linear_acceleration += AddNoiseVector(accelerometerNoise);
        }

        if (includeGyroscope)
        {
            Quaternion currentRotation = transform.rotation;
            float currentTime = Time.time;

            Quaternion deltaRotation = currentRotation * Quaternion.Inverse(previousRotation);
            Vector3 angularVelocity = new Vector3();

            float deltaTime = currentTime - previousTime;
            if (deltaTime > 0)
            {
                float angle;
                Vector3 axis;
                deltaRotation.ToAngleAxis(out angle, out axis);

                angularVelocity = axis * Mathf.Deg2Rad * angle / deltaTime;
            }

            imuData.angular_velocity = transform.InverseTransformDirection(angularVelocity);

            imuData.angular_velocity += AddNoiseVector(gyroscopeNoise);
        }

        if (includeMagnetometer)
        {
            imuData.magnetic_field = transform.InverseTransformDirection(new Vector3(22.9f, 0f, 44.4f));

            imuData.magnetic_field += AddNoiseVector(magnetometerNoise);
        }

        imuData.orientation = transform.rotation;

        previousPosition = transform.position;
        previousRotation = transform.rotation;
        previousTime = Time.time;

        return imuData;
    }

    private Vector3 AddNoiseVector(float noiseLevel)
    {
        return new Vector3(
            Random.Range(-noiseLevel, noiseLevel),
            Random.Range(-noiseLevel, noiseLevel),
            Random.Range(-noiseLevel, noiseLevel)
        );
    }

    public void PublishIMUData()
    {
        IMUData data = GetIMUData();

        Debug.Log($"IMU Data - Accel: {data.linear_acceleration}, Gyro: {data.angular_velocity}");
    }

    public Vector3 GetWorldAcceleration()
    {
        IMUData data = GetIMUData();
        return transform.TransformDirection(data.linear_acceleration);
    }

    public Vector3 GetWorldAngularVelocity()
    {
        IMUData data = GetIMUData();
        return transform.TransformDirection(data.angular_velocity);
    }
}

Unity C# Script for Physics-Based Environment Generation

using System.Collections.Generic;
using UnityEngine;

public class PhysicsEnvironmentGenerator : MonoBehaviour
{
    [Header("Terrain Generation")]
    public int terrainWidth = 200;
    public int terrainLength = 200;
    public float terrainHeight = 20f;
    public int resolution = 256;

    [Header("Object Placement")]
    public GameObject[] obstaclePrefabs;
    public int minObstacles = 10;
    public int maxObstacles = 50;
    public float placementAreaPadding = 5f;

    [Header("Material Properties")]
    public PhysicMaterial defaultMaterial;
    public float dynamicFriction = 0.6f;
    public float staticFriction = 0.6f;
    public float bounciness = 0.1f;

    [Header("Environment Features")]
    public bool generateRandomTerrain = true;
    public bool addStaticObstacles = true;
    public bool addDynamicObstacles = true;

    private List<GameObject> spawnedObjects = new List<GameObject>();
    private float[,] heightMap;

    void Start()
    {
        GenerateEnvironment();
    }

    public void GenerateEnvironment()
    {
        ClearEnvironment();

        if (generateRandomTerrain)
        {
            GenerateTerrain();
        }

        if (addStaticObstacles)
        {
            PlaceStaticObstacles();
        }

        if (addDynamicObstacles)
        {
            PlaceDynamicObstacles();
        }
    }

    void ClearEnvironment()
    {
        foreach (GameObject obj in spawnedObjects)
        {
            if (obj != null)
            {
                DestroyImmediate(obj);
            }
        }
        spawnedObjects.Clear();
    }

    void GenerateTerrain()
    {
        GameObject terrainObj = new GameObject("GeneratedTerrain");
        Terrain terrain = terrainObj.AddComponent<Terrain>();
        TerrainCollider terrainCollider = terrainObj.AddComponent<TerrainCollider>();

        TerrainData terrainData = new TerrainData();
        terrainData.heightmapResolution = resolution;
        terrainData.size = new Vector3(terrainWidth, terrainHeight, terrainLength);

        heightMap = GenerateHeightMap(resolution, resolution);
        terrainData.SetHeights(0, 0, heightMap);

        terrain.terrainData = terrainData;
        terrainCollider.terrainData = terrainData;

        spawnedObjects.Add(terrainObj);
    }

    float[,] GenerateHeightMap(int width, int height)
    {
        float[,] heights = new float[width, height];

        for (int x = 0; x < width; x++)
        {
            for (int y = 0; y < height; y++)
            {
                float xCoord = (float)x / width * 10f;
                float yCoord = (float)y / height * 10f;

                float elevation = 0f;
                float amplitude = 1f;
                float frequency = 1f;
                float persistence = 0.5f;

                for (int octave = 0; octave < 4; octave++)
                {
                    elevation += Mathf.PerlinNoise(xCoord * frequency, yCoord * frequency) * amplitude;
                    amplitude *= persistence;
                    frequency *= 2f;
                }

                heights[x, y] = elevation / 4f;
            }
        }

        return heights;
    }

    void PlaceStaticObstacles()
    {
        int numObstacles = Random.Range(minObstacles, maxObstacles + 1);

        for (int i = 0; i < numObstacles; i++)
        {
            if (obstaclePrefabs.Length == 0) continue;

            GameObject obstaclePrefab = obstaclePrefabs[Random.Range(0, obstaclePrefabs.Length)];

            Vector3 position = new Vector3(
                Random.Range(placementAreaPadding, terrainWidth - placementAreaPadding),
                10f,
                Random.Range(placementAreaPadding, terrainLength - placementAreaPadding)
            );

            GameObject obstacle = Instantiate(obstaclePrefab, position, Quaternion.identity);

            if (obstacle.GetComponent<Rigidbody>() == null)
            {
                Rigidbody rb = obstacle.AddComponent<Rigidbody>();
                rb.isKinematic = true;
            }

            SetPhysicsMaterial(obstacle);

            spawnedObjects.Add(obstacle);
        }
    }

    void PlaceDynamicObstacles()
    {
        int numObstacles = Random.Range(minObstacles / 2, maxObstacles / 2 + 1);

        for (int i = 0; i < numObstacles; i++)
        {
            if (obstaclePrefabs.Length == 0) continue;

            GameObject obstaclePrefab = obstaclePrefabs[Random.Range(0, obstaclePrefabs.Length)];

            Vector3 position = new Vector3(
                Random.Range(placementAreaPadding, terrainWidth - placementAreaPadding),
                10f,
                Random.Range(placementAreaPadding, terrainLength - placementAreaPadding)
            );

            GameObject obstacle = Instantiate(obstaclePrefab, position, Quaternion.identity);

            Rigidbody rb = obstacle.GetComponent<Rigidbody>();
            if (rb == null)
            {
                rb = obstacle.AddComponent<Rigidbody>();
            }

            rb.isKinematic = false;
            rb.useGravity = true;

            rb.velocity = new Vector3(
                Random.Range(-1f, 1f),
                0f,
                Random.Range(-1f, 1f)
            ) * 2f;

            SetPhysicsMaterial(obstacle);

            spawnedObjects.Add(obstacle);
        }
    }

    void SetPhysicsMaterial(GameObject obj)
    {
        PhysicMaterial material = defaultMaterial;
        if (material == null)
        {
            material = new PhysicMaterial("GeneratedMaterial");
            material.dynamicFriction = dynamicFriction;
            material.staticFriction = staticFriction;
            material.bounciness = bounciness;
        }

        Collider[] colliders = obj.GetComponentsInChildren<Collider>();
        foreach (Collider collider in colliders)
        {
            collider.material = material;
        }
    }
}

Summary

Unity provides comprehensive sensor simulation capabilities for robotics applications. LIDAR sensors use raycasting for distance measurements, IMU sensors track motion and orientation, and camera sensors capture visual data. These simulated sensors enable effective robot perception testing in photorealistic environments.

Key Takeaways

LIDAR simulation uses raycasting to generate realistic point cloud data
IMU sensors track acceleration and rotation with configurable noise models
Procedural environment generation creates diverse testing scenarios
Performance optimization ensures real-time sensor simulation

What's Next

In the next chapter, we'll explore NVIDIA Isaac Sim, learning how to leverage GPU-accelerated simulation for advanced robotics development and training.

Overview​

Learning Objectives​

Unity Sensor Simulation​

Camera Simulation​

LIDAR Simulation​

IMU Simulation​

Code Examples​

Unity C# Script for LIDAR Simulation​

Unity C# Script for IMU Simulation​

Unity C# Script for Physics-Based Environment Generation​

Summary​

Key Takeaways​

What's Next​