From 63d5b314c69c5c87bf92e4ac34389befd0caaf49 Mon Sep 17 00:00:00 2001 From: Axel Date: Thu, 4 Nov 2021 15:02:43 +0100 Subject: [PATCH] Added instance semantic segmentation sensors --- LibCarla/source/carla/sensor/SensorRegistry.h | 3 + PythonAPI/examples/manual_control.py | 5 +- .../Source/Carla/Game/TaggedComponent.cpp | 309 ++++++++++++++++++ .../Carla/Source/Carla/Game/TaggedComponent.h | 87 +++++ .../Carla/Source/Carla/Game/Tagger.cpp | 102 ++++++ .../Plugins/Carla/Source/Carla/Game/Tagger.h | 4 + .../Sensor/InstanceSegmentationCamera.cpp | 98 ++++++ .../Carla/Sensor/InstanceSegmentationCamera.h | 31 ++ 8 files changed, 637 insertions(+), 2 deletions(-) create mode 100644 Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/TaggedComponent.cpp create mode 100644 Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/TaggedComponent.h create mode 100644 Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.cpp create mode 100644 Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.h diff --git a/LibCarla/source/carla/sensor/SensorRegistry.h b/LibCarla/source/carla/sensor/SensorRegistry.h index a0443ba7e..507da2280 100644 --- a/LibCarla/source/carla/sensor/SensorRegistry.h +++ b/LibCarla/source/carla/sensor/SensorRegistry.h @@ -41,6 +41,7 @@ class ARayCastSemanticLidar; class ARayCastLidar; class ASceneCaptureCamera; class ASemanticSegmentationCamera; +class AInstanceSegmentationCamera; class ARssSensor; class FWorldObserver; @@ -69,6 +70,7 @@ namespace sensor { std::pair, std::pair, std::pair, + std::pair, std::pair >; @@ -94,6 +96,7 @@ namespace sensor { #include "Carla/Sensor/RssSensor.h" #include "Carla/Sensor/SceneCaptureCamera.h" #include "Carla/Sensor/SemanticSegmentationCamera.h" +#include "Carla/Sensor/InstanceSegmentationCamera.h" #include "Carla/Sensor/WorldObserver.h" #endif // LIBCARLA_SENSOR_REGISTRY_WITH_SENSOR_INCLUDES diff --git a/PythonAPI/examples/manual_control.py b/PythonAPI/examples/manual_control.py index 657c06bbd..19bc8aaab 100755 --- a/PythonAPI/examples/manual_control.py +++ b/PythonAPI/examples/manual_control.py @@ -1058,8 +1058,9 @@ class CameraManager(object): ['sensor.camera.depth', cc.Depth, 'Camera Depth (Gray Scale)', {}], ['sensor.camera.depth', cc.LogarithmicDepth, 'Camera Depth (Logarithmic Gray Scale)', {}], ['sensor.camera.semantic_segmentation', cc.Raw, 'Camera Semantic Segmentation (Raw)', {}], - ['sensor.camera.semantic_segmentation', cc.CityScapesPalette, - 'Camera Semantic Segmentation (CityScapes Palette)', {}], + ['sensor.camera.semantic_segmentation', cc.CityScapesPalette, 'Camera Semantic Segmentation (CityScapes Palette)', {}], + ['sensor.camera.instance_segmentation', cc.CityScapesPalette, 'Camera Instance Segmentation (CityScapes Palette)', {}], + ['sensor.camera.instance_segmentation', cc.Raw, 'Camera Instance Segmentation (Raw)', {}], ['sensor.lidar.ray_cast', None, 'Lidar (Ray-Cast)', {'range': '50'}], ['sensor.camera.dvs', cc.Raw, 'Dynamic Vision Sensor', {}], ['sensor.camera.rgb', cc.Raw, 'Camera RGB Distorted', diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/TaggedComponent.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/TaggedComponent.cpp new file mode 100644 index 000000000..c6d2b0306 --- /dev/null +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/TaggedComponent.cpp @@ -0,0 +1,309 @@ +#include "Carla.h" +#include "TaggedComponent.h" + +#include "Rendering/SkeletalMeshRenderData.h" + +// +// UTaggedComponent +// +UTaggedComponent::UTaggedComponent(const FObjectInitializer& ObjectInitializer) : + UPrimitiveComponent(ObjectInitializer), + Color(1, 1, 1, 1) +{ + FString MaterialPath = TEXT("Material'/Carla/PostProcessingMaterials/AnnotationColor.AnnotationColor'"); + static ConstructorHelpers::FObjectFinder TaggedMaterialObject(*MaterialPath); + // TODO: Replace with VertexColorViewModeMaterial_ColorOnly? + + TaggedMaterial = TaggedMaterialObject.Object; + PrimaryComponentTick.bCanEverTick = true; +} + +void UTaggedComponent::OnRegister() +{ + Super::OnRegister(); + + TaggedMID = UMaterialInstanceDynamic::Create(TaggedMaterial, this, TEXT("TaggedMaterialMID")); + + if (!IsValid(TaggedMID)) + { + UE_LOG(LogCarla, Error, TEXT("Failed to create MID!")); + } + + SetColor(Color); +} + +void UTaggedComponent::SetColor(FLinearColor NewColor) +{ + Color = NewColor; + + if (IsValid(TaggedMID)) + { + TaggedMID->SetVectorParameterValue("AnnotationColor", Color); + } +} + +FLinearColor UTaggedComponent::GetColor() +{ + return Color; +} + +FBoxSphereBounds UTaggedComponent::CalcBounds(const FTransform & LocalToWorld) const +{ + USceneComponent * ParentComponent = GetAttachParent(); + if(ParentComponent) + { + return ParentComponent->CalcBounds(LocalToWorld); + } + return FBoxSphereBounds(); +} + +FPrimitiveSceneProxy * UTaggedComponent::CreateSceneProxy() +{ + USceneComponent * ParentComponent = GetAttachParent(); + + if (!IsValid(ParentComponent)) + { + UE_LOG(LogCarla, Error, TEXT("Invalid parent component")); + return NULL; + } + + USkeletalMeshComponent * SkeletalMeshComponent = Cast(ParentComponent); + UStaticMeshComponent * StaticMeshComponent = Cast(ParentComponent); + UHierarchicalInstancedStaticMeshComponent * HierarchicalInstancedStaticMeshComponent = + Cast(ParentComponent); + UInstancedStaticMeshComponent* InstancedStaticMeshComponent = Cast(ParentComponent); + if (HierarchicalInstancedStaticMeshComponent) + { + return CreateSceneProxy(HierarchicalInstancedStaticMeshComponent); + } + else if (InstancedStaticMeshComponent) + { + return CreateSceneProxy(InstancedStaticMeshComponent); + } + else + if (IsValid(StaticMeshComponent)) + { + return CreateSceneProxy(StaticMeshComponent); + } + else if (IsValid(SkeletalMeshComponent)) + { + bSkeletalMesh = true; + return CreateSceneProxy(SkeletalMeshComponent); + } + + UE_LOG(LogCarla, Error, TEXT("Unknown type of parent component: %s"), *ParentComponent->GetClass()->GetName()); + + return NULL; +} + +FPrimitiveSceneProxy * UTaggedComponent::CreateSceneProxy(UStaticMeshComponent * StaticMeshComponent) +{ + // Make sure static mesh has render data + UStaticMesh * StaticMesh = StaticMeshComponent->GetStaticMesh(); + + if (StaticMesh == NULL) + { + UE_LOG(LogCarla, Error, TEXT("Failed to create scene proxy for static mesh component (because static mesh is null): %s"), *StaticMeshComponent->GetReadableName()); + return NULL; + } + + if (StaticMesh->RenderData == NULL) + { + UE_LOG(LogCarla, Error, TEXT("Failed to create scene proxy for static mesh component (because render data is null): %s"), *StaticMeshComponent->GetReadableName()); + return NULL; + } + + + if (StaticMesh->RenderData->LODResources.Num() == 0) + { + UE_LOG(LogCarla, Error, TEXT("Failed to create scene proxy for static mesh component (because num LOD resources is 0): %s"), *StaticMeshComponent->GetReadableName()); + return NULL; + } + + return new FTaggedStaticMeshSceneProxy(StaticMeshComponent, true, TaggedMID); +} + +FPrimitiveSceneProxy * UTaggedComponent::CreateSceneProxy(USkeletalMeshComponent * SkeletalMeshComponent) +{ + + FSkeletalMeshRenderData * SkeletalMeshRenderData = SkeletalMeshComponent->GetSkeletalMeshRenderData(); + + // Only create a scene proxy for rendering if properly initialized + if (SkeletalMeshRenderData == NULL) + { + UE_LOG(LogCarla, Error, TEXT("Failed to create scene proxy for skeletal mesh component: %s"), *SkeletalMeshComponent->GetReadableName()); + return NULL; + } + + if (SkeletalMeshRenderData->LODRenderData.IsValidIndex(SkeletalMeshComponent->PredictedLODLevel) == false) + { + UE_LOG(LogCarla, Error, TEXT("Failed to create scene proxy for skeletal mesh component: %s"), *SkeletalMeshComponent->GetReadableName()); + return NULL; + } + + if (SkeletalMeshComponent->MeshObject == NULL) + { + UE_LOG(LogCarla, Error, TEXT("Failed to create scene proxy for skeletal mesh component: %s"), *SkeletalMeshComponent->GetReadableName()); + return NULL; + } + + // FIXME: How does this allocation get freed? + return new FTaggedSkeletalMeshSceneProxy(SkeletalMeshComponent, SkeletalMeshRenderData, TaggedMID); +} + +FPrimitiveSceneProxy * UTaggedComponent::CreateSceneProxy(UHierarchicalInstancedStaticMeshComponent * MeshComponent) +{ + // Verify that the mesh is valid before using it. + const bool bMeshIsValid = + // make sure we have instances + (MeshComponent->PerInstanceRenderData.IsValid()) && + // make sure we have an actual staticmesh + MeshComponent->GetStaticMesh() && + MeshComponent->GetStaticMesh()->HasValidRenderData(false) && + // You really can't use hardware instancing on the consoles with multiple elements because they share the same index buffer. + // @todo: Level error or something to let LDs know this + 1;//GetStaticMesh()->LODModels(0).Elements.Num() == 1; + + if (bMeshIsValid) + { + bool bIsGrass = !MeshComponent->PerInstanceSMData.Num(); + return new FTaggedHierarchicalStaticMeshSceneProxy(MeshComponent, bIsGrass, GetWorld()->FeatureLevel, TaggedMID); + } + return nullptr; +} + +FPrimitiveSceneProxy * UTaggedComponent::CreateSceneProxy(UInstancedStaticMeshComponent * MeshComponent) +{ + // Verify that the mesh is valid before using it. + const bool bMeshIsValid = + // make sure we have instances + (MeshComponent->PerInstanceRenderData.IsValid()) && + // make sure we have an actual staticmesh + MeshComponent->GetStaticMesh() && + MeshComponent->GetStaticMesh()->HasValidRenderData(false) && + // You really can't use hardware instancing on the consoles with multiple elements because they share the same index buffer. + // @todo: Level error or something to let LDs know this + 1;//GetStaticMesh()->LODModels(0).Elements.Num() == 1; + + if (bMeshIsValid) + { + return new FTaggedInstancedStaticMeshSceneProxy(MeshComponent, GetWorld()->FeatureLevel, TaggedMID); + } + return nullptr; +} + +void UTaggedComponent::TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction * ThisTickFunction) +{ + Super::TickComponent(DeltaTime, TickType, ThisTickFunction); + + // // TODO: Try removing this + if (bSkeletalMesh) + { + MarkRenderStateDirty(); + } +} + +// +// FTaggedStaticMeshSceneProxy +// +FTaggedStaticMeshSceneProxy::FTaggedStaticMeshSceneProxy(UStaticMeshComponent * Component, bool bForceLODsShareStaticLighting, UMaterialInstance * MaterialInstance) : + FStaticMeshSceneProxy(Component, bForceLODsShareStaticLighting) +{ + TaggedMaterialInstance = MaterialInstance; + + // Replace materials with tagged material + bVerifyUsedMaterials = false; + + for (FLODInfo& LODInfo : LODs) { + for (FLODInfo::FSectionInfo& SectionInfo : LODInfo.Sections) { + SectionInfo.Material = TaggedMaterialInstance; + } + } +} + +FPrimitiveViewRelevance FTaggedStaticMeshSceneProxy::GetViewRelevance(const FSceneView * View) const +{ + FPrimitiveViewRelevance ViewRelevance = FStaticMeshSceneProxy::GetViewRelevance(View); + + ViewRelevance.bDrawRelevance = !View->Family->EngineShowFlags.NotDrawTaggedComponents; + + return ViewRelevance; +} + +// +// FTaggedSkeletalMeshSceneProxy +// +FTaggedSkeletalMeshSceneProxy::FTaggedSkeletalMeshSceneProxy(const USkinnedMeshComponent * Component, FSkeletalMeshRenderData * InSkeletalMeshRenderData, UMaterialInstance * MaterialInstance) : + FSkeletalMeshSceneProxy(Component, InSkeletalMeshRenderData) +{ + TaggedMaterialInstance = MaterialInstance; + + // Replace materials with tagged material + bVerifyUsedMaterials = false; + + for (FLODSectionElements& LODSection : LODSections) { + for (FSectionElementInfo& ElementInfo : LODSection.SectionElements) { + ElementInfo.Material = TaggedMaterialInstance; + } + } +} + +FPrimitiveViewRelevance FTaggedSkeletalMeshSceneProxy::GetViewRelevance(const FSceneView * View) const +{ + FPrimitiveViewRelevance ViewRelevance = FSkeletalMeshSceneProxy::GetViewRelevance(View); + + ViewRelevance.bDrawRelevance = !View->Family->EngineShowFlags.NotDrawTaggedComponents; + + return ViewRelevance; +} + +FTaggedInstancedStaticMeshSceneProxy::FTaggedInstancedStaticMeshSceneProxy( + UInstancedStaticMeshComponent * Component, ERHIFeatureLevel::Type InFeatureLevel, UMaterialInstance * MaterialInstance) + : FInstancedStaticMeshSceneProxy(Component, InFeatureLevel) +{ + TaggedMaterialInstance = MaterialInstance; + + // Replace materials with tagged material + bVerifyUsedMaterials = false; + + for (FLODInfo& LODInfo : LODs) { + for (FLODInfo::FSectionInfo& SectionInfo : LODInfo.Sections) { + SectionInfo.Material = TaggedMaterialInstance; + } + } +} + +FPrimitiveViewRelevance FTaggedInstancedStaticMeshSceneProxy::GetViewRelevance(const FSceneView * View) const +{ + FPrimitiveViewRelevance ViewRelevance = FInstancedStaticMeshSceneProxy::GetViewRelevance(View); + + ViewRelevance.bDrawRelevance = !View->Family->EngineShowFlags.NotDrawTaggedComponents; + + return ViewRelevance; +} + + +FTaggedHierarchicalStaticMeshSceneProxy::FTaggedHierarchicalStaticMeshSceneProxy( + UHierarchicalInstancedStaticMeshComponent * Component, bool bInIsGrass, ERHIFeatureLevel::Type InFeatureLevel, UMaterialInstance * MaterialInstance) + : FHierarchicalStaticMeshSceneProxy(bInIsGrass, Component, InFeatureLevel) +{ + TaggedMaterialInstance = MaterialInstance; + + // Replace materials with tagged material + bVerifyUsedMaterials = false; + + for (FLODInfo& LODInfo : LODs) { + for (FLODInfo::FSectionInfo& SectionInfo : LODInfo.Sections) { + SectionInfo.Material = TaggedMaterialInstance; + } + } +} + +FPrimitiveViewRelevance FTaggedHierarchicalStaticMeshSceneProxy::GetViewRelevance(const FSceneView * View) const +{ + FPrimitiveViewRelevance ViewRelevance = FHierarchicalStaticMeshSceneProxy::GetViewRelevance(View); + + ViewRelevance.bDrawRelevance = !View->Family->EngineShowFlags.NotDrawTaggedComponents; + + return ViewRelevance; +} diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/TaggedComponent.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/TaggedComponent.h new file mode 100644 index 000000000..92dbaada0 --- /dev/null +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/TaggedComponent.h @@ -0,0 +1,87 @@ +#pragma once + +#include "CoreMinimal.h" +#include "Materials/MaterialInstanceDynamic.h" +#include "Engine/InstancedStaticMesh.h" +#include "Components/HierarchicalInstancedStaticMeshComponent.h" + +#include "TaggedComponent.generated.h" + +UCLASS( meta=(BlueprintSpawnableComponent) ) +class CARLA_API UTaggedComponent : public UPrimitiveComponent +{ + GENERATED_BODY() + +public: + UTaggedComponent(const FObjectInitializer& ObjectInitializer); + + virtual FPrimitiveSceneProxy * CreateSceneProxy() override; + virtual void TickComponent(float DeltaTime, enum ELevelTick TickType, FActorComponentTickFunction * ThisTickFunction) override; + virtual void OnRegister() override; + virtual FBoxSphereBounds CalcBounds(const FTransform & LocalToWorld) const ; + + // FIXME: Should take an int32 and convert to FVector4 + void SetColor(FLinearColor color); + FLinearColor GetColor(); + +private: + FLinearColor Color; + + UPROPERTY() + UMaterial * TaggedMaterial; + + UPROPERTY() + UMaterialInstanceDynamic * TaggedMID; + + bool bSkeletalMesh = false; + + FPrimitiveSceneProxy * CreateSceneProxy(UStaticMeshComponent * StaticMeshComponent); + FPrimitiveSceneProxy * CreateSceneProxy(USkeletalMeshComponent * SkeletalMeshComponent); + FPrimitiveSceneProxy * CreateSceneProxy(UHierarchicalInstancedStaticMeshComponent * MeshComponent); + FPrimitiveSceneProxy * CreateSceneProxy(UInstancedStaticMeshComponent * MeshComponent); +}; + +class FTaggedStaticMeshSceneProxy : public FStaticMeshSceneProxy +{ +public: + FTaggedStaticMeshSceneProxy(UStaticMeshComponent * Component, bool bForceLODsShareStaticLighting, UMaterialInstance * MaterialInstance); + + virtual FPrimitiveViewRelevance GetViewRelevance(const FSceneView * View) const override; + +private: + UMaterialInstance * TaggedMaterialInstance; +}; + +class FTaggedSkeletalMeshSceneProxy : public FSkeletalMeshSceneProxy +{ +public: + FTaggedSkeletalMeshSceneProxy(const USkinnedMeshComponent * Component, FSkeletalMeshRenderData * InSkeletalMeshRenderData, UMaterialInstance * MaterialInstance); + + virtual FPrimitiveViewRelevance GetViewRelevance(const FSceneView * View) const override; + +private: + UMaterialInstance * TaggedMaterialInstance; +}; + +class FTaggedInstancedStaticMeshSceneProxy : public FInstancedStaticMeshSceneProxy +{ +public: + FTaggedInstancedStaticMeshSceneProxy(UInstancedStaticMeshComponent * Component, ERHIFeatureLevel::Type InFeatureLevel, UMaterialInstance * MaterialInstance); + + virtual FPrimitiveViewRelevance GetViewRelevance(const FSceneView * View) const override; + +private: + UMaterialInstance * TaggedMaterialInstance; +}; + + +class FTaggedHierarchicalStaticMeshSceneProxy : public FHierarchicalStaticMeshSceneProxy +{ +public: + FTaggedHierarchicalStaticMeshSceneProxy(UHierarchicalInstancedStaticMeshComponent * Component, bool bInIsGrass, ERHIFeatureLevel::Type InFeatureLevel, UMaterialInstance * MaterialInstance); + + virtual FPrimitiveViewRelevance GetViewRelevance(const FSceneView * View) const override; + +private: + UMaterialInstance * TaggedMaterialInstance; +}; diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/Tagger.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/Tagger.cpp index 6bda5d19a..838dc33af 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/Tagger.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/Tagger.cpp @@ -6,6 +6,7 @@ #include "Carla.h" #include "Tagger.h" +#include "TaggedComponent.h" #include "Components/SkeletalMeshComponent.h" #include "Components/StaticMeshComponent.h" @@ -58,6 +59,30 @@ void ATagger::SetStencilValue( (Label != crp::CityObjectLabel::None)); } +bool ATagger::IsThing(const crp::CityObjectLabel &Label) +{ + return (Label == crp::CityObjectLabel::Pedestrians || + Label == crp::CityObjectLabel::TrafficSigns || + Label == crp::CityObjectLabel::Vehicles || + Label == crp::CityObjectLabel::TrafficLight); +} + +FLinearColor ATagger::GetActorLabelColor(const AActor &Actor, const crp::CityObjectLabel &Label) +{ + uint32 id = Actor.GetUniqueID(); + // TODO: Warn if id > 0xffff. + + // Encode label and id like semantic segmentation does + // TODO: Steal bits from R channel and maybe A channel? + FLinearColor Color(0.0f, 0.0f, 0.0f, 1.0f); + Color.R = CastEnum(Label) / 255.0f; + Color.G = ((id & 0x00ff) >> 0) / 255.0f; + Color.B = ((id & 0xff00) >> 8) / 255.0f; + + return Color; +} + + // ============================================================================= // -- static ATagger functions ------------------------------------------------- // ============================================================================= @@ -78,6 +103,44 @@ void ATagger::TagActor(const AActor &Actor, bool bTagForSemanticSegmentation) UE_LOG(LogCarla, Log, TEXT(" + StaticMeshComponent: %s"), *Component->GetName()); UE_LOG(LogCarla, Log, TEXT(" - Label: \"%s\""), *GetTagAsString(Label)); #endif // CARLA_TAGGER_EXTRA_LOG + + // Don't instance segment non-things (i.e., stuff) + // if (!IsThing(Label)) { + // continue; + // } + + // Find a tagged component that is attached to this component + UTaggedComponent *TaggedComponent = NULL; + TArray AttachedComponents = Component->GetAttachChildren(); + for (USceneComponent *SceneComponent : AttachedComponents) { + UTaggedComponent *TaggedSceneComponent = Cast(SceneComponent); + if (IsValid(TaggedSceneComponent)) { + TaggedComponent = TaggedSceneComponent; +#ifdef CARLA_TAGGER_EXTRA_LOG + UE_LOG(LogCarla, Log, TEXT(" - Found Tag")); +#endif // CARLA_TAGGER_EXTRA_LOG + break; + } + } + + // If not found, then create new tagged component and attach it to this component + if (!TaggedComponent) { + TaggedComponent = NewObject(Component); + TaggedComponent->SetupAttachment(Component); + TaggedComponent->RegisterComponent(); +#ifdef CARLA_TAGGER_EXTRA_LOG + UE_LOG(LogCarla, Log, TEXT(" - Added Tag")); +#endif // CARLA_TAGGER_EXTRA_LOG + } + + // Set tagged component color + FLinearColor Color = GetActorLabelColor(Actor, Label); +#ifdef CARLA_TAGGER_EXTRA_LOG + UE_LOG(LogCarla, Log, TEXT(" - Color: %s"), *Color.ToString()); +#endif // CARLA_TAGGER_EXTRA_LOG + + TaggedComponent->SetColor(Color); + TaggedComponent->MarkRenderStateDirty(); } // Iterate skeletal meshes. @@ -90,6 +153,45 @@ void ATagger::TagActor(const AActor &Actor, bool bTagForSemanticSegmentation) UE_LOG(LogCarla, Log, TEXT(" + SkeletalMeshComponent: %s"), *Component->GetName()); UE_LOG(LogCarla, Log, TEXT(" - Label: \"%s\""), *GetTagAsString(Label)); #endif // CARLA_TAGGER_EXTRA_LOG + + // Don't instance segment non-things (i.e., stuff) + // if (!IsThing(Label)) { + // continue; + // } + + // Find a tagged component that is attached to this component + UTaggedComponent *TaggedComponent = NULL; + TArray AttachedComponents = Component->GetAttachChildren(); + for (USceneComponent *SceneComponent : AttachedComponents) { + UTaggedComponent *TaggedSceneComponent = Cast(SceneComponent); + if (IsValid(TaggedSceneComponent)) { + TaggedComponent = TaggedSceneComponent; +#ifdef CARLA_TAGGER_EXTRA_LOG + UE_LOG(LogCarla, Log, TEXT(" - Found Tag")); +#endif // CARLA_TAGGER_EXTRA_LOG + break; + } + } + + // If not found, then create new tagged component and attach it to this component + if (!TaggedComponent) { + TaggedComponent = NewObject(Component); + TaggedComponent->SetupAttachment(Component); + TaggedComponent->RegisterComponent(); +#ifdef CARLA_TAGGER_EXTRA_LOG + UE_LOG(LogCarla, Log, TEXT(" - Added Tag")); +#endif // CARLA_TAGGER_EXTRA_LOG + } + + // Set tagged component color + FLinearColor Color = GetActorLabelColor(Actor, Label); +#ifdef CARLA_TAGGER_EXTRA_LOG + UE_LOG(LogCarla, Log, TEXT(" - Color: %s"), *Color.ToString()); +#endif // CARLA_TAGGER_EXTRA_LOG + + TaggedComponent->SetColor(Color); + TaggedComponent->MarkRenderStateDirty(); + } } diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/Tagger.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/Tagger.h index 2213a65f4..049f90ef0 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/Tagger.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/Tagger.h @@ -84,6 +84,10 @@ public: static void SetStencilValue(UPrimitiveComponent &Component, const crp::CityObjectLabel &Label, const bool bSetRenderCustomDepth); + static FLinearColor GetActorLabelColor(const AActor &Actor, const crp::CityObjectLabel &Label); + + static bool IsThing(const crp::CityObjectLabel &Label); + ATagger(); protected: diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.cpp new file mode 100644 index 000000000..c9701ad78 --- /dev/null +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma +// de Barcelona (UAB). +// +// This work is licensed under the terms of the MIT license. +// For a copy, see . + +#include "Carla.h" +#include "Carla/Sensor/InstanceSegmentationCamera.h" +#include "Carla/Game/TaggedComponent.h" + +#include "Carla/Sensor/PixelReader.h" + +#include "Components/SceneCaptureComponent2D.h" + +FActorDefinition AInstanceSegmentationCamera::GetSensorDefinition() +{ + return UActorBlueprintFunctionLibrary::MakeCameraDefinition(TEXT("instance_segmentation")); +} + +AInstanceSegmentationCamera::AInstanceSegmentationCamera( + const FObjectInitializer &ObjectInitializer) + : Super(ObjectInitializer) +{ + AddPostProcessingMaterial(TEXT("Material'/Carla/PostProcessingMaterials/PhysicLensDistortion.PhysicLensDistortion'")); + // TODO: Setup OnActorSpawnHandler so we can refresh components + // World->AddOnActorSpawnedHandler(FOnActorSpawned::FDelegate::CreateRaw(this, &AInstanceSegmentationCamera::OnActorSpawned)); +} + +void AInstanceSegmentationCamera::SetUpSceneCaptureComponent(USceneCaptureComponent2D &SceneCapture) +{ + Super::SetUpSceneCaptureComponent(SceneCapture); + + ApplyViewMode(VMI_Unlit, true, SceneCapture.ShowFlags); + + SceneCapture.ShowFlags.SetNotDrawTaggedComponents(false); // TaggedComponent detects this and sets view relevance for proxy material + //SceneCapture.ShowFlags.SetDecals(false); + //SceneCapture.ShowFlags.SetPostProcessing(false); + //SceneCapture.ShowFlags.SetAmbientCubemap(false); + //SceneCapture.ShowFlags.SetVignette(false); + //SceneCapture.ShowFlags.SetRectLights(false); + //SceneCapture.ShowFlags.SetToneCurve(false); + //SceneCapture.ShowFlags.SetScreenPercentage(false); + //SceneCapture.ShowFlags.SetReflectionEnvironment(false); + //SceneCapture.ShowFlags.SetSpecular(false); + //SceneCapture.ShowFlags.SetContactShadows(false); + //SceneCapture.ShowFlags.SetRayTracedDistanceFieldShadows(false); + //SceneCapture.ShowFlags.SetCapsuleShadows(false); + //SceneCapture.ShowFlags.SetVolumetricLightmap(false); + //SceneCapture.ShowFlags.SetIndirectLightingCache(false); + //SceneCapture.ShowFlags.SetTexturedLightProfiles(false); + //SceneCapture.ShowFlags.SetDeferredLighting(false); + //SceneCapture.ShowFlags.SetTranslucency(false); + //SceneCapture.ShowFlags.SetBillboardSprites(false); + //SceneCapture.ShowFlags.SetBSPTriangles(false); // hmm + //SceneCapture.ShowFlags.SetLandscape(false); // hmm + //SceneCapture.ShowFlags.SetBSP(false); // hmm + //SceneCapture.ShowFlags.SetPostProcessMaterial(false); // hmm + SceneCapture.ShowFlags.SetAtmosphere(false); + //SceneCapture.ShowFlags.SetPrecomputedVisibility(false); + //SceneCapture.ShowFlags.SetPaper2DSprites(false); + //SceneCapture.ShowFlags.SetDistanceFieldAO(false); + //SceneCapture.ShowFlags.SetWidgetComponents(false); + //SceneCapture.ShowFlags.SetMediaPlanes(false); + + //SceneCapture.PostProcessSettings.AutoExposureBias = 0; + + SceneCapture.PrimitiveRenderMode = ESceneCapturePrimitiveRenderMode::PRM_UseShowOnlyList; + + TArray TaggedComponents; + GetObjectsOfClass(UTaggedComponent::StaticClass(), TaggedComponents, false, EObjectFlags::RF_ClassDefaultObject, EInternalObjectFlags::AllFlags); + + UE_LOG(LogCarla, Warning, TEXT("AInstanceSegmentationCamera::SetUpSceneCaptureComponent with %d tagged components!"), TaggedComponents.Num()); + + TArray ShowOnlyComponents; + for (UObject *Object : TaggedComponents) { + UPrimitiveComponent *Component = Cast(Object); + SceneCapture.ShowOnlyComponents.Emplace(Component); + } +} + +void AInstanceSegmentationCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) +{ + TRACE_CPUPROFILER_EVENT_SCOPE(AInstanceSegmentationCamera::PostPhysTick); + + USceneCaptureComponent2D* SceneCapture = GetCaptureComponent2D(); + TArray TaggedComponents; + GetObjectsOfClass(UTaggedComponent::StaticClass(), TaggedComponents, false, EObjectFlags::RF_ClassDefaultObject, EInternalObjectFlags::AllFlags); + + // UE_LOG(LogCarla, Warning, TEXT("AInstanceSegmentationCamera::SetUpSceneCaptureComponent with %d tagged components!"), TaggedComponents.Num()); + SceneCapture->ClearShowOnlyComponents(); + for (UObject *Object : TaggedComponents) { + UPrimitiveComponent *Component = Cast(Object); + SceneCapture->ShowOnlyComponents.Emplace(Component); + } + + FPixelReader::SendPixelsInRenderThread(*this); + +} diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.h new file mode 100644 index 000000000..15494dd1c --- /dev/null +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.h @@ -0,0 +1,31 @@ +// Copyright (c) 2017 Computer Vision Center (CVC) at the Universitat Autonoma +// de Barcelona (UAB). +// +// This work is licensed under the terms of the MIT license. +// For a copy, see . + +#pragma once + +#include "Carla/Sensor/ShaderBasedSensor.h" + +#include "Carla/Actor/ActorDefinition.h" + +#include "InstanceSegmentationCamera.generated.h" + +/// Sensor that produces "Instance segmentation" images. +UCLASS() +class CARLA_API AInstanceSegmentationCamera : public AShaderBasedSensor +{ + GENERATED_BODY() + +public: + + static FActorDefinition GetSensorDefinition(); + + AInstanceSegmentationCamera(const FObjectInitializer &ObjectInitializer); + +protected: + + void SetUpSceneCaptureComponent(USceneCaptureComponent2D &SceneCapture) override; + void PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) override; +};