From dd7969e7d734942bde790e3d461d5b4971ec31dc Mon Sep 17 00:00:00 2001 From: "CVC\\jbelon" Date: Tue, 20 Feb 2018 17:45:24 +0100 Subject: [PATCH] Now ScreenCaptureCamera read pixels in render queue. --- .../Plugins/Carla/Source/Carla/Carla.Build.cs | 2 + .../Carla/Source/Carla/CityMapGenerator.cpp | 21 ++---- .../Source/Carla/Game/CarlaGameController.cpp | 25 +++++-- .../Carla/Game/CarlaVehicleController.cpp | 22 ++++-- .../Carla/Game/CarlaVehicleController.h | 1 + .../Carla/Source/Carla/SceneCaptureCamera.cpp | 73 ++++++++++++++++++- .../Carla/Source/Carla/SceneCaptureCamera.h | 1 + 7 files changed, 118 insertions(+), 27 deletions(-) diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Carla.Build.cs b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Carla.Build.cs index 06b471539..82e281463 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Carla.Build.cs +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Carla.Build.cs @@ -23,6 +23,8 @@ public class Carla : ModuleRules new string[] { "Core", + "RenderCore", + "RHI" // ... add other public dependencies that you statically link with here ... } ); diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/CityMapGenerator.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/CityMapGenerator.cpp index b18aa766a..c3b07e4a6 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/CityMapGenerator.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/CityMapGenerator.cpp @@ -291,21 +291,16 @@ void ACityMapGenerator::GenerateRoadMap() // Do the ray tracing. FHitResult Hit; if (LineTrace(World, Start, End, Hit)) { - auto InstancedStaticMeshComponent = Cast(Hit.Component.Get()); - if (InstancedStaticMeshComponent == nullptr) { + auto StaticMeshComponent = Cast(Hit.Component.Get()); + if (StaticMeshComponent == nullptr) { UE_LOG(LogCarla, Error, TEXT("Road component is not UInstancedStaticMeshComponent")); } else { - FTransform InstanceTransform; - if (!InstancedStaticMeshComponent->GetInstanceTransform(Hit.Item, InstanceTransform, true)) { - UE_LOG(LogCarla, Error, TEXT("Failed to get instance's transform")); - } else { - RoadMap->SetPixelAt( - PixelX, - PixelY, - GetTag(*InstancedStaticMeshComponent->GetStaticMesh()), - InstanceTransform, - bLeftHandTraffic); - } + RoadMap->SetPixelAt( + PixelX, + PixelY, + GetTag(*StaticMeshComponent->GetStaticMesh()), + StaticMeshComponent->GetOwner()->GetTransform(), + bLeftHandTraffic); } } } diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameController.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameController.cpp index 6c80d71d4..78a07f603 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameController.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaGameController.cpp @@ -11,6 +11,8 @@ #include "Settings/CarlaSettings.h" #include "CarlaServer.h" +#include "RenderingThread.h" + using Errc = CarlaServer::ErrorCode; @@ -115,16 +117,27 @@ void CarlaGameController::Tick(float DeltaSeconds) // Send measurements. { check(GameState != nullptr); - if (Errc::Error == Server->SendMeasurements( + ENQUEUE_UNIQUE_RENDER_COMMAND_FOURPARAMETER( /** @TODO: move to the screencapturecamera */ + FSendMeasurements, + CarlaServer*,Server,Server.Get(), /** @TODO: move unique pointer lambda reference */ + const ACarlaGameState*, GameState, GameState, + const ACarlaPlayerState&, PlayerState,Player->GetPlayerState(), + //,ParamType4,ParamName4,ParamValue4,Code + bool,bSendNonPlayerAgentsInfo,CarlaSettings->bSendNonPlayerAgentsInfo, + { + if (Errc::Error == Server->SendMeasurements( *GameState, - Player->GetPlayerState(), - CarlaSettings->bSendNonPlayerAgentsInfo)) { - Server = nullptr; - return; - } + PlayerState, + bSendNonPlayerAgentsInfo)) + { + Server = nullptr; + } + }); + } // Read control, block if the settings say so. + if(Server!=nullptr) { const bool bShouldBlock = CarlaSettings->bSynchronousMode; if (Errc::Error == Server->ReadControl(*Player, bShouldBlock)) { diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaVehicleController.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaVehicleController.cpp index d2a29baa8..469eb9333 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaVehicleController.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaVehicleController.cpp @@ -14,6 +14,11 @@ #include "GameFramework/Pawn.h" #include "WheeledVehicle.h" #include "WheeledVehicleMovementComponent.h" +#include "CarlaPlayerState.h" +#include "CarlaWheeledVehicle.h" +#include "CarlaHUD.h" +#include "MapGen/RoadMap.h" +#include "Settings/CameraPostProcessParameters.h" // ============================================================================= // -- Constructor and destructor ----------------------------------------------- @@ -99,11 +104,18 @@ void ACarlaVehicleController::Tick(float DeltaTime) IntersectPlayerWithRoadMap(); const auto NumberOfCameras = SceneCaptureCameras.Num(); check(NumberOfCameras == CarlaPlayerState->Images.Num()); - for (auto i = 0; i < NumberOfCameras; ++i) { - auto &Image = CarlaPlayerState->Images[i]; - if (!SceneCaptureCameras[i]->ReadPixels(Image.BitMap)) { - Image.BitMap.Empty(); - } + + for (auto i = 0; i < NumberOfCameras; ++i) + { + FCapturedImage *Image = &CarlaPlayerState->Images[i]; + ASceneCaptureCamera *Camera = SceneCaptureCameras[i]; + ENQUEUE_UNIQUE_RENDER_COMMAND_TWOPARAMETER( + FReadFromTexture, + FCapturedImage*, Image, Image, + ASceneCaptureCamera *, Camera, Camera, + { + Camera->WritePixels(Image->BitMap); + }); } } } diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaVehicleController.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaVehicleController.h index cbe7e0b7c..8fd69b7dc 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaVehicleController.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/CarlaVehicleController.h @@ -9,6 +9,7 @@ #include "WheeledVehicleController.h" #include "CarlaVehicleController.generated.h" +struct FCameraPostProcessParameters; class ACarlaHUD; class ACarlaPlayerState; class ASceneCaptureCamera; diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/SceneCaptureCamera.cpp b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/SceneCaptureCamera.cpp index bca6eac3d..439f54fb0 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/SceneCaptureCamera.cpp +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/SceneCaptureCamera.cpp @@ -15,8 +15,7 @@ #include "HighResScreenshot.h" #include "Materials/Material.h" #include "Paths.h" -#include "StaticMeshResources.h" -#include "TextureResource.h" + static constexpr auto DEPTH_MAT_PATH = @@ -198,7 +197,7 @@ bool ASceneCaptureCamera::ReadPixels(TArray &BitMap) const UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render target")); return false; } - FTextureRenderTargetResource* RTResource = CaptureRenderTarget->GameThread_GetRenderTargetResource(); + FTextureRenderTargetResource* RTResource = CaptureRenderTarget->GetRenderTargetResource(); // CaptureRenderTarget->GameThread_GetRenderTargetResource(); if (RTResource == nullptr) { UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render target")); return false; @@ -208,6 +207,74 @@ bool ASceneCaptureCamera::ReadPixels(TArray &BitMap) const return RTResource->ReadPixels(BitMap, ReadPixelFlags); } +void ASceneCaptureCamera::WritePixels(TArray& RawData) + { + check(IsInRenderingThread()); + if(!CaptureRenderTarget) + { + UE_LOG(LogCarla, Error, TEXT("SceneCaptureCamera: Missing render target")); + return ; + } + FRHITexture2D *texture = CaptureRenderTarget->GetRenderTargetResource()->GetRenderTargetTexture(); + + uint32 width = texture->GetSizeX(); + uint32 height = texture->GetSizeY(); + uint32 stride; + uint8 *src = reinterpret_cast(RHILockTexture2D(texture, 0, RLM_ReadOnly, stride, false)); + const uint32 bufferSize = stride * height; + //Alternative Method 0: deeplearning + /*void *buffer = FMemory::Malloc(bufferSize, 4); + if(!buffer) + { + UE_LOG(LogCarla, Display, TEXT("Error allocating memory")); + RHIUnlockTexture2D(texture, 0, false); + return ; + } + //FMemory::BigBlockMemcpy(buffer, src, bufferSize);*/ + RawData.Empty(width*height); + for(unsigned y = 0; y < height; ++y) + { + for(unsigned x = 0; x < width; ++x) + { + //RGBA -> + RawData.Add(FColor(src[2],src[1],src[0],src[3])); + src += 4; + } + } + + RHIUnlockTexture2D(texture, 0, false); + //Alternative Method 1 + /*FColor* RHIData = reinterpret_cast(RHILockTexture2D(TextureRHI, 0, RLM_ReadOnly, stride, false, false)); + if(!RHIData) { + UE_LOG(LogCarla,Display,TEXT("ASceneCaptureCamera::WritePixels : RHIData is null")); + return; + } + RawData.Append(RHIData,Width*Height);*/ + //Alternative Method 2 + /*uint8* OriginBuffer = reinterpret_cast(RHILockTexture2D(TextureRHI, 0, RLM_ReadOnly, stride, false,false)); + if(OriginBuffer) + { + for (int32 y = 0; y < Height; y++) + { + uint8* OriginPtr = &OriginBuffer[(Height - 1 - y) * stride]; + for (int32 x = 0; x < Width; x++) + { + RawData[(Height - 1 - y) * Width] = FColor(*OriginPtr++,*OriginPtr++,*OriginPtr++,*OriginPtr++); + } + } + } + */ + //Alternative Method 3 + /* + FRHITexture2D* Texture2D = TextureRHI->GetTexture2D(); + uint8* TextureBuffer = (uint8*)RHILockTexture2D(Texture2D, 0, RLM_ReadOnly, stride, false, false); + FMemory::Memcpy( RawData.GetData(), TextureBuffer, stride + (Width*Height*4) ); + */ + //End of Alternative methods lock + //RHIUnlockTexture2D(TextureRHI, 0, false, false); +} + + void ASceneCaptureCamera::UpdateDrawFrustum() { if(DrawFrustum && CaptureComponent2D) diff --git a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/SceneCaptureCamera.h b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/SceneCaptureCamera.h index 360bbebfd..d88dca765 100644 --- a/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/SceneCaptureCamera.h +++ b/Unreal/CarlaUE4/Plugins/Carla/Source/Carla/SceneCaptureCamera.h @@ -65,6 +65,7 @@ public: const FCameraPostProcessParameters &OverridePostProcessParameters); bool ReadPixels(TArray &BitMap) const; + void WritePixels(TArray& RawData); private: