p71924506
/
carla
mirror of https://github.com/carla-simulator/carla.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for depth cameras

This commit is contained in:
nsubiron 2017-03-23 12:27:40 +01:00
parent 9b96eea8f1
commit f2cf748009
4 changed files with 76 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
Source/Carla/Game/CarlaGameController.cpp
View File

@ -11,6 +11,8 @@
#include <carla/CarlaServer.h>
#define CSTEXT(text) TEXT("CarlaServer: " text)
// =============================================================================
// -- Set functions ------------------------------------------------------------
// =============================================================================
@ -52,7 +54,7 @@ static bool ReadSceneInit(carla::CarlaServer &Server)
{
carla::Mode mode;
uint32 scene;
UE_LOG(LogCarla, Log, TEXT("Waiting for tryReadSceneInit..."));
UE_LOG(LogCarla, Log, CSTEXT("Waiting for tryReadSceneInit..."));
while (!Server.tryReadSceneInit(mode, scene)) {
if (Server.needsRestart())
return false;
@ -72,23 +74,23 @@ static bool SendAndReadSceneValues(
for (APlayerStart *StartSpot : AvailableStartSpots) {
check(StartSpot != nullptr);
const FVector &Location = StartSpot->GetActorLocation();
UE_LOG(LogCarla, Log, TEXT("Found start position {%f, %f}"), Location.X, Location.Y);
UE_LOG(LogCarla, Log, CSTEXT("Found start position {%f, %f}"), Location.X, Location.Y);
sceneValues.possible_positions.push_back({Location.X, Location.Y});
}
// Send scene values.
UE_LOG(LogCarla, Log, TEXT("Send scene values: %d positions"), sceneValues.possible_positions.size());
UE_LOG(LogCarla, Log, CSTEXT("Send scene values: %d positions"), sceneValues.possible_positions.size());
Server.sendSceneValues(sceneValues);
// Wait till we receive the answer.
uint32 EndIndex;
UE_LOG(LogCarla, Log, TEXT("Waiting for episode start..."));
UE_LOG(LogCarla, Log, CSTEXT("Waiting for episode start..."));
while (!Server.tryReadEpisodeStart(StartIndex, EndIndex)) {
if (Server.needsRestart())
return false;
}
UE_LOG(LogCarla, Log, TEXT("Episode start received: %d -> %d"), StartIndex, EndIndex);
UE_LOG(LogCarla, Log, CSTEXT("Episode start received: %d -> %d"), StartIndex, EndIndex);
// Make sure the index is in range.
if (StartIndex >= AvailableStartSpots.Num()) {
UE_LOG(LogCarla, Warning, TEXT("Received invalid start index, using zero instead"));
UE_LOG(LogCarla, Warning, CSTEXT("Received invalid start index, using zero instead"));
StartIndex = 0u;
}
return true;
@ -97,7 +99,8 @@ static bool SendAndReadSceneValues(
static void SendReward(
carla::CarlaServer &Server,
const ACarlaPlayerState &PlayerState,
const std::array<const ASceneCaptureCamera *, 2u> &Cameras)
const std::array<const ASceneCaptureCamera *, 2u> &RGBCameras,
const std::array<const ASceneCaptureCamera *, 2u> &DepthCameras)
{
carla::Reward_Values reward;
reward.timestamp = FMath::RoundHalfToZero(1000.0 * FPlatformTime::Seconds());
@ -110,15 +113,15 @@ static void SendReward(
Set(reward.collision_general, PlayerState.GetCollisionIntensityOther());
// Set(reward.intersect_other_lane, );
// Set(reward.intersect_offroad, );
if (Cameras[0u] != nullptr) {
reward.image_width = Cameras[0u]->GetImageSizeX();
reward.image_height = Cameras[0u]->GetImageSizeY();
if (RGBCameras[0u] != nullptr) { // Do not add any camera if first is invalid.
reward.image_width = RGBCameras[0u]->GetImageSizeX();
reward.image_height = RGBCameras[0u]->GetImageSizeY();
Set(reward.image_rgb_0, RGBCameras[0u]);
Set(reward.image_rgb_1, RGBCameras[1u]);
Set(reward.image_depth_0, DepthCameras[0u]);
Set(reward.image_depth_1, DepthCameras[1u]);
}
Set(reward.image_rgb_0, Cameras[0u]);
Set(reward.image_rgb_1, Cameras[1u]);
// Set(reward.image_depth_0, );
// Set(reward.image_depth_1, );
UE_LOG(LogCarla, Log, TEXT("Sending reward"));
UE_LOG(LogCarla, Log, CSTEXT("Sending reward"));
Server.sendReward(reward);
}
@ -129,7 +132,7 @@ static void TryReadControl(carla::CarlaServer &Server, ACarlaVehicleController &
if (Server.tryReadControl(steer, throttle)) {
Player.SetSteeringInput(steer);
Player.SetThrottleInput(throttle);
UE_LOG(LogCarla, Log, TEXT("Read control: steer = %f, throttle = %f"), steer, throttle);
UE_LOG(LogCarla, Log, CSTEXT("Read control: steer = %f, throttle = %f"), steer, throttle);
}
}
@ -140,25 +143,27 @@ static void TryReadControl(carla::CarlaServer &Server, ACarlaVehicleController &
CarlaGameController::CarlaGameController() :
Server(MakeUnique<carla::CarlaServer>(2001u, 2002u, 2000u)),
Player(nullptr),
Cameras({{nullptr}}) {}
RGBCameras({{nullptr}}),
DepthCameras({{nullptr}}) {}
CarlaGameController::~CarlaGameController()
{
UE_LOG(LogCarla, Log, TEXT("Destroying CarlaGameController..."));
UE_LOG(LogCarla, Log, CSTEXT("Destroying CarlaGameController..."));
}
void CarlaGameController::Initialize()
{
if (bServerNeedsRestart) {
UE_LOG(LogCarla, Log, TEXT("Initializing CarlaServer"));
UE_LOG(LogCarla, Log, CSTEXT("Initializing CarlaServer"));
Server->init(1u);
if (ReadSceneInit(*Server)) {
bServerNeedsRestart = false;
} else {
UE_LOG(LogCarla, Warning, TEXT("Read scene init failed, server needs restart"));
UE_LOG(LogCarla, Warning, CSTEXT("Read scene init failed, server needs restart"));
}
}
Cameras = {{nullptr}};
RGBCameras = {{nullptr}};
DepthCameras = {{nullptr}};
}
APlayerStart *CarlaGameController::ChoosePlayerStart(
@ -166,7 +171,7 @@ APlayerStart *CarlaGameController::ChoosePlayerStart(
{
uint32 StartIndex;
if (!SendAndReadSceneValues(*Server, AvailableStartSpots, StartIndex)) {
UE_LOG(LogCarla, Warning, TEXT("Read scene values failed, server needs restart"));
UE_LOG(LogCarla, Warning, CSTEXT("Read scene values failed, server needs restart"));
StartIndex = 0u;
}
return AvailableStartSpots[StartIndex];
@ -180,17 +185,27 @@ void CarlaGameController::RegisterPlayer(AController &NewPlayer)
void CarlaGameController::RegisterCaptureCamera(const ASceneCaptureCamera &CaptureCamera)
{
const auto Effect = CaptureCamera.GetPostProcessEffect();
check((Effect == EPostProcessEffect::None) || (Effect == EPostProcessEffect::Depth))
auto &Cameras = (Effect == EPostProcessEffect::None ? RGBCameras : DepthCameras);
for (auto i = 0u; i < Cameras.size(); ++i) {
if (Cameras[i] == nullptr) {
Cameras[i] = &CaptureCamera;
UE_LOG(LogCarla, Log, TEXT("Registered capture camera %d"), i);
UE_LOG(
LogCarla,
Log,
CSTEXT("Registered capture camera %d with postprocess \"%s\""),
i,
*CaptureCamera.GetPostProcessEffectAsString());
return;
}
}
UE_LOG(
LogCarla,
Warning,
TEXT("Attempting to register a camera but already have %d, captures from this camera won't be send"),
CSTEXT("Attempting to register a capture camera of type \"%d\" but already have %d, captures from this camera won't be sent"),
*CaptureCamera.GetPostProcessEffectAsString(),
Cameras.size());
}
@ -208,18 +223,18 @@ void CarlaGameController::Tick(float DeltaSeconds)
}
if (Server->newEpisodeRequested()) {
UE_LOG(LogCarla, Log, TEXT("New episode requested"));
UE_LOG(LogCarla, Log, CSTEXT("New episode requested"));
RestartLevel(false);
return;
}
SendReward(*Server, Player->GetPlayerState(), Cameras);
SendReward(*Server, Player->GetPlayerState(), RGBCameras, DepthCameras);
TryReadControl(*Server, *Player);
}
void CarlaGameController::RestartLevel(bool ServerNeedsRestart)
{
UE_LOG(LogCarla, Log, TEXT("Restarting..."));
UE_LOG(LogCarla, Log, CSTEXT("Restarting..."));
bServerNeedsRestart = ServerNeedsRestart;
Player->RestartLevel();
}

4
Source/Carla/Game/CarlaGameController.h
View File

@ -39,7 +39,9 @@ private:
ACarlaVehicleController *Player;
std::array<const ASceneCaptureCamera *, 2u> Cameras;
std::array<const ASceneCaptureCamera *, 2u> RGBCameras;
std::array<const ASceneCaptureCamera *, 2u> DepthCameras;
bool bServerNeedsRestart = true;
};

11
Source/Carla/SceneCaptureCamera.cpp
View File

@ -17,7 +17,8 @@
ASceneCaptureCamera::ASceneCaptureCamera(const FObjectInitializer& ObjectInitializer) :
Super(ObjectInitializer),
SizeX(200u),
SizeY(200u)
SizeY(200u),
PostProcessEffect(EPostProcessEffect::None)
{
PrimaryActorTick.bCanEverTick = true;
PrimaryActorTick.TickGroup = TG_PrePhysics;
@ -92,6 +93,14 @@ void ASceneCaptureCamera::Tick(float Delta)
RTResource->ReadPixels(ImageBitMap, ReadPixelFlags);
}
FString ASceneCaptureCamera::GetPostProcessEffectAsString() const
{
const UEnum* ptr = FindObject<UEnum>(ANY_PACKAGE, TEXT("EPostProcessEffect"), true);
if(!ptr)
return FString("Invalid");
return ptr->GetEnumName(static_cast<int32>(PostProcessEffect));
}
void ASceneCaptureCamera::UpdateDrawFrustum()
{
if(DrawFrustum && CaptureComponent2D)

23
Source/Carla/SceneCaptureCamera.h
View File

@ -15,7 +15,16 @@ class UDrawFrustumComponent;
class USceneCaptureComponent2D;
class UStaticMeshComponent;
UCLASS(hidecategories=(Collision, Material, Attachment, Actor))
UENUM(BlueprintType)
enum class EPostProcessEffect : uint8
{
None UMETA(DisplayName = "RGB"),
Depth UMETA(DisplayName = "Depth Map"),
SIZE UMETA(Hidden)
};
UCLASS(hidecategories=(Collision, Attachment, Actor))
class CARLA_API ASceneCaptureCamera : public AActor
{
GENERATED_BODY()
@ -44,6 +53,13 @@ public:
return SizeY;
}
EPostProcessEffect GetPostProcessEffect() const
{
return PostProcessEffect;
}
FString GetPostProcessEffectAsString() const;
const TArray<FColor> &GetImage() const
{
return ImageBitMap;
@ -61,6 +77,9 @@ private:
UPROPERTY(Category = SceneCapture, EditAnywhere)
uint32 SizeY;
UPROPERTY(Category = SceneCapture, EditAnywhere)
EPostProcessEffect PostProcessEffect;
/** To display the 3d camera in the editor. */
UPROPERTY()
class UStaticMeshComponent* MeshComp;
@ -74,7 +93,7 @@ private:
class UTextureRenderTarget2D* CaptureRenderTarget;
/** Scene capture component. */
UPROPERTY(Transient)
UPROPERTY(EditAnywhere)
class USceneCaptureComponent2D* CaptureComponent2D;
UPROPERTY()