NVIDIA DeepLearning Dataset Synthesizer (NDDS)
 All Classes Namespaces Functions Variables Typedefs Pages
NVSceneCapturerActor.h
1 /*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * NVIDIA CORPORATION and its licensors retain all intellectual property
5 * and proprietary rights in and to this software, related documentation
6 * and any modifications thereto. Any use, reproduction, disclosure or
7 * distribution of this software and related documentation without an express
8 * license agreement from NVIDIA CORPORATION is strictly prohibited.
9 */
10 
11 #pragma once
12 
13 #include "GameFramework/Actor.h"
14 #include "NVSceneCapturerUtils.h"
15 #include "NVSceneFeatureExtractor.h"
16 #include "NVSceneCapturerViewpointComponent.h"
17 #include "NVImageExporter.h"
18 #include "NVSceneDataHandler.h"
19 #include "NVSceneCapturerActor.generated.h"
20 
21 USTRUCT(BlueprintType)
22 struct NVSCENECAPTURER_API FNCapturerSettingExportedActorData
23 {
24  GENERATED_BODY();
25 
26 public:
27  UPROPERTY()
28  FString Class;
29 
30  UPROPERTY()
31  uint32 segmentation_class_id;
32 
33  UPROPERTY()
34  uint32 segmentation_instance_id;
35 
36  UPROPERTY()
37  FMatrix fixed_model_transform;
38 
39  UPROPERTY()
40  FVector cuboid_dimensions;
41 
42  UPROPERTY(Transient)
43  FVector CuboidCenterLocal;
44 
45  UPROPERTY(Transient)
46  AActor* ActorRef;
47 };
48 
49 USTRUCT(BlueprintType)
50 struct NVSCENECAPTURER_API FNVSceneAnnotatedActorData
51 {
52  GENERATED_BODY();
53 
54 public:
55  UPROPERTY()
56  TArray<FString> exported_object_classes;
57 
58  UPROPERTY()
59  TArray<FNCapturerSettingExportedActorData> exported_objects;
60 };
61 
62 
63 USTRUCT(BlueprintType)
64 struct NVSCENECAPTURER_API FNVCapturerSettingExportData
65 {
66  GENERATED_BODY();
67 
68 public:
69  UPROPERTY()
70  FNVSceneCapturerSettings CapturersSettings;
71 
72  UPROPERTY()
73  uint32 ExportedObjectCount;
74 
75  UPROPERTY()
76  TArray<FNCapturerSettingExportedActorData> ExportedObjects;
77 };
78 
79 USTRUCT(BlueprintType)
80 struct NVSCENECAPTURER_API FNVViewpointSettingExportData
81 {
82  GENERATED_BODY();
83 
84 public:
85  UPROPERTY()
86  FString Name;
87 
88  /// Horizontal field-of-view
89  UPROPERTY()
90  float horizontal_fov;
91 
92  UPROPERTY()
93  FCameraIntrinsicSettings intrinsic_settings;
94 
95  UPROPERTY()
96  FNVImageSize captured_image_size;
97 
98  UPROPERTY(Transient)
99  FMatrix CameraProjectionMatrix;
100 };
101 
102 USTRUCT(BlueprintType)
103 struct NVSCENECAPTURER_API FNVCameraSettingExportData
104 {
105  GENERATED_BODY();
106 
107 public:
108  UPROPERTY()
109  TArray<FNVViewpointSettingExportData> camera_settings;
110 };
111 
114 
115 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FNVSceneCapturer_Started, ANVSceneCapturerActor*, SceneCapturer);
116 DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FNVSceneCapturer_Stopped, ANVSceneCapturerActor*, SceneCapturer);
117 DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FNVSceneCapturer_Completed, ANVSceneCapturerActor*, SceneCapturer, bool, bIsSucceeded);
118 
119 ///
120 /// The scene exporter actor.
121 ///
122 /// @cond DOXYGEN_SUPPRESSED_CODE
123 UCLASS(Blueprintable, ClassGroup = (NVIDIA), Config=Engine,
124  HideCategories = (Replication, Tick, Tags, Input, Actor, Rendering, Collision, Physics, Navigation, Shape, Cooking, HLOD, Mobile))
125 /// @endcond DOXYGEN_SUPPRESSED_CODE
126 class NVSCENECAPTURER_API ANVSceneCapturerActor : public AActor
127 {
128  GENERATED_BODY()
129 
130 public:
131  /// Use SpawnActor() to create this instance.
132  ANVSceneCapturerActor(const FObjectInitializer& ObjectInitializer);
133 
134  /// Setter and Getter for Number of scene to capture
135  UFUNCTION(BlueprintCallable, Category = "Capturer")
136  void SetNumberOfFramesToCapture(int32 NewSceneCount);
137  UFUNCTION(BlueprintCallable, Category = "Capturer")
138  int32 GetNumberOfFramesToCapture() const
139  {
140  return NumberOfFramesToCapture;
141  }
142 
143  /// Capture controls
144  UFUNCTION(BlueprintCallable, Category = "Capturer")
145  void StartCapturing();
146  UFUNCTION(BlueprintCallable, Category = "Capturer")
147  void StopCapturing();
148  UFUNCTION(BlueprintCallable, Category = "Capturer")
149  void PauseCapturing();
150  UFUNCTION(BlueprintCallable, Category = "Capturer")
151  void ResumeCapturing();
152 
153  /// return false means PlayerController view.
154  /// true means Viewport is taken over.
155  UFUNCTION(BlueprintCallable, Category = "Capturer")
156  bool ToggleTakeOverViewport();
157 
158  UFUNCTION(BlueprintCallable, Category = "Capturer")
159  void TakeOverViewport();
160  UFUNCTION(BlueprintCallable, Category = "Capturer")
161  void ReturnViewportToPlayerController();
162 
163  UFUNCTION(BlueprintCallable, Category = "Capturer")
164  ENVSceneCapturerState GetCurrentState() const
165  {
166  return CurrentState;
167  }
168 
169  /// Frame counters
170  UFUNCTION(BlueprintCallable, Category = "Capturer")
172  {
173  return CapturedFrameCounter;
174  }
175 
176  /// Capturing information
177  UFUNCTION(BlueprintCallable, Category = "Capturer")
178  float GetCapturedFPS() const;
179  UFUNCTION(BlueprintCallable, Category = "Capturer")
180  int32 GetNumberOfFramesLeftToCapture() const;
181  UFUNCTION(BlueprintCallable, Category = "Capturer")
182  float GetEstimatedTimeUntilFinishCapturing() const;
183  UFUNCTION(BlueprintCallable, Category = "Capturer")
184  float GetCaptureProgressFraction() const;
185  UFUNCTION(BlueprintCallable, Category = "Capturer")
186  float GetCapturedDuration() const;
187 
188  UFUNCTION(BlueprintCallable, Category = "Capturer")
189  TArray<UNVSceneCapturerViewpointComponent*> GetViewpointList();
190 
191  /// Control what to do with the captured scene data
192  UFUNCTION(BlueprintCallable, Category = "Capturer")
194 
195  /// Control what to do with the captured scene data
197 
198  static TArray<FNVNamedImageSizePreset> const& GetImageSizePresets();
199 
200  /// Event properties
201  UPROPERTY(BlueprintAssignable, Category = "Events")
202  FNVSceneCapturer_Started OnStartedEvent;
203  UPROPERTY(BlueprintAssignable, Category = "Events")
204  FNVSceneCapturer_Stopped OnStoppedEvent;
205  UPROPERTY(BlueprintAssignable, Category = "Events")
206  FNVSceneCapturer_Completed OnCompletedEvent;
207 
208 protected:
209  virtual void PostLoad() final;
210  virtual void BeginPlay() final;
211  virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) final;
212  virtual void PostInitializeComponents() final;
213  virtual void Tick(float DeltaTime) final;
214 #if WITH_EDITORONLY_DATA
215  virtual void PostEditChangeProperty(struct FPropertyChangedEvent& PropertyChangedEvent) override;
216  void OnActorSelected(UObject* Object);
217 #endif //WITH_EDITORONLY_DATA
218 
219  void ResetCounter();
220  void UpdateSettingsFromCommandLine();
221  void UpdateViewpointList();
222  void StartCapturing_Internal();
223  void CaptureSceneToPixelsData();
224  void CheckCaptureScene();
225  void UpdateCapturerSettings();
226  void OnCompleted();
227  bool CanHandleMoreSceneData() const;
228 
229 public: // Editor properties
230  /// Whether this capturer actor is active and can start capturing or not
231  UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Capture")
232  bool bIsActive;
233 
234  UPROPERTY(EditAnywhere, SimpleDisplay, Category = Settings, meta = (ShowOnlyInnerProperties))
235  FNVSceneCapturerSettings CapturerSettings;
236 
237  /// List of the feature extractors this capturer support
238  UPROPERTY(EditAnywhere, SimpleDisplay, Category = FeatureExtraction, meta = (ShowOnlyInnerProperties))
240 
241 protected: // Editor properties
242  /// Collision of the capturer actor
243  UPROPERTY(VisibleAnywhere, BlueprintReadOnly, meta = (AllowPrivateAccess = "true"))
244  class USphereComponent* CollisionComponent;
245 
246  /// If true, this capturer will automatically start capturing the scene right when the game start (every TimeBetweenSceneExport seconds)
247  UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Capture")
249 
250  /// NOTE: TimeBetweenSceneExport <= 0 mean export every frame
251  UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Capture")
253 
254  /// Maximum number of scenes (>= 0) to export before stopping
255  /// NOTE: If TotalNumberOfScenesToExport == 0 then the exporter will keep exporting without limit until told to stop
256  UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Capture", meta=(UIMin=0))
258 
259  /// If true, the player's camera will be tied to this exporter's location and rotation
260  UPROPERTY(EditAnywhere, Category = "Capture")
262 
263  /// Control what to do with the captured scene data
264  UPROPERTY(EditAnywhere, Instanced, BlueprintReadOnly, Category = "Capture")
266 
267  /// Control what to do with the captured scene data
268  UPROPERTY(EditAnywhere, Instanced, BlueprintReadOnly, Category = "Capture")
270 
271  /// If true, the capturer will pause the game logic when it's trying to flushing - handle the scene data from previous frame
272  UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "Capture")
274 
275  /// List of available image size presets
276  UPROPERTY(config)
278 
279 protected: // Transient properties
280  UPROPERTY(Transient)
281  float StartCapturingTimestamp;
282 
283  UPROPERTY(Transient)
284  float CapturedDuration;
285 
286  UPROPERTY(Transient)
287  float StartCapturingDuration;
288 
289  UPROPERTY(Transient)
290  float LastCaptureTimestamp;
291 
292  UPROPERTY(Transient)
293  FNVFrameCounter CapturedFrameCounter;
294 
295  UPROPERTY(Transient)
296  AActor* CachedPlayerControllerViewTarget;
297 
298  UPROPERTY(Transient)
299  ENVSceneCapturerState CurrentState;
300 
301  UPROPERTY(Transient)
302  int32 NumberOfFramesToCapture;
303 
304  UPROPERTY(Transient)
305  bool bNeedToExportScene;
306 
307  UPROPERTY(Transient)
308  bool bTakingOverViewport;
309 
310  UPROPERTY(Transient)
311  bool bSkipFirstFrame;
312 
313  UPROPERTY(Transient)
314  FTimerHandle TimeHandle_StartCapturingDelay;
315 
316  UPROPERTY(Transient)
317  TArray<UNVSceneCapturerViewpointComponent*> ViewpointList;
318 };
UNVSceneCapturerViewpointComponent: Represents each viewpoint from where the capturer captures data...
int32 MaxNumberOfFramesToCapture
Maximum number of scenes (>= 0) to export before stopping NOTE: If TotalNumberOfScenesToExport == 0 t...
float TimeBetweenSceneCapture
NOTE: TimeBetweenSceneExport <= 0 mean export every frame.
TArray< FNVNamedImageSizePreset > ImageSizePresets
List of available image size presets.
class USphereComponent * CollisionComponent
Collision of the capturer actor.
FNVSceneCapturer_Started OnStartedEvent
Event properties.
class UNVSceneDataHandler * SceneDataHandler
Control what to do with the captured scene data.
UNVSceneDataHandler * GetSceneDataHandler() const
Control what to do with the captured scene data.
class UNVSceneDataVisualizer * SceneDataVisualizer
Control what to do with the captured scene data.
float GetCapturedFPS() const
Capturing information.
UNVSceneDataVisualizer * GetSceneDataVisualizer() const
Control what to do with the captured scene data.
TArray< FNVFeatureExtractorSettings > FeatureExtractorSettings
List of the feature extractors this capturer support.
ANVSceneCapturerActor(const FObjectInitializer &ObjectInitializer)
Use SpawnActor() to create this instance.
bool bPauseGameLogicWhenFlushing
If true, the capturer will pause the game logic when it's trying to flushing - handle the scene data ...
bool bAutoStartCapturing
If true, this capturer will automatically start capturing the scene right when the game start (every ...
bool bIsActive
Whether this capturer actor is active and can start capturing or not.
FNVFrameCounter GetCapturedFrameCounter() const
Frame counters.
bool bTakeOverGameViewport
If true, the player's camera will be tied to this exporter's location and rotation.
void StartCapturing()
Capture controls.
NVSceneDataVisualizer - visualize all the captured data (image buffer and object annotation info) usi...
void SetNumberOfFramesToCapture(int32 NewSceneCount)
Setter and Getter for Number of scene to capture.
The scene exporter actor.
bool ToggleTakeOverViewport()
return false means PlayerController view. true means Viewport is taken over.
Base interface for serializing/visualizing captured pixel and annotation data.