NVIDIA DeepLearning Dataset Synthesizer (NDDS)
 All Classes Namespaces Functions Variables Typedefs Pages
NVSceneDataHandler.h
1 /*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * NVIDIA CORPORATION and its licensors retain all intellectual property
5 * and proprietary rights in and to this software, related documentation
6 * and any modifications thereto. Any use, reproduction, disclosure or
7 * distribution of this software and related documentation without an express
8 * license agreement from NVIDIA CORPORATION is strictly prohibited.
9 */
10 
11 #pragma once
12 
13 #include "NVSceneCapturerUtils.h"
14 #include "NVImageExporter.h"
15 #include "NVSceneDataHandler.generated.h"
16 
17 DECLARE_LOG_CATEGORY_EXTERN(LogNVSceneDataHandler, Log, All)
18 
19 ///
20 /// Base interface for serializing/visualizing captured pixel and annotation data.
21 ///
22 UCLASS(NotBlueprintable, Abstract, DefaultToInstanced, editinlinenew, ClassGroup = (NVIDIA))
23 class NVSCENECAPTURER_API UNVSceneDataHandler : public UObject
24 {
25  GENERATED_BODY()
26 
27 public:
28  UNVSceneDataHandler() { };
29 
30  /// Check whether this handler can process more scene data
31  /// If it can't then we should stop getting more data until it's available again
32  virtual bool CanHandleMoreData() const PURE_VIRTUAL(UNVSceneDataHandler::CanHandleMoreData, return false; );
33 
34  virtual bool IsHandlingData() const PURE_VIRTUAL(UNVSceneDataHandler::IsHandlingData, return false; );
35 
36  /// Handle the pixels data captured from the scene
37  /// @param CapturedPixelData - The scene's pixels data
38  /// @param CapturedFeatureExtractor - The feature extractor which captured the data
39  /// @param CapturedViewpoint - The viewpoint which captured the data
40  /// @param FrameIndex - The frame when the data is captured
41  virtual bool HandleScenePixelsData(const FNVTexturePixelData& CapturedPixelData,
42  class UNVSceneFeatureExtractor_PixelData* CapturedFeatureExtractor,
43  class UNVSceneCapturerViewpointComponent* CapturedViewpoint,
44  int32 FrameIndex) PURE_VIRTUAL(UNVSceneDataHandler::HandleScenePixelsData, return false; );
45 
46  /// Handle the annotation data captured from the scene
47  /// @param CapturedData - The scene's annotated data
48  /// @param CapturedFeatureExtractor - The feature extractor which captured the data
49  /// @param CapturedViewpoint - The viewpoint which captured the data
50  /// @param FrameIndex - The frame when the data is captured
51  virtual bool HandleSceneAnnotationData(const TSharedPtr<FJsonObject>& CapturedData,
52  class UNVSceneFeatureExtractor_AnnotationData* CapturedFeatureExtractor,
53  class UNVSceneCapturerViewpointComponent* CapturedViewpoint,
54  int32 FrameIndex) PURE_VIRTUAL(UNVSceneDataHandler::HandleSceneAnnotationData, return false; );
55 
56  virtual void OnStartCapturingSceneData() PURE_VIRTUAL(UNVSceneDataHandler::OnStartCapturingSceneData, return; );
57  virtual void OnStopCapturingSceneData() PURE_VIRTUAL(UNVSceneDataHandler::OnStopCapturingSceneData, return; );
58  virtual void OnCapturingCompleted() PURE_VIRTUAL(UNVSceneDataHandler::OnCapturingCompleted, return; );
59 };
60 
61 //=================================== UNVSceneDataExporter ===================================
62 UENUM(BlueprintType)
63 enum class ENVCaptureDirectoryConflictHandleType : uint8
64 {
65  /// Just overwrite existing files in the same directory
66  OverwriteExistingFiles UMETA(DisplayName = "Overwrite existing files"),
67 
68  /// Clean the directory by removing all files in it
69  CleanDirectory UMETA(DisplayName = "Remove existing files"),
70 
71  /// Create new directory with timestamp as postfix in name
72  CreateNewDirectoryWithTimestampPostfix UMETA(DisplayName = "Create new directory with timestamp as postfix in name"),
73 
74  CaptureDirectoryConflictHandleType_MAX UMETA(Hidden)
75 };
76 
77 ///
78 /// NVSceneDataExporter - export all the captured data (image buffer and object annotation info) to files on disk
79 ///
80 UCLASS(Blueprintable, ClassGroup = (NVIDIA))
81 class NVSCENECAPTURER_API UNVSceneDataExporter : public UNVSceneDataHandler
82 {
83  GENERATED_BODY()
84 
85 public:
87 
88  /// Check whether this handler can process more scene data
89  /// If it can't then we should stop getting more data until it's available again
90  virtual bool CanHandleMoreData() const override;
91  virtual bool IsHandlingData() const override;
92 
93  /// Handle the pixels data captured from the scene
94  /// @param CapturedPixelData - The scene's pixels data
95  /// @param CapturedFeatureExtractor - The feature extractor which captured the data
96  /// @param CapturedViewpoint - The viewpoint which captured the data
97  /// @param FrameIndex - The frame when the data is captured
98  virtual bool HandleScenePixelsData(const FNVTexturePixelData& CapturedPixelData,
99  UNVSceneFeatureExtractor_PixelData* CapturedFeatureExtractor,
100  UNVSceneCapturerViewpointComponent* CapturedViewpoint,
101  int32 FrameIndex) override;
102 
103  /// Handle the annotation data captured from the scene
104  /// @param CapturedData - The scene's annotated data
105  /// @param CapturedFeatureExtractor - The feature extractor which captured the data
106  /// @param CapturedViewpoint - The viewpoint which captured the data
107  /// @param FrameIndex - The frame when the data is captured
108  virtual bool HandleSceneAnnotationData(const TSharedPtr<FJsonObject>& CapturedData,
109  class UNVSceneFeatureExtractor_AnnotationData* CapturedFeatureExtractor,
110  class UNVSceneCapturerViewpointComponent* CapturedViewpoint,
111  int32 FrameIndex) override;
112 
113  virtual void OnStartCapturingSceneData() override;
114  virtual void OnStopCapturingSceneData() override;
115  virtual void OnCapturingCompleted() override;
116 
117  UFUNCTION(BlueprintCallable, Category = "Exporter")
118  FString GetRootCaptureDirectoryPath() const;
119  UFUNCTION(BlueprintCallable, Category = "Exporter")
120  virtual FString GetExportFolderName() const;
121 
122  UFUNCTION(BlueprintCallable, Category = "Exporter")
123  FString GetConfiguredOutputDirectoryPath() const;
124  UFUNCTION(BlueprintCallable, Category = "Exporter")
125  FString GetConfiguredOutputDirectoryName() const;
126  UFUNCTION(BlueprintCallable, Category = "Exporter")
127  FString GetFullOutputDirectoryPath() const;
128 
129  UFUNCTION(BlueprintCallable, Category = "Exporter")
130  virtual FString GetSubFolderName() const;
131  UFUNCTION(BlueprintCallable, Category = "Exporter")
132  void SetSubFolderName(const FString& NewSubFolderName);
133 
134  UFUNCTION(BlueprintCallable, Category = "Exporter")
135  FString GetExportFilePath(class UNVSceneFeatureExtractor* CapturedFeatureExtractor,
136  UNVSceneCapturerViewpointComponent* CapturedViewpoint,
137  int32 FrameIndex,
138  const FString& FileExtension) const;
139 
140 protected:
141  void ExportCapturerSettings();
142 
143 public: // Editor properties
144  // ToDo: move to protected.
145  /// If true, the exporter will use the current map's name for the export folder, otherwise it will use the ExportFolderName
146  /// NOTE: If ExportFolderName is empty, it will fallback to use the map's name
147  UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "Save Path")
148  bool bUseMapNameForCapturedDirectory;
149 
150  UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "Save Path", meta = (EditCondition = "!bUseMapNameForCapturedDirectory"))
151  FString CustomDirectoryName;
152 
153 protected: // Editor properties
154  /// Path to the directory where to save captured data
155  UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Save Path")
156  FDirectoryPath RootCapturedDirectoryPath;
157 
158  /// How to handle conflict files
159  UPROPERTY(EditAnywhere, Category = "Save Path")
160  ENVCaptureDirectoryConflictHandleType DirectoryConflictHandleType;
161 
162  /// If true, this exporter will automatically open the exported directory after it finish exporting
163  UPROPERTY(BlueprintReadWrite, EditAnywhere, Category = "Capture")
164  bool bAutoOpenExportedDirectory;
165 
166  UPROPERTY(EditAnywhere, AdvancedDisplay, Category = "Capture")
167  uint32 MaxSaveImageAsyncCount;
168 
169 protected: // Transient
170  UPROPERTY(Transient)
171  FString SubFolderName;
172 
173  UPROPERTY(Transient)
174  FString FullOutputDirectoryPath;
175 
176  TUniquePtr<FNVImageExporter_Thread> ImageExporterThread;
177  IImageWrapperModule* ImageWrapperModule;
178 
179  static const FString DefaultDataOutputFolder;
180 };
181 
182 //=================================== UNVSceneDataVisualizer ===================================
183 ///
184 /// NVSceneDataVisualizer - visualize all the captured data (image buffer and object annotation info) using material, UI
185 ///
186 UCLASS(Blueprintable, ClassGroup = (NVIDIA))
187 class NVSCENECAPTURER_API UNVSceneDataVisualizer : public UNVSceneDataHandler
188 {
189  GENERATED_BODY()
190 
191 public:
193 
194  virtual void Init();
195  virtual bool CanHandleMoreData() const override;
196  virtual bool IsHandlingData() const override;
197 
198  /// Handle the pixels data captured from the scene
199  /// @param CapturedPixelData - The scene's pixels data
200  /// @param CapturedFeatureExtractor - The feature extractor which captured the data
201  /// @param CapturedViewpoint - The viewpoint which captured the data
202  /// @param FrameIndex - The frame when the data is captured
203  virtual bool HandleScenePixelsData(const FNVTexturePixelData& CapturedPixelData,
204  UNVSceneFeatureExtractor_PixelData* CapturedFeatureExtractor,
205  UNVSceneCapturerViewpointComponent* CapturedViewpoint,
206  int32 FrameIndex) override;
207 
208  /// Handle the annotation data captured from the scene
209  /// @param CapturedData - The scene's annotated data
210  /// @param CapturedFeatureExtractor - The feature extractor which captured the data
211  /// @param CapturedViewpoint - The viewpoint which captured the data
212  /// @param FrameIndex - The frame when the data is captured
213  virtual bool HandleSceneAnnotationData(const TSharedPtr<FJsonObject>& CapturedData,
214  class UNVSceneFeatureExtractor_AnnotationData* CapturedFeatureExtractor,
215  class UNVSceneCapturerViewpointComponent* CapturedViewpoint,
216  int32 FrameIndex) override;
217 
218  virtual void OnCapturingCompleted() override;
219 
220  TArray<FString> GetVizNameList() const;
221  UTextureRenderTarget2D* GetTexture(const FString& VizName);
222 
223 protected: // Transient
224  /// Map between a feature extractor name and its visualized texture
225  UPROPERTY(Transient)
226  TMap<FString, UTextureRenderTarget2D*> VizTextureMap;
227 };
UNVSceneCapturerViewpointComponent: Represents each viewpoint from where the capturer captures data...
NVSceneDataExporter - export all the captured data (image buffer and object annotation info) to files...
virtual bool HandleSceneAnnotationData(const TSharedPtr< FJsonObject > &CapturedData, class UNVSceneFeatureExtractor_AnnotationData *CapturedFeatureExtractor, class UNVSceneCapturerViewpointComponent *CapturedViewpoint, int32 FrameIndex)
Handle the annotation data captured from the scene.
virtual bool CanHandleMoreData() const
Check whether this handler can process more scene data If it can't then we should stop getting more d...
NVSceneDataVisualizer - visualize all the captured data (image buffer and object annotation info) usi...
Base class for all the feature extractors that capture the scene view in pixel data format...
Base interface for serializing/visualizing captured pixel and annotation data.
virtual bool HandleScenePixelsData(const FNVTexturePixelData &CapturedPixelData, class UNVSceneFeatureExtractor_PixelData *CapturedFeatureExtractor, class UNVSceneCapturerViewpointComponent *CapturedViewpoint, int32 FrameIndex)
Handle the pixels data captured from the scene.