12 #include "NVSceneCapturerModule.h"
13 #include "IImageWrapper.h"
14 #include "Runtime/Core/Public/Misc/FileHelper.h"
15 #include "Kismet/BlueprintFunctionLibrary.h"
16 #include "Serialization/JsonSerializerMacros.h"
17 #include "Serialization/JsonTypes.h"
19 #include "SharedPointer.h"
20 #include "Engine/TextureRenderTarget2D.h"
21 #include "JsonObjectConverter.h"
23 #include "NVCameraSettings.h"
24 #include "NVSceneCapturerUtils.generated.h"
28 enum class ENVImageFormat : uint8
31 PNG UMETA(DisplayName =
"PNG (Portable Network Graphics)."),
34 JPEG UMETA(DisplayName =
"JPEG (Joint Photographic Experts Group)."),
37 GrayscaleJPEG UMETA(DisplayName =
"GrayscaleJPEG (Single channel jpeg"),
40 BMP UMETA(DisplayName =
"BMP (Windows Bitmap"),
47 NVImageFormat_MAX UMETA(Hidden)
50 EImageFormat ConvertExportFormatToImageFormat(ENVImageFormat ExportFormat);
51 FString GetExportImageExtension(ENVImageFormat ExportFormat);
55 enum ENVCapturedPixelFormat
74 NVCapturedPixelFormat_MAX UMETA(Hidden)
77 ETextureRenderTargetFormat ConvertCapturedFormatToRenderTargetFormat(ENVCapturedPixelFormat PixelFormat);
86 TArray<uint8> PixelData;
88 EPixelFormat PixelFormat;
106 FVector2D SocketLocation;
113 enum class ENVCuboidVertexType : uint8
124 CuboidVertexType_MAX UMETA(Hidden)
127 USTRUCT(BlueprintType)
135 FVector Vertexes[(uint8)ENVCuboidVertexType::CuboidVertexType_MAX];
137 static uint8 TotalVertexesCount;
141 FNVCuboidData(const FBox& AABB);
142 FNVCuboidData(const FBox& LocalBox, const FTransform& LocalTransform);
144 void BuildFromAABB(const FBox& AABB);
145 void BuildFromOOBB(const FBox& OOBB, const FTransform& LocalTransform);
147 FVector GetCenter()
const
151 FVector GetVertex(ENVCuboidVertexType VertexType)
const
153 return Vertexes[(uint8)VertexType];
155 FVector GetExtent()
const
157 return LocalBox.GetExtent();
159 FVector GetDimension()
const
161 return LocalBox.GetExtent() * 2.f;
163 FVector GetDirection()
const
165 return Rotation.Vector();
167 FQuat GetRotation()
const
173 return (LocalBox.IsValid != 0);
193 FNVBox2D(
const FBox2D& box = FBox2D())
195 top_left = FVector2D(box.Min.Y, box.Min.X);
196 bottom_right = FVector2D(box.Max.Y, box.Max.X);
204 FVector2D bottom_right;
235 FVector dimensions_worldspace;
238 FVector location_worldspace;
244 FRotator rotation_worldspace;
247 FQuat quaternion_worldspace;
253 FQuat quaternion_xyzw;
256 FMatrix actor_to_world_matrix_ue4;
259 FMatrix actor_to_world_matrix_opencv;
262 FMatrix actor_to_camera_matrix;
265 FMatrix pose_transform;
268 FVector bounding_box_center_worldspace;
271 FVector cuboid_centroid;
274 FVector2D projected_cuboid_centroid;
277 FVector bounding_box_forward_direction;
280 FVector2D bounding_box_forward_direction_imagespace;
283 float viewpoint_azimuth_angle;
286 float viewpoint_altitude_angle;
289 float distance_scale;
296 TArray<FVector> cuboid;
299 TArray<FVector2D> projected_cuboid;
304 TSharedPtr<FJsonObject> custom_data;
315 FVector location_worldframe;
318 FQuat quaternion_xyzw_worldframe;
321 FMatrix ProjectionMatrix;
324 FMatrix ViewProjectionMatrix;
345 TArray<FCapturedObjectData> Objects;
359 TArray<FColor> SceneBitmap;
363 UCLASS(ClassGroup = (NVIDIA), meta = (BlueprintSpawnableComponent))
364 class NVSCENECAPTURER_API UNVCapturableActorTag : public UActorComponent
368 UNVCapturableActorTag() : bIncludeMe(true) {};
370 bool IsValid()
const {
return bIncludeMe && !Tag.IsEmpty(); }
373 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category =
"Config")
376 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Config")
381 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Config")
382 bool bExportAllMeshSocketInfo;
385 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Config", meta = (editcondition = "!bExportAllMeshSocketInfo"))
386 TArray<FName> SocketNameToExportList;
390 enum class ENVIncludeObjects : uint8
397 enum class ENVBoundsGenerationType : uint8
400 VE_AABB UMETA(DisplayName =
"AABB"),
402 VE_OOBB UMETA(DisplayName =
"OOBB"),
405 VE_TightOOBB UMETA(DisplayName =
"Tight Arbitrary OOBB")
409 enum class ENVBoundBox2dGenerationType : uint8
415 FromMeshBodyCollision,
418 USTRUCT(BlueprintType)
424 FNVSceneExporterConfig();
427 UPROPERTY(EditAnywhere, Category =
"Export")
428 bool bExportObjectData;
430 UPROPERTY(EditAnywhere, Category =
"Export")
431 bool bExportScreenShot;
433 UPROPERTY(EditAnywhere, Category =
"Export")
434 ENVIncludeObjects IncludeObjectsType;
438 UPROPERTY(EditAnywhere, Category =
"Export")
439 bool bIgnoreHiddenActor;
442 UPROPERTY(EditAnywhere, Category = "Export")
443 ENVBoundsGenerationType BoundsType;
446 UPROPERTY(EditAnywhere, Category = "Export")
447 ENVBoundBox2dGenerationType BoundingBox2dType;
449 UPROPERTY(EditAnywhere, Category = "Export")
450 bool bOutputEvenIfNoObjectsAreInView;
453 UPROPERTY(EditAnywhere, Category = "Export")
454 FFloatInterval DistanceScaleRange;
457 USTRUCT(BlueprintType)
463 FNVSceneCapturerSettings();
465 float GetFOVAngle()
const;
466 void RandomizeSettings();
469 #if WITH_EDITORONLY_DATA
470 void PostEditChangeProperty(
struct FPropertyChangedEvent& PropertyChangedEvent);
471 #endif //WITH_EDITORONLY_DATA
474 UPROPERTY(VisibleAnywhere, Category = CapturerSettings)
475 ENVImageFormat ExportImageFormat;
477 UPROPERTY(Transient, VisibleAnywhere, Category = CapturerSettings, meta = (EditCondition =
"!bUseExplicitCameraIntrinsic"))
480 UPROPERTY(EditAnywhere, Category = CapturerSettings, meta = (DisplayName =
"Field of View", UIMin =
"5.0", UIMax =
"170", ClampMin =
"0.001", ClampMax =
"360.0", EditCondition =
"!bUseExplicitCameraIntrinsic"))
481 FFloatInterval FOVAngleRange;
483 UPROPERTY(EditAnywhere, Category = CapturerSettings)
487 UPROPERTY(EditAnywhere, AdvancedDisplay, Category = CapturerSettings)
488 int32 MaxSaveImageAsyncTaskCount;
490 UPROPERTY(EditAnywhere, Category = CapturerSettings)
491 bool bUseExplicitCameraIntrinsic;
493 UPROPERTY(EditAnywhere, Category = CapturerSettings, meta = (EditCondition=bUseExplicitCameraIntrinsic))
496 UPROPERTY(VisibleAnywhere, AdvancedDisplay, Category = CapturerSettings)
497 FMatrix CameraIntrinsicMatrix;
499 UPROPERTY(VisibleAnywhere, AdvancedDisplay, Category = CapturerSettings)
500 FMatrix CameraProjectionMatrix;
504 enum class ENVSceneCapturerState : uint8
506 NotActive UMETA(DisplayName =
"NotActive. The capturer is not active."),
507 Active UMETA(DisplayName =
"Active. The capturer is active. but not started."),
508 Running UMETA(DisplayName =
"Running. The capturer is running/exporting."),
509 Paused UMETA(DisplayName =
"Paused. The capturer is paused, can be resumed."),
510 Completed UMETA(DisplayName =
"Completed. The capturer finished exporting a batch."),
512 NVSceneCapturerState_MAX UMETA(Hidden)
518 extern const FString NotStarted_Str;
519 extern const FString Running_Str;
520 extern const FString Paused_Str;
521 extern const FString Completed_Str;
523 NVSCENECAPTURER_API FString ConvertExporterStateToString(ENVSceneCapturerState ExporterState);
526 USTRUCT(BlueprintType)
538 int32 GetTotalFrameCount()
const
540 return TotalFrameCount;
544 void IncreaseFrameCount(
int AdditionalFrameCount = 1);
545 void SetFrameCount(
int NewFrameCount);
546 void AddFrameDuration(
float NewDuration,
bool bIncreaseFrame =
false);
551 int32 TotalFrameCount;
553 int FPSAccumulatedFrames;
554 float FPSAccumulatedDuration;
559 extern const FMatrix UE4ToOpenCVMatrix;
560 extern const FMatrix OpenCVToUE4Matrix;
561 extern const FMatrix ObjToUE4Matrix;
562 extern const uint32 MaxVertexColorID;
566 NVSCENECAPTURER_API FQuat ConvertQuaternionToOpenCVCoordinateSystem(
const FQuat& InQuat);
567 NVSCENECAPTURER_API FVector ConvertDimensionToOpenCVCoordinateSystem(
const FVector& InDimension);
569 NVSCENECAPTURER_API FString GetGameDataOutputFolder();
570 NVSCENECAPTURER_API FString GetDefaultDataOutputFolder();
571 NVSCENECAPTURER_API FString GetOutputFileFullPath(uint32 Index, FString Extension,
const FString& Subfolder, FString Filename = TEXT(
""), uint8 ZeroPad = 6);
573 NVSCENECAPTURER_API TSharedPtr<FJsonValue> CustomPropertyToJsonValueFunc(UProperty* PropertyType,
const void* Value);
576 NVSCENECAPTURER_API TSharedPtr<FJsonValue> CustomPropertyToJsonValueFunc(UProperty* PropertyType,
const void* Value);
578 template<
typename InStructType> TSharedPtr<FJsonObject> UStructToJsonObject(
const InStructType& InStructData, int64 CheckFlags=0, int64 SkipFlags=0)
580 FJsonObjectConverter::CustomExportCallback CustomPropertyToJsonValue;
581 if (!CustomPropertyToJsonValue.IsBound())
583 CustomPropertyToJsonValue.BindStatic(&NVSceneCapturerUtils::CustomPropertyToJsonValueFunc);
586 TSharedPtr<FJsonObject> JsonObj = FJsonObjectConverter::UStructToJsonObject(InStructData, CheckFlags, SkipFlags, &CustomPropertyToJsonValue);
590 NVSCENECAPTURER_API
bool SaveJsonObjectToFile(
const TSharedPtr<FJsonObject>& JsonObjData,
const FString& Filename);
592 NVSCENECAPTURER_API FString GetExportImageExtension(EImageFormat ImageFormat);
594 NVSCENECAPTURER_API UMeshComponent* GetFirstValidMeshComponent(
const AActor* CheckActor);
596 NVSCENECAPTURER_API TArray<FVector> GetSimpleCollisionVertexes(
const class UMeshComponent* MeshComp);
599 NVSCENECAPTURER_API uint8 GetBitCountPerChannel(EPixelFormat PixelFormat);
601 NVSCENECAPTURER_API uint8 GetColorChannelCount(EPixelFormat PixelFormat);
602 NVSCENECAPTURER_API uint8 GetPixelByteSize(EPixelFormat PixelFormat);
604 NVSCENECAPTURER_API FColor ConvertByteIndexToColor(uint8 Index);
605 NVSCENECAPTURER_API FColor ConvertInt32ToRGB(uint32 Value);
606 NVSCENECAPTURER_API FColor ConvertInt32ToRGBA(uint32 Value);
607 NVSCENECAPTURER_API FColor ConvertInt32ToVertexColor(uint32 Value);
610 NVSCENECAPTURER_API
void SetMeshVertexColor(AActor* MeshOwnerActor,
const FColor& VertexColor);
611 NVSCENECAPTURER_API
void ClearMeshVertexColor(AActor* MeshOwnerActor);
616 NVSCENECAPTURER_API
void CalculateSphericalCoordinate(
const FVector& TargetLocation,
const FVector& SourceLocation,
const FVector& ForwardDirection,
617 float& OutTargetAzimuthAngle,
float& OutTargetAltitudeAngle);
621 NVSCENECAPTURER_API
FNVCuboidData GetMeshCuboid_AABB(
const class UMeshComponent* MeshComp);
627 NVSCENECAPTURER_API
FNVCuboidData GetMeshCuboid_OOBB_Simple(
const class UMeshComponent* MeshComp,
bool bInWorldSpace =
true,
bool bCheckMeshCollision =
true);
632 NVSCENECAPTURER_API
FNVCuboidData GetMeshCuboid_OOBB_Complex(
const class UMeshComponent* MeshComp);
634 NVSCENECAPTURER_API
FNVCuboidData GetActorCuboid_AABB(
const AActor* CheckActor);
635 NVSCENECAPTURER_API
FNVCuboidData GetActorCuboid_OOBB_Simple(
const AActor* CheckActor,
bool bCheckMeshCollision =
true);
636 NVSCENECAPTURER_API
FNVCuboidData GetActorCuboid_OOBB_Complex(
const AActor* CheckActor);
Name of parameters that used in message format FText.
Data to be captured and exported for each socket.