nx_metadata_sdk  1.0
Metadata SDK
device_agent.h
1 // Copyright 2018-present Network Optix, Inc. Licensed under MPL 2.0: www.mozilla.org/MPL/2.0/
2 
3 #pragma once
4 
5 #include <thread>
6 #include <atomic>
7 #include <memory>
8 #include <condition_variable>
9 #include <vector>
10 #include <string>
11 #include <deque>
12 
13 #include <nx/sdk/analytics/helpers/video_frame_processing_device_agent.h>
14 #include <nx/sdk/analytics/helpers/pixel_format.h>
15 #include <nx/sdk/analytics/helpers/object_metadata_packet.h>
16 
17 #include "engine.h"
18 #include "objects/random.h"
19 
20 namespace nx {
21 namespace vms_server_plugins {
22 namespace analytics {
23 namespace stub {
24 
26 {
27 public:
28  DeviceAgent(Engine* engine, const nx::sdk::IDeviceInfo* deviceInfo);
29  virtual ~DeviceAgent() override;
30 
31 protected:
32  virtual void getPluginSideSettings(
33  nx::sdk::Result<const nx::sdk::ISettingsResponse*>* outResult) const override;
34 
35  virtual void doSetNeededMetadataTypes(
36  nx::sdk::Result<void>* outValue,
37  const nx::sdk::analytics::IMetadataTypes* neededMetadataTypes) override;
38 
39  virtual std::string manifestString() const override;
40 
42 
43  virtual bool pushCompressedVideoFrame(
44  const nx::sdk::analytics::ICompressedVideoPacket* videoFrame) override;
45 
46  virtual bool pushUncompressedVideoFrame(
47  const nx::sdk::analytics::IUncompressedVideoFrame* videoFrame) override;
48 
49  virtual bool pullMetadataPackets(
50  std::vector<nx::sdk::analytics::IMetadataPacket*>* metadataPackets) override;
51 
52 private:
53  nx::sdk::analytics::IMetadataPacket* cookSomeEvents();
54 
55  std::vector<nx::sdk::analytics::IMetadataPacket*> cookSomeObjects();
56 
57  nx::sdk::Ptr<nx::sdk::analytics::IObjectMetadata> cookBlinkingObjectIfNeeded(
58  int64_t metadataTimestampUs);
59 
60  void addBlinkingObjectIfNeeded(
61  int64_t metadataTimestampUs,
62  std::vector<nx::sdk::analytics::IMetadataPacket*>* metadataPackets,
64 
65  int64_t usSinceEpoch() const;
66 
67  void processVideoFrame(const nx::sdk::analytics::IDataPacket* videoFrame, const char* func);
68 
69  bool checkVideoFrame(const nx::sdk::analytics::IUncompressedVideoFrame* frame) const;
70 
71  bool checkVideoFramePlane(
73  const nx::sdk::analytics::PixelFormatDescriptor* pixelFormatDescriptor,
74  int plane) const;
75 
76  void dumpSomeFrameBytes(
78  int plane) const;
79 
80  void startFetchingMetadata(const nx::sdk::analytics::IMetadataTypes* metadataTypes);
81 
82  void stopFetchingMetadata();
83 
84  void processEvents();
85 
86  void processPluginDiagnosticEvents();
87 
88  void setObjectCount(int objectCount);
89 
90  void cleanUpTimestampQueue();
91 
92  void parseSettings();
93 
94  template<typename ObjectType>
95  void setIsObjectTypeGenerationNeeded(bool isObjectTypeGenerationNeeded)
96  {
97  if (isObjectTypeGenerationNeeded)
98  {
99  m_objectGenerator.registerObjectFactory<ObjectType>(
100  []() { return std::make_unique<ObjectType>(); });
101  }
102  else
103  {
104  m_objectGenerator.unregisterObjectFactory<ObjectType>();
105  }
106  }
107 
108  void updateObjectGenerationParameters();
109 
110  void updateEventGenerationParameters();
111 
112 private:
113  Engine* const m_engine;
114 
115  std::atomic<bool> m_terminated{false};
116 
117  std::unique_ptr<std::thread> m_pluginDiagnosticEventThread;
118  std::mutex m_pluginDiagnosticEventGenerationLoopMutex;
119  std::condition_variable m_pluginDiagnosticEventGenerationLoopCondition;
120  std::atomic<bool> m_needToThrowPluginDiagnosticEvents{false};
121 
122  std::unique_ptr<std::thread> m_eventThread;
123  std::mutex m_eventGenerationLoopMutex;
124  std::condition_variable m_eventGenerationLoopCondition;
125  std::atomic<bool> m_eventsNeeded{false};
126 
127  int m_frameCounter = 0;
128  std::string m_eventTypeId;
129 
130  std::deque<int64_t> m_frameTimestampUsQueue;
131  int64_t m_lastVideoFrameTimestampUs = 0;
132  int64_t m_lastBlinkingObjectTimestampUs = 0;
133 
134  struct DeviceAgentSettings
135  {
136  bool needToGenerateObjects() const
137  {
138  return generateCars
139  || generateTrucks
140  || generatePedestrians
141  || generateHumanFaces
142  || generateBicycles
143  || blinkingObjectPeriodMs.load() != std::chrono::milliseconds::zero();
144  }
145 
146  std::atomic<bool> generateEvents{true};
147 
148  std::atomic<bool> generateCars{true};
149  std::atomic<bool> generateTrucks{true};
150  std::atomic<bool> generatePedestrians{true};
151  std::atomic<bool> generateHumanFaces{true};
152  std::atomic<bool> generateBicycles{true};
153 
154  std::atomic<std::chrono::milliseconds> blinkingObjectPeriodMs{
155  std::chrono::milliseconds::zero()};
156 
157  std::atomic<bool> blinkingObjectInDedicatedPacket{false};
158 
159  std::atomic<int> numberOfObjectsToGenerate{1};
160  std::atomic<int> generateObjectsEveryNFrames{1};
161 
162  std::atomic<bool> generatePreviews{true};
163 
164  std::atomic<bool> throwPluginDiagnosticEvents{false};
165  std::atomic<bool> leakFrames{false};
166 
167  std::atomic<std::chrono::milliseconds> additionalFrameProcessingDelayMs{
168  std::chrono::milliseconds::zero()};
169 
170  std::atomic<std::chrono::milliseconds> overallMetadataDelayMs{
171  std::chrono::milliseconds::zero()};
172 
173  std::atomic<int> numberOfFramesBeforePreviewGeneration{30};
174  };
175 
176  DeviceAgentSettings m_deviceAgentSettings;
177 
178  struct EventContext
179  {
180  int currentEventTypeIndex = 0;
181  bool isCurrentEventActive = false;
182  };
183 
184  EventContext m_eventContext;
185 
186  struct ObjectContext
187  {
188  ObjectContext() = default;
189  ObjectContext(std::unique_ptr<AbstractObject> object): object(std::move(object)) {}
190 
191  ObjectContext& operator=(std::unique_ptr<AbstractObject>&& otherObject)
192  {
193  reset();
194  object = std::move(otherObject);
195  return *this;
196  }
197 
198  void reset()
199  {
200  object.reset();
201  isPreviewGenerated = false;
202  frameCounter = 0;
203  }
204 
205  bool operator!() const { return !object; }
206 
207  std::unique_ptr<AbstractObject> object;
208  bool isPreviewGenerated = false;
209  int frameCounter = 0;
210  };
211 
212  std::mutex m_objectGenerationMutex;
213  RandomObjectGenerator m_objectGenerator;
214  std::vector<ObjectContext> m_objectContexts;
215 };
216 
217 const std::string kLineCrossingEventType = "nx.stub.lineCrossing";
218 const std::string kObjectInTheAreaEventType = "nx.stub.objectInTheArea";
219 const std::string kLoiteringEventType = "nx.stub.loitering";
220 const std::string kIntrusionEventType = "nx.stub.intrusion";
221 const std::string kGunshotEventType = "nx.stub.gunshot";
222 const std::string kSuspiciousNoiseEventType = "nx.stub.suspiciousNoise";
223 const std::string kSoundRelatedEventGroup = "nx.stub.soundRelatedEvent";
224 const std::string kBlinkingObjectType = "nx.stub.blinkingObject";
225 
226 } // namespace stub
227 } // namespace analytics
228 } // namespace vms_server_plugins
229 } // namespace nx
virtual bool pushCompressedVideoFrame(const nx::sdk::analytics::ICompressedVideoPacket *videoFrame) override
Definition: device_agent.cpp:230
virtual nx::sdk::Result< const nx::sdk::IStringMap * > settingsReceived() override
Definition: device_agent.cpp:186
Definition: i_metadata_packet.h:16
virtual bool pushUncompressedVideoFrame(const nx::sdk::analytics::IUncompressedVideoFrame *videoFrame) override
Definition: device_agent.cpp:242
Definition: i_compressed_video_packet.h:16
Definition: ptr.h:18
virtual std::string manifestString() const override
Definition: device_agent.cpp:136
Definition: i_metadata_types.h:13
Definition: result.h:81
Definition: i_device_info.h:14
Definition: result.h:47
Definition: apple_utils.h:6
Definition: video_frame_processing_device_agent.h:37
Definition: pixel_format.h:26
Definition: i_uncompressed_video_frame.h:13
Definition: i_data_packet.h:16