nx_metadata_sdk  1.0
Metadata SDK
device_agent.h
1 // Copyright 2018-present Network Optix, Inc. Licensed under MPL 2.0: www.mozilla.org/MPL/2.0/
2 
3 #pragma once
4 
5 #include <thread>
6 #include <atomic>
7 #include <memory>
8 #include <condition_variable>
9 #include <vector>
10 #include <string>
11 #include <deque>
12 
13 #include <nx/sdk/analytics/helpers/consuming_device_agent.h>
14 #include <nx/sdk/analytics/helpers/pixel_format.h>
15 #include <nx/sdk/analytics/helpers/object_metadata_packet.h>
16 
17 #include "engine.h"
18 #include "objects/random.h"
19 #include "stub_analytics_plugin_ini.h"
20 
21 namespace nx {
22 namespace vms_server_plugins {
23 namespace analytics {
24 namespace stub {
25 
27 {
28 public:
29  DeviceAgent(Engine* engine, const nx::sdk::IDeviceInfo* deviceInfo);
30  virtual ~DeviceAgent() override;
31 
32 protected:
33  virtual void getPluginSideSettings(
34  nx::sdk::Result<const nx::sdk::ISettingsResponse*>* outResult) const override;
35 
36  virtual void doSetNeededMetadataTypes(
37  nx::sdk::Result<void>* outValue,
38  const nx::sdk::analytics::IMetadataTypes* neededMetadataTypes) override;
39 
40  virtual std::string manifestString() const override;
41 
43 
44  virtual bool pushCompressedVideoFrame(
45  const nx::sdk::analytics::ICompressedVideoPacket* videoFrame) override;
46 
47  virtual bool pushUncompressedVideoFrame(
48  const nx::sdk::analytics::IUncompressedVideoFrame* videoFrame) override;
49 
50  virtual bool pushCustomMetadataPacket(
51  const nx::sdk::analytics::ICustomMetadataPacket* customMetadataPacket) override;
52 
53  virtual bool pullMetadataPackets(
54  std::vector<nx::sdk::analytics::IMetadataPacket*>* metadataPackets) override;
55 
56 private:
57  nx::sdk::analytics::IMetadataPacket* cookSomeEvents();
58 
59  std::vector<nx::sdk::analytics::IMetadataPacket*> cookSomeObjects();
60 
61  nx::sdk::Ptr<nx::sdk::analytics::IObjectMetadata> cookBlinkingObjectIfNeeded(
62  int64_t metadataTimestampUs);
63 
64  void addBlinkingObjectIfNeeded(
65  int64_t metadataTimestampUs,
66  std::vector<nx::sdk::analytics::IMetadataPacket*>* metadataPackets,
68 
69  void addFixedObjectIfNeeded(
71 
72  void addCounterIfNeeded(
74 
75  int64_t usSinceEpoch() const;
76 
77  void processVideoFrame(const nx::sdk::analytics::IDataPacket* videoFrame, const char* func);
78 
79  void processCustomMetadataPacket(
80  const nx::sdk::analytics::ICustomMetadataPacket* customMetadataPacket,
81  const char* func);
82 
83  bool checkVideoFrame(const nx::sdk::analytics::IUncompressedVideoFrame* frame) const;
84 
85  bool checkVideoFramePlane(
87  const nx::sdk::analytics::PixelFormatDescriptor* pixelFormatDescriptor,
88  int plane) const;
89 
90  void dumpSomeFrameBytes(
92  int plane) const;
93 
94  void startFetchingMetadata(const nx::sdk::analytics::IMetadataTypes* metadataTypes);
95 
96  void stopFetchingMetadata();
97 
98  void processEvents();
99 
100  void processPluginDiagnosticEvents();
101 
102  void setObjectCount(int objectCount);
103 
104  void cleanUpTimestampQueue();
105 
106  void parseSettings();
107 
108  template<typename ObjectType>
109  void setIsObjectTypeGenerationNeeded(bool isObjectTypeGenerationNeeded)
110  {
111  if (isObjectTypeGenerationNeeded)
112  {
113  m_objectGenerator.registerObjectFactory<ObjectType>(
114  []() { return std::make_unique<ObjectType>(); });
115  }
116  else
117  {
118  m_objectGenerator.unregisterObjectFactory<ObjectType>();
119  }
120  }
121 
122  void updateObjectGenerationParameters();
123 
124  void updateEventGenerationParameters();
125 
126  void updateManifest();
127 
128  void processFrameMotion(
130 
131  std::string capabilities() const;
132 
133 private:
134  Engine* const m_engine;
135 
136  std::atomic<bool> m_terminated{false};
137 
138  std::unique_ptr<std::thread> m_pluginDiagnosticEventThread;
139  std::mutex m_pluginDiagnosticEventGenerationLoopMutex;
140  std::condition_variable m_pluginDiagnosticEventGenerationLoopCondition;
141  std::atomic<bool> m_needToThrowPluginDiagnosticEvents{false};
142 
143  std::unique_ptr<std::thread> m_eventThread;
144  std::mutex m_eventGenerationLoopMutex;
145  std::condition_variable m_eventGenerationLoopCondition;
146  std::atomic<bool> m_eventsNeeded{false};
147 
148  int m_frameCounter = 0;
149  std::string m_eventTypeId;
150 
151  std::deque<int64_t> m_frameTimestampUsQueue;
152  int64_t m_lastVideoFrameTimestampUs = 0;
153  int64_t m_lastBlinkingObjectTimestampUs = 0;
154 
155  struct DeviceAgentSettings
156  {
157  bool needToGenerateObjects() const
158  {
159  return !ini().visualizeMotion
160  && (generateCars
161  || generateTrucks
162  || generatePedestrians
163  || generateHumanFaces
164  || generateBicycles
165  || generateStones
166  || generateFixedObject
167  || generateCounter
168  || blinkingObjectPeriodMs.load() != std::chrono::milliseconds::zero());
169  }
170 
171  std::atomic<bool> generateEvents{true};
172  std::atomic<bool> generateCars{true};
173  std::atomic<bool> generateTrucks{true};
174  std::atomic<bool> generatePedestrians{true};
175  std::atomic<bool> generateHumanFaces{true};
176  std::atomic<bool> generateBicycles{true};
177  std::atomic<bool> generateStones{false};
178  std::atomic<bool> generateFixedObject{false};
179  std::atomic<bool> generateCounter{false};
180 
181  std::atomic<std::chrono::milliseconds> blinkingObjectPeriodMs{
182  std::chrono::milliseconds::zero()};
183 
184  std::atomic<bool> blinkingObjectInDedicatedPacket{false};
185 
186  std::atomic<int> numberOfObjectsToGenerate{1};
187  std::atomic<int> generateObjectsEveryNFrames{1};
188 
189  std::atomic<bool> generatePreviews{true};
190 
191  std::atomic<bool> throwPluginDiagnosticEvents{false};
192  std::atomic<bool> leakFrames{false};
193 
194  std::atomic<std::chrono::milliseconds> additionalFrameProcessingDelayMs{
195  std::chrono::milliseconds::zero()};
196 
197  std::atomic<std::chrono::milliseconds> overallMetadataDelayMs{
198  std::chrono::milliseconds::zero()};
199 
200  std::atomic<int> numberOfFramesBeforePreviewGeneration{30};
201 
202  std::atomic<float> counterBoundingBoxSideSize{0};
203  std::atomic<float> counterBoundingBoxXOffset{0};
204  std::atomic<float> counterBoundingBoxYOffset{0};
205 
206  std::atomic<bool> declareAdditionalEventTypes{false};
207 
208  std::vector<char> previewImage;
209  std::string previewImageFormat;
210  std::string previewImageUrl;
211  };
212 
213  DeviceAgentSettings m_deviceAgentSettings;
214 
215  struct EventContext
216  {
217  int currentEventTypeIndex = 0;
218  bool isCurrentEventActive = false;
219  };
220 
221  EventContext m_eventContext;
222 
223  struct ObjectContext
224  {
225  ObjectContext() = default;
226  ObjectContext(std::unique_ptr<AbstractObject> object): object(std::move(object)) {}
227 
228  ObjectContext& operator=(std::unique_ptr<AbstractObject>&& otherObject)
229  {
230  reset();
231  object = std::move(otherObject);
232  return *this;
233  }
234 
235  void reset()
236  {
237  object.reset();
238  isPreviewGenerated = false;
239  frameCounter = 0;
240  }
241 
242  bool operator!() const { return !object; }
243 
244  std::unique_ptr<AbstractObject> object;
245  bool isPreviewGenerated = false;
246  int frameCounter = 0;
247  };
248 
249  std::mutex m_objectGenerationMutex;
250  RandomObjectGenerator m_objectGenerator;
251  std::vector<ObjectContext> m_objectContexts;
252  int m_counterObjectAttributeValue = 0;
253 };
254 
255 const std::string kLineCrossingEventType = "nx.stub.lineCrossing";
256 const std::string kObjectInTheAreaEventType = "nx.stub.objectInTheArea";
257 const std::string kLoiteringEventType = "nx.stub.loitering";
258 const std::string kIntrusionEventType = "nx.stub.intrusion";
259 const std::string kGunshotEventType = "nx.stub.gunshot";
260 const std::string kSuspiciousNoiseEventType = "nx.stub.suspiciousNoise";
261 const std::string kSoundRelatedEventGroup = "nx.stub.soundRelated";
262 const std::string kBlinkingObjectType = "nx.stub.blinkingObject";
263 const std::string kFixedObjectType = "nx.stub.fixedObject";
264 const std::string kCounterObjectType = "nx.stub.counter";
265 
266 } // namespace stub
267 } // namespace analytics
268 } // namespace vms_server_plugins
269 } // namespace nx
virtual bool pushCompressedVideoFrame(const nx::sdk::analytics::ICompressedVideoPacket *videoFrame) override
Definition: device_agent.cpp:283
Definition: i_metadata_packet.h:16
virtual bool pushUncompressedVideoFrame(const nx::sdk::analytics::IUncompressedVideoFrame *videoFrame) override
Definition: device_agent.cpp:299
Definition: i_compressed_video_packet.h:34
Definition: ptr.h:18
virtual nx::sdk::Result< const nx::sdk::ISettingsResponse * > settingsReceived() override
Definition: device_agent.cpp:213
virtual bool pushCustomMetadataPacket(const nx::sdk::analytics::ICustomMetadataPacket *customMetadataPacket) override
Definition: device_agent.cpp:315
virtual std::string manifestString() const override
Definition: device_agent.cpp:146
Definition: consuming_device_agent.h:38
Definition: i_metadata_types.h:13
Definition: result.h:81
Definition: i_device_info.h:14
static TestIni & ini()
Definition: ini_config_ut.cpp:43
Definition: result.h:47
Definition: apple_utils.h:6
Definition: i_custom_metadata_packet.h:16
Definition: pixel_format.h:26
Definition: i_uncompressed_video_frame.h:73
Definition: i_data_packet.h:16
Definition: i_list.h:9