nx_metadata_sdk  1.0
Metadata SDK
device_agent.h
1 // Copyright 2018-present Network Optix, Inc. Licensed under MPL 2.0: www.mozilla.org/MPL/2.0/
2 
3 #pragma once
4 
5 #include <thread>
6 #include <atomic>
7 #include <memory>
8 #include <condition_variable>
9 #include <vector>
10 #include <string>
11 #include <deque>
12 
13 #include <nx/sdk/analytics/helpers/consuming_device_agent.h>
14 #include <nx/sdk/analytics/helpers/pixel_format.h>
15 #include <nx/sdk/analytics/helpers/object_metadata_packet.h>
16 
17 #include "engine.h"
18 #include "objects/random.h"
19 #include "stub_analytics_plugin_ini.h"
20 #include "modules/standard_taxonomy/module.h"
21 
22 namespace nx {
23 namespace vms_server_plugins {
24 namespace analytics {
25 namespace stub {
26 
28 {
29 public:
30  DeviceAgent(Engine* engine, const nx::sdk::IDeviceInfo* deviceInfo);
31  virtual ~DeviceAgent() override;
32 
33 protected:
34  virtual void getPluginSideSettings(
35  nx::sdk::Result<const nx::sdk::ISettingsResponse*>* outResult) const override;
36 
37  virtual void doSetNeededMetadataTypes(
38  nx::sdk::Result<void>* outValue,
39  const nx::sdk::analytics::IMetadataTypes* neededMetadataTypes) override;
40 
41  virtual std::string manifestString() const override;
42 
44 
45  virtual bool pushCompressedVideoFrame(
46  const nx::sdk::analytics::ICompressedVideoPacket* videoFrame) override;
47 
48  virtual bool pushUncompressedVideoFrame(
49  const nx::sdk::analytics::IUncompressedVideoFrame* videoFrame) override;
50 
51  virtual bool pushCustomMetadataPacket(
52  const nx::sdk::analytics::ICustomMetadataPacket* customMetadataPacket) override;
53 
54  virtual bool pullMetadataPackets(
55  std::vector<nx::sdk::analytics::IMetadataPacket*>* metadataPackets) override;
56 
57 private:
58  nx::sdk::analytics::IMetadataPacket* cookSomeEvents();
59 
60  std::vector<nx::sdk::analytics::IMetadataPacket*> cookSomeObjects();
61 
62  nx::sdk::Ptr<nx::sdk::analytics::IObjectMetadata> cookBlinkingObjectIfNeeded(
63  int64_t metadataTimestampUs);
64 
65  void addBlinkingObjectIfNeeded(
66  int64_t metadataTimestampUs,
67  std::vector<nx::sdk::analytics::IMetadataPacket*>* metadataPackets,
69 
70  void addFixedObjectIfNeeded(
72 
73  void addCounterIfNeeded(
75 
76  int64_t usSinceEpoch() const;
77 
78  void processVideoFrame(const nx::sdk::analytics::IDataPacket* videoFrame, const char* func);
79 
80  void processCustomMetadataPacket(
81  const nx::sdk::analytics::ICustomMetadataPacket* customMetadataPacket,
82  const char* func);
83 
84  bool checkVideoFrame(const nx::sdk::analytics::IUncompressedVideoFrame* frame) const;
85 
86  bool checkVideoFramePlane(
88  const nx::sdk::analytics::PixelFormatDescriptor* pixelFormatDescriptor,
89  int plane) const;
90 
91  void dumpSomeFrameBytes(
93  int plane) const;
94 
95  void startFetchingMetadata(const nx::sdk::analytics::IMetadataTypes* metadataTypes);
96 
97  void stopFetchingMetadata();
98 
99  void processEvents();
100 
101  void processPluginDiagnosticEvents();
102 
103  void setObjectCount(int objectCount);
104 
105  void cleanUpTimestampQueue();
106 
107  void parseSettings();
108 
109  template<typename ObjectType>
110  void setIsObjectTypeGenerationNeeded(bool isObjectTypeGenerationNeeded)
111  {
112  if (isObjectTypeGenerationNeeded)
113  {
114  m_objectGenerator.registerObjectFactory<ObjectType>(
115  []() { return std::make_unique<ObjectType>(); });
116  }
117  else
118  {
119  m_objectGenerator.unregisterObjectFactory<ObjectType>();
120  }
121  }
122 
123  void updateObjectGenerationParameters();
124 
125  void updateEventGenerationParameters();
126 
127  void updateManifest();
128 
129  void processFrameMotion(
131 
132  std::string capabilities() const;
133 
134 private:
135  Engine* const m_engine;
136 
137  std::atomic<bool> m_terminated{false};
138 
139  std::unique_ptr<std::thread> m_pluginDiagnosticEventThread;
140  std::mutex m_pluginDiagnosticEventGenerationLoopMutex;
141  std::condition_variable m_pluginDiagnosticEventGenerationLoopCondition;
142  std::atomic<bool> m_needToThrowPluginDiagnosticEvents{false};
143 
144  std::unique_ptr<std::thread> m_eventThread;
145  std::mutex m_eventGenerationLoopMutex;
146  std::condition_variable m_eventGenerationLoopCondition;
147  std::atomic<bool> m_eventsNeeded{false};
148 
149  int m_frameCounter = 0;
150  std::string m_eventTypeId;
151 
152  std::deque<int64_t> m_frameTimestampUsQueue;
153  int64_t m_lastVideoFrameTimestampUs = 0;
154  int64_t m_lastBlinkingObjectTimestampUs = 0;
155 
156  struct DeviceAgentSettings
157  {
158  bool needToGenerateObjects() const
159  {
160  return !ini().visualizeMotion
161  && (generateCars
162  || generateTrucks
163  || generatePedestrians
164  || generateHumanFaces
165  || generateBicycles
166  || generateStones
167  || generateFixedObject
168  || generateCounter
169  || blinkingObjectPeriodMs.load() != std::chrono::milliseconds::zero());
170  }
171 
172  std::atomic<bool> generateEvents{true};
173  std::atomic<bool> generateCars{true};
174  std::atomic<bool> generateTrucks{true};
175  std::atomic<bool> generatePedestrians{true};
176  std::atomic<bool> generateHumanFaces{true};
177  std::atomic<bool> generateBicycles{true};
178  std::atomic<bool> generateStones{false};
179  std::atomic<bool> generateFixedObject{false};
180 
181  std::mutex fixedObjectColorMutex;
182  std::string fixedObjectColor;
183 
184  std::atomic<bool> generateCounter{false};
185 
186  std::atomic<std::chrono::milliseconds> blinkingObjectPeriodMs{
187  std::chrono::milliseconds::zero()};
188 
189  std::atomic<bool> blinkingObjectInDedicatedPacket{false};
190 
191  std::atomic<int> numberOfObjectsToGenerate{1};
192  std::atomic<int> generateObjectsEveryNFrames{1};
193 
194  std::atomic<bool> generatePreviews{true};
195 
196  std::atomic<bool> throwPluginDiagnosticEvents{false};
197  std::atomic<bool> leakFrames{false};
198 
199  std::atomic<std::chrono::milliseconds> additionalFrameProcessingDelayMs{
200  std::chrono::milliseconds::zero()};
201 
202  std::atomic<std::chrono::milliseconds> overallMetadataDelayMs{
203  std::chrono::milliseconds::zero()};
204 
205  std::atomic<int> numberOfFramesBeforePreviewGeneration{30};
206 
207  std::atomic<float> counterBoundingBoxSideSize{0};
208  std::atomic<float> counterBoundingBoxXOffset{0};
209  std::atomic<float> counterBoundingBoxYOffset{0};
210 
211  std::atomic<bool> declareAdditionalEventTypes{false};
212 
213  std::vector<char> previewImage;
214  std::string previewImageFormat;
215  std::string previewImageUrl;
216  };
217 
218  DeviceAgentSettings m_deviceAgentSettings;
219 
220  struct EventContext
221  {
222  int currentEventTypeIndex = 0;
223  };
224 
225  EventContext m_eventContext;
226 
227  struct ObjectContext
228  {
229  ObjectContext() = default;
230  ObjectContext(std::unique_ptr<AbstractObject> object): object(std::move(object)) {}
231 
232  ObjectContext& operator=(std::unique_ptr<AbstractObject>&& otherObject)
233  {
234  reset();
235  object = std::move(otherObject);
236  return *this;
237  }
238 
239  void reset()
240  {
241  object.reset();
242  isPreviewGenerated = false;
243  frameCounter = 0;
244  }
245 
246  bool operator!() const { return !object; }
247 
248  std::unique_ptr<AbstractObject> object;
249  bool isPreviewGenerated = false;
250  int frameCounter = 0;
251  };
252 
253  std::mutex m_objectGenerationMutex;
254  RandomObjectGenerator m_objectGenerator;
255  std::vector<ObjectContext> m_objectContexts;
256  int m_counterObjectAttributeValue = 0;
257 
258  modules::standard_taxonomy::Module m_standardTaxonomyModule;
259 };
260 
261 const std::string kLineCrossingEventType = "nx.stub.lineCrossing";
262 const std::string kObjectInTheAreaEventType = "nx.stub.objectInTheArea";
263 const std::string kLoiteringEventType = "nx.stub.loitering";
264 const std::string kIntrusionEventType = "nx.stub.intrusion";
265 const std::string kGunshotEventType = "nx.stub.gunshot";
266 const std::string kSuspiciousNoiseEventType = "nx.stub.suspiciousNoise";
267 const std::string kSoundRelatedEventGroup = "nx.stub.soundRelated";
268 const std::string kBlinkingObjectType = "nx.stub.blinkingObject";
269 const std::string kFixedObjectType = "nx.stub.fixedObject";
270 const std::string kCounterObjectType = "nx.stub.counter";
271 const std::string kAdditionalEventType = "nx.stub.additionalEvent1";
272 const std::string kAdditionalEventType2 = "nx.stub.additionalEvent2";
273 
274 } // namespace stub
275 } // namespace analytics
276 } // namespace vms_server_plugins
277 } // namespace nx
virtual bool pushCompressedVideoFrame(const nx::sdk::analytics::ICompressedVideoPacket *videoFrame) override
Definition: device_agent.cpp:383
Definition: i_metadata_packet.h:16
virtual bool pushUncompressedVideoFrame(const nx::sdk::analytics::IUncompressedVideoFrame *videoFrame) override
Definition: device_agent.cpp:399
Definition: i_compressed_video_packet.h:34
Definition: ptr.h:18
virtual nx::sdk::Result< const nx::sdk::ISettingsResponse * > settingsReceived() override
Definition: device_agent.cpp:303
virtual bool pushCustomMetadataPacket(const nx::sdk::analytics::ICustomMetadataPacket *customMetadataPacket) override
Definition: device_agent.cpp:415
virtual std::string manifestString() const override
Definition: device_agent.cpp:194
Definition: consuming_device_agent.h:38
Definition: i_metadata_types.h:13
Definition: result.h:81
Definition: i_device_info.h:14
static TestIni & ini()
Definition: ini_config_ut.cpp:72
Definition: result.h:47
Definition: apple_utils.h:6
Definition: i_custom_metadata_packet.h:16
Definition: pixel_format.h:26
Definition: i_uncompressed_video_frame.h:73
Definition: i_data_packet.h:17
Definition: i_list.h:9