C++ API minimal examples¶
Video processing¶
This example shows how to initialize the video intelligence class and process a video frame.
#include <cstdint>
#include <cstdio>
#include <vector>
#include "plumerai/video_intelligence.h"
int main() {
// Settings, to be changed as needed
constexpr int width = 1600; // camera image width in pixels
constexpr int height = 1200; // camera image height in pixels
constexpr auto image_format = plumerai::ImageFormat::PACKED_RGB888;
// Initialize the video intelligence algorithm
auto pvi = plumerai::VideoIntelligence(height, width);
// Set whether the video stream is night mode (IR) or not.
auto error_code = pvi.set_night_mode(false);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Loop over frames in a video stream (example: 10 frames)
for (int t = 0; t < 10; ++t) {
// Some example input here, normally this is where camera data is acquired
auto image = std::vector<std::uint8_t>(height * width * 3); // 3 for RGB
// Duration between the *previous* frame passed to `process_frame` and the
// *current* frame, in seconds.
//
// Here we assume a fixed capture rate of 30 fps, so delta_t = 1 / 30.
// If your camera runs at a different or variable frame rate, be sure to
// update `delta_t` accordingly. The function `process_frame` relies on this
// value to keep motion tracking and temporal filters in sync.
const float delta_t = 1.f / 30.f;
// Process the frame
error_code = pvi.process_frame(
plumerai::ImagePointer<image_format>(image.data()), delta_t);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
std::vector<BoxPrediction> predictions;
error_code = pvi.object_detection().get_detections(predictions);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Display the results to stdout
for (auto &p : predictions) {
printf("Box class %d with @ (x,y) -> (%.2f,%.2f) till (%.2f,%.2f)\n",
p.class_id, p.x_min, p.y_min, p.x_max, p.y_max);
}
}
return 0;
}
Automatic Face Enrollment¶
This example extends the code above and shows how to use the automatic face enrollment functionality.
The changes compared to the minimal example above are highlighted.
#include <cstdint>
#include <cstdio>
#include <vector>
#include "plumerai/video_intelligence.h"
int main() {
// Settings, to be changed as needed
constexpr int width = 1600; // camera image width in pixels
constexpr int height = 1200; // camera image height in pixels
constexpr auto image_format = plumerai::ImageFormat::PACKED_RGB888;
// Initialize the video intelligence algorithm
auto pvi = plumerai::VideoIntelligence(height, width);
// Set whether the video stream is night mode (IR) or not.
auto error_code = pvi.set_night_mode(false);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Loop over frames in a video stream (example: 10 frames)
for (int t = 0; t < 10; ++t) {
// Some example input here, normally this is where camera data is acquired
auto image = std::vector<std::uint8_t>(height * width * 3); // 3 for RGB
// Duration between the *previous* frame passed to `process_frame` and the
// *current* frame, in seconds.
//
// Here we assume a fixed capture rate of 30 fps, so delta_t = 1 / 30.
// If your camera runs at a different or variable frame rate, be sure to
// update `delta_t` accordingly. The function `process_frame` relies on this
// value to keep motion tracking and temporal filters in sync.
const float delta_t = 1.f / 30.f;
// Process the frame
error_code = pvi.process_frame(
plumerai::ImagePointer<image_format>(image.data()), delta_t);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Report the number of faces in the library so far. At first the library
// will be empty, but as soon as a face is well visible for a while, it
// will be added to the library with a new unique face ID. The library
// will grow over time, unless `remove_face_embedding` is called.
std::vector<int> face_ids;
error_code = pvi.face_enrollment_automatic().get_face_ids(face_ids);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
printf("Total of %zu people in the familiar face-ID library\n",
face_ids.size());
std::vector<BoxPrediction> predictions;
error_code = pvi.object_detection().get_detections(predictions);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Display the results to stdout
for (auto &p : predictions) {
if (p.class_id == CLASS_PERSON) {
// `face_id` will be one from 'face_ids'
const auto face_id = pvi.face_identification().get_face_id(p);
printf("Box with face ID %d @ (x,y) -> (%.2f,%.2f) till (%.2f,%.2f)\n",
face_id, p.x_min, p.y_min, p.x_max, p.y_max);
}
}
}
return 0;
}
Manual face enrollment¶
This example shows how to use the manual face enrollment functionality. It consists of two main loops:
- An example enrollment loop, which runs for a fixed number of frames and computes a face embedding vector to enroll one person in the face library.
- An example video processing loop, similar to the first example.
#include <cstdint>
#include <cstdio>
#include <vector>
#include "plumerai/video_intelligence.h"
int main() {
// Settings, to be changed as needed
constexpr int width = 1600; // camera image width in pixels
constexpr int height = 1200; // camera image height in pixels
constexpr auto image_format = plumerai::ImageFormat::PACKED_RGB888;
// Initialize the `VideoIntelligence` object
auto pvi = plumerai::VideoIntelligence(height, width);
// Set whether the video stream is night mode (IR) or not.
auto error_code = pvi.set_night_mode(false);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// ---------------------- Enrollment starting ------------------------------
error_code = pvi.face_enrollment_manual().start_enrollment();
if (error_code != plumerai::ErrorCode::ENROLLMENT_IN_PROGRESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Enroll for 10 frames (just an example, more frames is better)
for (int t = 0; t < 10; ++t) {
// Some example input here, normally this is where camera data is acquired
auto image = std::vector<std::uint8_t>(height * width * 3); // 3 for RGB
// Process the frame
// If the enrollment frames come from a video source, then use
// `process_frame` instead:
// error_code = pvi.process_frame(
// plumerai::ImagePointer<image_format>(image.data()), delta_t);
error_code =
pvi.single_image(plumerai::ImagePointer<image_format>(image.data()));
printf("Enrollment status: %s\n", plumerai::error_code_string(error_code));
}
// Finish enrollment
std::vector<std::int8_t> embedding;
error_code = pvi.face_enrollment_manual().finish_enrollment(embedding);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Add the embedding to the library with face ID '1'.
error_code = pvi.face_enrollment_manual().add_embedding(embedding, 1);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// ---------------------- Enrollment finished ------------------------------
// Loop over frames in a video stream (example: 10 frames)
for (int t = 0; t < 10; ++t) {
// Some example input here, normally this is where camera data is acquired
auto image = std::vector<std::uint8_t>(height * width * 3); // 3 for RGB
// Duration between the *previous* frame passed to `process_frame` and the
// *current* frame, in seconds.
//
// Here we assume a fixed capture rate of 30 fps, so delta_t = 1 / 30.
// If your camera runs at a different or variable frame rate, be sure to
// update `delta_t` accordingly. The function `process_frame` relies on this
// value to keep motion tracking and temporal filters in sync.
const float delta_t = 1.f / 30.f;
// Process the frame
error_code = pvi.process_frame(
plumerai::ImagePointer<image_format>(image.data()), delta_t);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
std::vector<BoxPrediction> predictions;
error_code = pvi.object_detection().get_detections(predictions);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Display the results to stdout
for (auto &p : predictions) {
if (p.class_id == CLASS_PERSON) {
printf("Box with face ID %d @ (x,y) -> (%.2f,%.2f) till (%.2f,%.2f)\n",
pvi.face_identification().get_face_id(p), p.x_min, p.y_min,
p.x_max, p.y_max);
}
}
if (predictions.size() == 0) {
printf("No bounding boxes found in frame\n");
}
}
return 0;
}
Advanced motion detection¶
This example demonstrates the motion detection capabilities. This is an example of how one could implement a simple user-configurable motion detection sensitivity setting.
#include <cstdint>
#include <cstdio>
#include <vector>
#include "plumerai/video_intelligence.h"
int main() {
// Settings, to be changed as needed
constexpr int width = 1600; // camera image width in pixels
constexpr int height = 1200; // camera image height in pixels
constexpr auto image_format = plumerai::ImageFormat::PACKED_RGB888;
// Initialize the video intelligence algorithm
auto pvi = plumerai::VideoIntelligence(height, width);
// Set whether the video stream is night mode (IR) or not.
auto error_code = pvi.set_night_mode(false);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Loop over frames in a video stream (example: 10 frames)
for (int t = 0; t < 10; ++t) {
// Some example input here, normally this is where camera data is acquired
auto image = std::vector<std::uint8_t>(height * width * 3); // 3 for RGB
// Duration between the *previous* frame passed to `process_frame` and the
// *current* frame, in seconds.
//
// Here we assume a fixed capture rate of 30 fps, so delta_t = 1 / 30.
// If your camera runs at a different or variable frame rate, be sure to
// update `delta_t` accordingly. The function `process_frame` relies on this
// value to keep motion tracking and temporal filters in sync.
const float delta_t = 1.f / 30.f;
// Process the frame
error_code = pvi.process_frame(
plumerai::ImagePointer<image_format>(image.data()), delta_t);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Retrieve the motion detection grid
const float* motion_grid = nullptr;
error_code = pvi.motion_detection().get_grid(&motion_grid);
if (error_code == plumerai::ErrorCode::MOTION_GRID_NOT_YET_READY) {
continue; // process another frame and wait for the grid to be ready
}
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
const auto grid_height = pvi.motion_detection().get_grid_height();
const auto grid_width = pvi.motion_detection().get_grid_width();
// The motion detection grid can be processed in multiple ways. In this
// example, we demonstrate a very simple thresholding mechanism, supporting
// 5 different user-configurable sensitivity levels. A more sophisticated
// approach might not only look at the maximum value in the grid, but also,
// for example, its surrounding values, the number of values above a certain
// threshold, or temporally/spatially aggregated values.
const auto user_sensitivity_level = 2; // 0-4, where 0 is most sensitive
const float sensitivity_thresholds[5] = {0.0f, 0.2f, 0.4f, 0.6f, 0.8f};
auto max_value = 0.0f;
for (int y = 0; y < grid_height; ++y) {
for (int x = 0; x < grid_width; ++x) {
max_value = std::max(max_value, motion_grid[y * grid_width + x]);
}
}
printf("[Frame %d] Max value in grid: %.2f. ", t, max_value);
if (max_value >= sensitivity_thresholds[user_sensitivity_level]) {
printf("Motion detected!\n");
} else {
printf("No significant motion detected.\n");
}
}
return 0;
}
VLM Video Collection and Embedder¶
This example demonstrates the VLM Video capabilities.
#include <cstdint>
#include <cstdio>
#include <vector>
#include "plumerai/video_intelligence.h"
#include "plumerai/vlm_video_embedder.h"
bool any_motion_detected(const plumerai::VideoIntelligence& pvi) {
// Check if any motion was detected in the current frame.
// See the minimal example for motion detection for more information about
// this logic.
const float* motion_grid = nullptr;
auto error_code = pvi.motion_detection().get_grid(&motion_grid);
if (error_code == plumerai::ErrorCode::MOTION_GRID_NOT_YET_READY) {
return false; // process another frame and wait for the grid to be ready
}
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return false;
}
const auto grid_height = pvi.motion_detection().get_grid_height();
const auto grid_width = pvi.motion_detection().get_grid_width();
auto max_motion = 0.0f;
for (int y = 0; y < grid_height; ++y) {
for (int x = 0; x < grid_width; ++x) {
max_motion = std::max(max_motion, motion_grid[y * grid_width + x]);
}
}
return max_motion > 0.1f; // Example threshold
}
int main() {
// Settings, to be changed as needed
constexpr int width = 1600; // camera image width in pixels
constexpr int height = 1200; // camera image height in pixels
constexpr auto image_format = plumerai::ImageFormat::PACKED_RGB888;
// Initialize the video intelligence algorithm
auto pvi = plumerai::VideoIntelligence(height, width);
// Initialize the VLM Video embedder
auto pvve = plumerai::VLMVideoEmbedder();
// Set whether the video stream is night mode (IR) or not.
auto error_code = pvi.set_night_mode(false);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// The VLM Video API processes video in user-defined 'clips'.
bool clip_has_started = false;
float clip_start_time = 0.0f;
float time_without_motion = 0.0f;
std::vector<std::uint8_t> clip_data;
std::vector<std::uint8_t> clip_embeddings;
float current_time = 0.0f;
// Loop over frames in a video stream (example: 10 frames)
for (int t = 0; t < 10; ++t) {
// Some example input here, normally this is where camera data is acquired
auto image = std::vector<std::uint8_t>(height * width * 3); // 3 for RGB
// Duration between the *previous* frame passed to `process_frame` and the
// *current* frame, in seconds. Variable framerates are supported.
const float delta_t = 1.f / 30.f;
// Process the frame
error_code = pvi.process_frame(
plumerai::ImagePointer<image_format>(image.data()), delta_t);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
if (!clip_has_started) {
// Example: check if we should start a new clip based on motion detection.
// This could also be based on object detection or fixed time intervals.
const bool should_start_clip = any_motion_detected(pvi);
if (should_start_clip) {
error_code = pvi.vlm_video_collection().start_clip();
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
clip_has_started = true;
clip_start_time = current_time;
time_without_motion = 0.0f;
}
} else {
// In this example we end the clip when there is no motion detected for at
// least 2 seconds, and we limit the clip length to 30 seconds.
// A more sophisticated method could also consider object detections.
constexpr float max_clip_duration = 30.0f;
constexpr float min_time_without_motion = 2.0f;
if (any_motion_detected(pvi)) {
time_without_motion = 0.0f;
} else {
time_without_motion += delta_t;
}
const auto clip_duration = current_time - clip_start_time;
const bool should_end_clip =
(clip_duration >= max_clip_duration) ||
(time_without_motion >= min_time_without_motion);
if (should_end_clip) {
error_code = pvi.vlm_video_collection().end_clip(clip_data);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
clip_has_started = false;
// Now process the collected clip data with VLM Video Embedder
// This is time-consuming: there is also a `compute_single_unit_only`
// option.
const bool compute_single_unit_only = false;
error_code = pvve.compute_embeddings(
clip_data.data(), clip_data.size(), clip_embeddings,
compute_single_unit_only);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Deallocate clip_data
clip_data.clear();
// `clip_embeddings` can now be stored (e.g. to disk) for e.g. video
// search.
}
}
current_time += delta_t;
}
return 0;
}
Re-Identification¶
This example demonstrates the Re-Identification functionality.
#include <chrono>
#include <cstdio>
#include <fstream>
#include <vector>
#include "plumerai/video_intelligence.h"
// Example code reading a binary blob from file
inline std::vector<std::uint8_t> read_from_file(const std::string &filename) {
std::ifstream file(filename, std::ios::binary);
std::vector<std::uint8_t> result;
if (file.good()) {
file.seekg(0, std::ios::end);
result.resize(file.tellg());
file.seekg(0, std::ios::beg);
file.read(reinterpret_cast<char *>(result.data()), result.size());
file.close();
}
return result;
}
// Example code writing a binary blob to file
inline void write_to_file(const std::string &filename,
const std::vector<std::uint8_t> &data) {
std::ofstream file(filename, std::ios::binary | std::ios::trunc);
if (file.good()) {
file.write(reinterpret_cast<const char *>(data.data()), data.size());
file.close();
}
}
int main() {
// Settings, to be changed as needed
constexpr int width = 1280; // camera image width in pixels
constexpr int height = 720; // camera image height in pixels
constexpr auto image_format = plumerai::ImageFormat::PACKED_RGB888;
// Get the current ReIdentification state. This data is the output from
// a previous call to `re_identification().get_re_id_state()` for a previous
// run of Plumerai Video Intelligence, either on the same camera on elsewhere.
// This data can be stored in memory, in a database, or in file. Here we read
// the data in from a file as an example.
const auto re_id_state = read_from_file("re_id_state.bin");
// Initialize the video intelligence algorithm
auto pvi = plumerai::VideoIntelligence(height, width);
auto error_code = plumerai::ErrorCode::SUCCESS;
// Set the clock time (required for ReIdentification)
const auto cur_time = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch())
.count();
error_code = pvi.set_clock_time(cur_time);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
error_code = pvi.set_camera_name("camera_01", "Front yard");
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Restore the ReIdentification state (if there is any)
if (re_id_state.size() > 0) {
error_code = pvi.re_identification().merge_re_id_state(re_id_state);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
}
// Loop over frames in a video stream (example: 20 frames)
for (int t = 0; t < 20; ++t) {
// Some example input here, normally this is where camera data is acquired
auto image = std::vector<std::uint8_t>(height * width * 3); // 3 for RGB
// Duration between the *previous* frame passed to `process_frame` and the
// *current* frame, in seconds.
const float delta_t = 1.f / 30.f; // example assuming 30 FPS, change this!
// Process the frame
error_code = pvi.process_frame(
plumerai::ImagePointer<image_format>(image.data()), delta_t);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
std::vector<BoxPrediction> predictions;
error_code = pvi.object_detection().get_detections(predictions);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Loop over all predictions made this frame
for (const auto &p : predictions) {
if (p.class_id != CLASS_PERSON) continue; // ReID only works for persons
// Now retrieve any matching track IDs from previous frames or from the
// previously loaded ReIdentification state, if any.
std::vector<std::int64_t> matching_track_ids;
error_code = pvi.re_identification().get_matching_track_ids(
p.track_id, matching_track_ids);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
// Print the results to stdout. Note that the list of matching track IDs
// will always contain at least the track ID itself. Only when there are
// two or more matches, a ReIdentification match has been found.
printf("Prediction for track ID %ld has %zu matching track IDs: ",
p.track_id, matching_track_ids.size());
for (const auto& id : matching_track_ids) {
printf("%ld ", id);
}
printf("\n");
}
}
// At the end we can write the current ReIdentification state back to
// storage, to be restored on the next run of the program.
std::vector<std::uint8_t> new_re_id_state;
error_code = pvi.re_identification().get_re_id_state(new_re_id_state);
if (error_code != plumerai::ErrorCode::SUCCESS) {
printf("Error: %s\n", plumerai::error_code_string(error_code));
return 1;
}
write_to_file("re_id_state.bin", new_re_id_state);
return 0;
}