diff --git a/README.md b/README.md index 273d845..38aee72 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,49 @@ # Augmenta Client C++ SDK -The goal of this library is to make consuming the stream output of an Augmenta server as easy as possible. As of right now this only refers to the data emitted through websocket by the Websocket Output node, but other network protocols might be added later on. +The goal of this library is to make consuming the stream output of an Augmenta server as easy as possible. +As of right now this only refers to the data emitted through websocket by the Websocket Output node, but other network protocols might be added later on. -## Building -We use CMake as our buildsystem. +## Features +The SDK can be used to parse different type of data received from Augmenta: +- **Tracking data** : A list of tracked objects/persons with their ID, position, bounding box and velocity +- **Volumetric data** : Filtered point cloud data, either per-cluster, per-zone, or from the whole scene +- **Zones events** : Be notified of events like enter, leave, presence... affecting a specific zone in the world +- **World hierarchy** : The structure of the world as set up on Augmenta's side: scenes, zones, etc. + +Features yet to be implemented are: +- **Skeleton tracking** +- **Custom cluster data** + +## Protocol +This SDK can help you parse data received from the server, but you'll still need to respect the [Augmenta Websocket Protocol Specification](https://augmentatech.notion.site/Augmenta-WebSocket-Protocol-Specification-v2-637551d8e04a4015a56526d80e1b10f0?pvs=74). Make sure to give it a read ! + +## Using the SDK +### Integrate in your project +The SDK is composed of a single header/implementation file pair. We provide support for CMake. You can integrate it the way you see fit: +- using CMake +- as a git submodule +- by dropping the files directly in your project + +### Implement in your codebase +The SDK revolves around creating an `Augmenta::Client` object and using it to parse data blobs and messages received. +See [examples/Example.cpp](examples/Example.cpp) for a full usage example. + +### Websocket implementation +This SDK does not come with a websocket implementation, as we expect most users' environments to contain one already. If that is not the case we can recommend the [websocketpp](https://github.com/zaphoyd/websocketpp) library. +### Lifetime +Augmenta's servers and software are designed to run perpetually. They will automatically restarts in case of crashes, reboot, power loss, etc. +Make sure to take this into account and handle the connection lifecycle properly. For example, you should try to reconnect to the server automatically in case the connection is lost (which could happen in cases of network outage for example). +### Options +Using the SDK, you can select options to control the format and kind of data that the Augmenta server will send (see the implementation for descriptions). + +While implementing a client, keep in mind that some of those options should be controllable by your users (which type of data will be sent) while some other might be locked by you, the developper (things like your software's coordinate system). + +> **Note:** Changed options will only be taken into account after re-initializing the client. + +## Contribute to the SDK +We use CMake as our buildsystem. To build a standalone version of the lib: ``` @@ -16,10 +54,8 @@ cmake -B build -S . cmake --build build ``` -## Using the SDK -The SDK revolves around creating an `Augmenta::Client` object and using it to parse data blobs and messages received. -See [examples/Example.cpp](examples/Example.cpp) for a full usage example. - -## Dependencies +## Dependencies and ressources - [zstd](https://github.com/facebook/zstd) - - [nlohmann::json](https://github.com/nlohmann/json) \ No newline at end of file + - [nlohmann::json](https://github.com/nlohmann/json) + +> Please reach out to us if you have any questions - Augmenta Team \ No newline at end of file diff --git a/examples/Example.cpp b/examples/Example.cpp index 6c1c75b..7b82857 100644 --- a/examples/Example.cpp +++ b/examples/Example.cpp @@ -4,13 +4,41 @@ #include #include -// Sample data types. Replace with your own types ! using Vector3f = std::array; +using String = std::string; + +// Sample data types. Replace with your own ! using PointCloud = std::vector; -struct Cluster { + +struct Cluster +{ + Vector3f boundingBoxPosition; + Vector3f boundingBoxSize; + Vector3f centroid; + Vector3f velocity; + float weight; + PointCloud pointCloud; +}; + +struct ZoneEvent +{ + std::array xyPad; + int presence; + float sliderValue; + int enters; + int leaves; + PointCloud pointCloud; +}; + +struct HierarchyObject +{ + String type; + String name; + String address; Vector3f position; + Vector3f orientation; + std::vector children; }; -using String = std::string; struct ExampleWebSocketClient { @@ -63,65 +91,88 @@ struct ExampleWebSocketClient // 4 (cont.) ------ void OnDataBlobReceived(const std::vector &dataBlob) { + // You might want to keep that somewhere else + std::vector frameClusters; + std::vector frameScenePointClouds; + std::vector frameZoneEvents; + auto parsedData = augmentaClient.parseDataBlob(dataBlob.data(), dataBlob.size()); // Scene info const auto &sceneInfo = parsedData.getSceneInfo(); - String scenePath; - scenePath.resize(sceneInfo.getAddressLength()); - sceneInfo.getAddress(scenePath.data()); + String scenePath = sceneInfo.getAddress(); - // Objects + // Objects (Clusters and Point Clouds) for (auto &object : parsedData.getObjects()) { const auto &objectID = object.getID(); if (object.hasCluster()) { auto &clusterInfo = object.getCluster(); - - if (clusterInfo.getState() == Augmenta::ClusterState::Entered) + + Cluster& cluster = frameClusters.emplace_back(); + cluster.boundingBoxPosition = clusterInfo.getBoundingBoxCenter(); + cluster.boundingBoxSize = clusterInfo.getBoundingBoxSize(); + cluster.centroid = clusterInfo.getCentroid(); + cluster.velocity = clusterInfo.getVelocity(); + cluster.weight = clusterInfo.getWeight(); + + if (object.hasPointCloud()) { - // This is a new cluster, add it to our list - } + // If the object has both a cluster and point cloud property, + // it is the cluster's contained point cloud + auto& pcInfo = object.getPointCloud(); - if (clusterInfo.getState() == Augmenta::ClusterState::WillLeave) - { - // Clean up leaving clusters + cluster.pointCloud.resize(pcInfo.getPointCount()); + pcInfo.getPointsData(cluster.pointCloud.data()); } } - - if (object.hasPointCloud()) + else if (object.hasPointCloud()) { auto &pcInfo = object.getPointCloud(); // Do something with the data // For example you can copy the point data to your own data structure. - PointCloud pc; + PointCloud &pc = frameScenePointClouds.emplace_back(); pc.resize(pcInfo.getPointCount()); pcInfo.getPointsData(pc.data()); } } - // Zones - for (const auto &zone : parsedData.getZones()) + // Zone Events + for (const auto &zoneEventInfo : parsedData.getZoneEvents()) { - for (const auto &property : zone.getProperties()) + const auto emitterZoneAddress = zoneEventInfo.getEmitterZoneAddress(); + // You could use that to map the event to your zone struct + + ZoneEvent& zoneEvent = frameZoneEvents.emplace_back(); + zoneEvent.enters = zoneEventInfo.getEnters(); + zoneEvent.leaves = zoneEventInfo.getLeaves(); + zoneEvent.presence = zoneEventInfo.getPresence(); + + for (const auto &property : zoneEventInfo.getProperties()) { switch (property.getType()) { case Augmenta::DataBlob::ZoneEventPacket::Property::Type::Slider: { const auto &sliderData = property.getSliderParameters(); - auto value = sliderData->getValue(); + zoneEvent.sliderValue = sliderData->getValue(); + break; } case Augmenta::DataBlob::ZoneEventPacket::Property::Type::XYPad: { const auto &xyData = property.getXYPadParameters(); - auto x = xyData->getX(); - auto y = xyData->getY(); + zoneEvent.xyPad = {xyData->getX(), xyData->getY()}; + break; + } + case Augmenta::DataBlob::ZoneEventPacket::Property::Type::PointCloud: + { + const auto& pcParams = property.getPointCloudParameters(); + zoneEvent.pointCloud.resize(pcParams->getPointCount()); + pcParams->getPointsData(zoneEvent.pointCloud.data()); } - // ... } } } diff --git a/include/AugmentaClientSDK.hpp b/include/AugmentaClientSDK.hpp index 89f64a1..18d5f5a 100644 --- a/include/AugmentaClientSDK.hpp +++ b/include/AugmentaClientSDK.hpp @@ -40,17 +40,26 @@ namespace Augmenta enum class CoordinateSpace { - Absolute, - Relative, - Normalized, + Absolute, // Augmenta's world space coordinates + Relative, // Coordinates relative to the parent object. Clusters are considered as children of the scene + Normalized, // Coordinates are normalized between 0 and 1 based on scene dimensions and origin mode }; + /// @brief Your desired coordinate system + /// @todo Should be renamed to something more specific AxisMode axis = AxisMode::ZUpRightHanded; + + /// @brief Defines the coordinate space in which coordinates sent by augmenta will be + CoordinateSpace coordinateSpace = CoordinateSpace::Absolute; + + /// @brief In Relative coordinates mode, the position of the scene's origin OriginMode origin = OriginMode::BottomLeft; + + /// @brief Finer controls to flip specific axis, useful if the client software uses an uncommon coordinate system. bool flipX = false; bool flipY = false; bool flipZ = false; - CoordinateSpace coordinateSpace = CoordinateSpace::Absolute; + // public originOffset; // TODO // public customMatrix; // TODO @@ -58,20 +67,49 @@ namespace Augmenta bool operator!=(const AxisTransform& other) const; }; + /// @brief Specify the protocol version which Augmenta will use to encode outgoing packets + /// You should probably leave that to default unless you now what you are doing ! int version = 2; + + /// @brief Tags can be used to filter the data sent by the server. Leave empty to receive everything. std::vector tags; + + /// @brief Downsampling factor for point clouds. + /// For example, a value of 2 means that only every 2nd point will be sent + /// Must be >= 1 int downSample = 1; + + /// @brief Whether the server should send the scene Point Cloud data bool streamClouds = true; + /// @brief Whether the server should send the scene Cluster data bool streamClusters = true; + /// @brief Whether the server should send the points that make up a Cluster along with it bool streamClusterPoints = true; - bool streamSkeletonData = true; - bool streamMetadata = true; + + // bool streamSkeletonData = true; // @TODO: Not implemented yet + // bool streamMetadata = true; // @TODO: Not implemented yet + + /// @brief Whether the server should send the points included in a zone along with zone packets. bool streamZonePoints = false; + + /// @brief Specify the orientation represention in which cluster rotations should be sent. RotationMode boxRotationMode = RotationMode::Quaternions; - AxisTransform axisTransform; // TODO: Default ? + + /// @brief Specify how coordinates sent by Augmenta should be represented. + /// Augmenta will take care of transforming outgoing data in your desired coordinate system. + AxisTransform axisTransform; + + /// @brief Enable ZSTD compression on the binary data stream. The SDK will handle decompression. bool useCompression = true; + + /// @brief By default, Augmenta will send a new frame as soon as it is available. + /// If that is overwhelming your application, you can enable this option to only receive + /// data on demand (by sending a poll request). + /// Note that this should be a last resort option. bool usePolling = false; - bool displayPointIntensity = false; + + /// @todo Not implemented yet + bool displayPointIntensity = false; bool operator==(const ProtocolOptions& other) const; bool operator!=(const ProtocolOptions& other) const; @@ -79,12 +117,14 @@ namespace Augmenta enum class ClusterState : int { - Entered = 0, - Updated = 1, - WillLeave = 2, - Ghost = 3, + Entered = 0, // First frame the cluster is present + Updated = 1, // Normal state of a cluster + WillLeave = 2, // Last frame the cluster is present, it will not be here on the next + Ghost = 3, // If ghosting is enabled, clusters will be kept a few frames in this state after being lost. From there they can either be picked back up (Update) or disappear (WillLeave). }; + /// @brief Used to query specific information from data frames received from the server + /// @warning Keeps references to the parsed buffer. Users are responsible for keeping the buffer available as long as they are using the DataBlob. class DataBlob { friend class DataBlobParser; @@ -101,6 +141,7 @@ namespace Augmenta std::string sceneAddress; }; + /// @brief A cluster represents a tracked group of points. This is often an objet or a person. class ClusterProperty { friend class DataBlobParser; @@ -227,11 +268,9 @@ namespace Augmenta std::array centroid; std::array velocity; std::array boundingBoxCenter; + std::array boundingBoxRotation; std::array boundingBoxSize; float weight; - - std::array boundingBoxRotation; - std::array lookAt; }; @@ -242,7 +281,7 @@ namespace Augmenta public: int getPointCount() const { return pointsCount; } - /// @brief Copy all the cloud points at once. outData should be a contiguous container of Vector3f type (3*float = 12 bytes) + /// @brief Copy all the cloud points at once. outData should point to a contiguous container of Vector3f type (3*float = 12 bytes) template void getPointsData(Vector3f *outData) const { @@ -255,12 +294,31 @@ namespace Augmenta Vector3f getPoint(size_t pointIdx) const { static_assert(sizeof(Vector3f) == 12); + assert(pointsCount < pointIdx); Vector3f outPoint; std::memcpy(&outPoint, pointsPtr + (pointIdx * sizeof(Vector3f)), sizeof(Vector3f)); return outPoint; } + float getPointX(size_t pointIdx) const + { + assert(pointsCount < pointIdx); + return pointsPtr + (pointIdx * sizeof(float) * 3); + } + + float getPointY(size_t pointIdx) const + { + assert(pointsCount < pointIdx); + return pointsPtr + (pointIdx * sizeof(float) * 3) + sizeof(float); + } + + float getPointZ(size_t pointIdx) const + { + assert(pointsCount < pointIdx); + return pointsPtr + (pointIdx * sizeof(float) * 3) + sizeof(float) * 2; + } + private: int pointsCount; const std::byte *pointsPtr = nullptr; @@ -336,8 +394,8 @@ namespace Augmenta enum class Type : uint8_t { Unknown = UINT8_MAX, - Slider = 0, - XYPad = 1, + Slider = 0, // 1D slider. Axis is defined by the zone's configuration in Augmenta + XYPad = 1, // 2D slider PointCloud = 2, }; @@ -426,6 +484,7 @@ namespace Augmenta std::vector zoneEvents; }; + /// @brief Used to query information from a Message (Setup/Update) received from the server class ControlMessage { friend class ControlMessageParser; @@ -650,7 +709,10 @@ namespace Augmenta class Client { public: + /// @brief Initialize the client with a name and a set of options void initialize(const std::string &clientName, const ProtocolOptions &options); + + /// @brief Clear internal state void shutdown(); /// @brief Clear requested tags list @@ -663,9 +725,12 @@ namespace Augmenta std::string getPollMessage() const; const ProtocolOptions& getCurrentOptions() const { return options; } - /// @brief Parse a data blob and return a DataBlob object that can be used to query relevant information from it. + /// @brief Parse a binary message received from the server and return a DataBlob object that can be used to query relevant information from it. /// @warning The DataBlob keeps references to the parsed buffer. Users are responsible for keeping the buffer available as long as they are using the DataBlob. DataBlob parseDataBlob(const std::byte *blob, size_t blobSize); + + /// @brief Parse a text message received from the server and return a ControlMessage object + /// that can be used to retrieve the information. ControlMessage parseControlMessage(const char *rawMessage); private: