Add Python bindings for libvillas

.gitlab-ci.yml

@@ -164,7 +164,7 @@ test:python:
     - pytest --verbose .
     - black --line-length=90 --extend-exclude=".*(\\.pyi|_pb2.py)$" --check .
     - flake8 --max-line-length=90 --extend-exclude="*.pyi,*_pb2.py" .
-    - mypy .
+    - mypy --explicit-package-bases .
   image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
   needs:
     - job: "build:source: [fedora]"
@@ -182,6 +182,28 @@ test:cppcheck:
       - cppcheck.log
     expose_as: cppcheck
 
+test:python_unit_integration:
+  stage: test
+  image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}
+  before_script:
+    # dependency from node.py, which gets imported because of __init__.py in node/villas/python/node
+    - pip install requests
+  script:
+    # build binding and symlink to villas.node folder
+    # helps with correct imports and compatiblity for CI, binding wrapper and binding install
+    - export PYTHONPATH=$PYTHONPATH:${PWD}/python
+    - cmake --build build ${CMAKE_BUILD_OPTS} --target python_binding
+
+    - binding_path=$(find ${PWD}/build/python/binding/ -name "python_binding*.so" | head -n1)
+    - link_path=${PWD}/python/villas/node/$(basename $binding_path)
+    - ln -sf $binding_path $link_path
+
+    - cmake --build build ${CMAKE_BUILD_OPTS} --target run-python-unit-tests run-python-integration-tests
+    - rm $link_path
+  needs:
+    - job: "build:source: [fedora]"
+      artifacts: true
+
 test:unit:
   stage: test
   image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG}

CMakeLists.txt

@@ -4,7 +4,7 @@
 # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University
 # SPDX-License-Identifier: Apache-2.0
 
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.15)
 
 project(villas-node
     VERSION 1.1.0

include/villas/node.h

@@ -6,6 +6,7 @@
  */
 
 #pragma once
+#include <cstdint>
 
 #include <jansson.h>
 #include <stdbool.h>
@@ -67,11 +68,11 @@ unsigned sample_length(vsample *smp);
 
 void sample_decref(vsample *smp);
 
-vsample *sample_pack(unsigned seq, struct timespec *ts_origin,
+vsample *sample_pack(uint64_t *seq, struct timespec *ts_origin,
                      struct timespec *ts_received, unsigned len,
                      double *values);
 
-void sample_unpack(vsample *s, unsigned *seq, struct timespec *ts_origin,
+void sample_unpack(vsample *s, uint64_t *seq, struct timespec *ts_origin,
                    struct timespec *ts_received, int *flags, unsigned *len,
                    double *values);
 

lib/node.cpp

@@ -440,7 +440,7 @@ Node *NodeFactory::make(json_t *json, const uuid_t &id,
   std::string type;
   Node *n;
 
-  if (json_is_object(json))
+  if (!json_is_object(json))
     throw ConfigError(json, "node-config-node",
                       "Node configuration must be an object");
 

lib/node_capi.cpp

@@ -152,12 +152,12 @@ unsigned sample_length(vsample *s) {
   return smp->length;
 }
 
-vsample *sample_pack(unsigned seq, struct timespec *ts_origin,
+vsample *sample_pack(uint64_t *seq, struct timespec *ts_origin,
                      struct timespec *ts_received, unsigned len,
                      double *values) {
   auto *smp = sample_alloc_mem(len);
 
-  smp->sequence = seq;
+  smp->sequence = *seq;
   smp->ts.origin = *ts_origin;
   smp->ts.received = *ts_received;
   smp->length = len;
@@ -169,7 +169,7 @@ vsample *sample_pack(unsigned seq, struct timespec *ts_origin,
   return (vsample *)smp;
 }
 
-void sample_unpack(vsample *s, unsigned *seq, struct timespec *ts_origin,
+void sample_unpack(vsample *s, uint64_t *seq, struct timespec *ts_origin,
                    struct timespec *ts_received, int *flags, unsigned *len,
                    double *values) {
   auto *smp = (Sample *)s;

lib/nodes/socket.cpp

@@ -67,6 +67,9 @@ int villas::node::socket_init(NodeCompat *n) {
   auto *s = n->getData<struct Socket>();
 
   s->formatter = nullptr;
+  s->sd = -1;
+  s->in.buf = nullptr;
+  s->out.buf = nullptr;
 
   return 0;
 }
@@ -77,6 +80,17 @@ int villas::node::socket_destroy(NodeCompat *n) {
   if (s->formatter)
     delete s->formatter;
 
+  if (s->in.buf != nullptr) {
+    memset(s->in.buf, 0, s->in.buflen);
+    delete[] s->in.buf;
+    s->in.buf = nullptr;
+  }
+  if (s->out.buf != nullptr) {
+    memset(s->out.buf, 0, s->out.buflen);
+    delete[] s->out.buf;
+    s->out.buf = nullptr;
+  }
+
   return 0;
 }
 
@@ -355,8 +369,16 @@ int villas::node::socket_stop(NodeCompat *n) {
       return ret;
   }
 
-  delete[] s->in.buf;
-  delete[] s->out.buf;
+  if (s->in.buf != nullptr) {
+    memset(s->in.buf, 0, s->in.buflen);
+    delete[] s->in.buf;
+    s->in.buf = nullptr;
+  }
+  if (s->out.buf != nullptr) {
+    memset(s->out.buf, 0, s->out.buflen);
+    delete[] s->out.buf;
+    s->out.buf = nullptr;
+  }
 
   return 0;
 }

packaging/docker/Dockerfile.debian

@@ -52,6 +52,8 @@ RUN apt-get update && \
 	libssl-dev \
 	libusb-1.0-0-dev \
 	libzmq3-dev \
+	python3-dev \
+	python3-pybind11 \
 	uuid-dev
 
 # Install unpackaged dependencies from source

packaging/docker/Dockerfile.debian-multiarch

@@ -58,6 +58,8 @@ RUN apt-get update && \
 	libssl-dev:${ARCH} \
 	libusb-1.0-0-dev:${ARCH} \
 	libzmq3-dev:${ARCH} \
+	python3-dev:${ARCH} \
+	python3-pybind11:${ARCH} \
 	uuid-dev:${ARCH}
 
 ADD cmake/toolchains/debian-${ARCH}.cmake /

packaging/docker/Dockerfile.fedora

@@ -27,7 +27,7 @@ RUN dnf -y install \
 	openssh-clients \
 	jq nmap-ncat \
 	iproute iproute-tc \
-	python python-devel python-pip \
+	python-devel python-pip \
 	gdb gdb-gdbserver \
 	cppcheck \
 	xmlto dblatex rubygem-asciidoctor \
@@ -63,6 +63,7 @@ RUN dnf -y install \
 	openssl-devel \
 	protobuf-c-devel \
 	protobuf-devel \
+	pybind11-devel \
 	spdlog-devel \
 	zeromq-devel
 

packaging/docker/Dockerfile.rocky

@@ -53,7 +53,9 @@ RUN dnf -y install \
 	nanomsg-devel \
 	libnice-devel \
 	libre-devel \
-	libwebsockets-devel
+	libwebsockets-devel \
+	python-devel \
+	pybind11-devel
 
 # Install unpackaged dependencies from source
 ADD packaging/patches /deps/patches

packaging/docker/Dockerfile.ubuntu

@@ -59,6 +59,8 @@ RUN apt-get update && \
 	libusb-1.0-0-dev \
 	libwebsockets-dev \
 	libzmq3-dev \
+	python3-dev \
+	python3-pybind11 \
 	uuid-dev
 
 # Install unpackaged dependencies from source

python/CMakeLists.txt

@@ -4,6 +4,8 @@
 # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University
 # SPDX-License-Identifier: Apache-2.0
 
+add_subdirectory(binding)
+
 if(DEFINED PROTOBUF_COMPILER AND PROTOBUF_FOUND)
     add_custom_command(
         OUTPUT

python/binding/CMakeLists.txt

@@ -0,0 +1,33 @@
+# Python-binding CMakeLists.
+#
+# Author: Kevin Vu te Laar <vu.te@rwth-aachen.de>
+# SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University
+# SPDX-License-Identifier: Apache-2.0
+
+set(PYBIND11_FINDPYTHON ON)
+find_package(pybind11 CONFIG)
+
+if(pybind11_FOUND)
+  find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
+
+  execute_process(
+    COMMAND "${Python3_EXECUTABLE}" -c "import sysconfig; print(sysconfig.get_path('purelib'))"
+    OUTPUT_VARIABLE PYTHON_SITE_PACKAGES
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+
+  message(STATUS "Found Python version: ${Python_VERSION}")
+  message(STATUS "Python major version: ${Python_VERSION_MAJOR}")
+  message(STATUS "Python minor version: ${Python_VERSION_MINOR}")
+
+  pybind11_add_module(python_binding capi_python_binding.cpp)
+  target_link_libraries(python_binding PUBLIC villas)
+
+  install(
+    TARGETS python_binding
+    COMPONENT lib
+    LIBRARY DESTINATION ${PYTHON_SITE_PACKAGES}/villas/node/
+  )
+else()
+  message(STATUS "pybind11 not found. Skipping Python wrapper build.")
+endif()