/* Interface related functions.

*

* Author: Steffen Vogel <post@steffenvogel.de>

* SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University

* SPDX-License-Identifier: Apache-2.0

*/

#include <cstdio>

#include <cstdlib>

#include <dirent.h>

#include <netlink/route/link.h>

#include <villas/cpuset.hpp>

#include <villas/exceptions.hpp>

#include <villas/kernel/if.hpp>

#include <villas/kernel/kernel.hpp>

#include <villas/kernel/nl.hpp>

#include <villas/kernel/tc.hpp>

#include <villas/kernel/tc_netem.hpp>

#include <villas/node/config.hpp>

#include <villas/nodes/socket.hpp>

#include <villas/super_node.hpp>

#include <villas/utils.hpp>

using namespace villas;

using namespace villas::node;

using namespace villas::utils;

using namespace villas::kernel;

Interface::Interface(struct rtnl_link *link, int aff)

: nl_link(link), tc_qdisc(nullptr), affinity(aff) {

logger = Log::get(fmt::format("kernel:if:{}", getName()));

int n = getIRQs();

if (n)

logger->warn("Did not found any interrupts");

logger->debug("Found {} IRQs", irqs.size());

}

Interface::~Interface() {

if (tc_qdisc)

rtnl_qdisc_put(tc_qdisc);

}

int Interface::start() {

logger->info("Starting interface which is used by {} nodes", nodes.size());

// Set affinity for network interfaces (skip _loopback_ dev)

if (affinity)

setAffinity(affinity);

// Assign fwmark's to nodes which have netem options

int ret, fwmark = 0;

for (auto *n : nodes) {

if (n->tc_qdisc && n->fwmark < 0)

n->fwmark = 1 + fwmark++;

}

// Abort if no node is using netem

if (fwmark == 0)

return 0;

if (getuid() != 0)

throw RuntimeError("Network emulation requires super-user privileges!");

// Replace root qdisc

ret = tc::prio(this, &tc_qdisc, TC_HANDLE(1, 0), TC_H_ROOT, fwmark);

if (ret)

throw RuntimeError("Failed to setup priority queuing discipline: {}",

nl_geterror(ret));

// Create netem qdisks and appropriate filter per netem node

for (auto *n : nodes) {

if (n->tc_qdisc) {

ret =

tc::mark(this, &n->tc_classifier, TC_HANDLE(1, n->fwmark), n->fwmark);

if (ret)

throw RuntimeError("Failed to setup FW mark classifier: {}",

nl_geterror(ret));

char *buf = tc::netem_print(n->tc_qdisc);

logger->debug("Starting network emulation for FW mark {}: {}", n->fwmark,

buf);

free(buf);

ret = tc::netem(this, &n->tc_qdisc, TC_HANDLE(0x1000 + n->fwmark, 0),

TC_HANDLE(1, n->fwmark));

if (ret)

throw RuntimeError("Failed to setup netem qdisc: {}", nl_geterror(ret));

}

}

return 0;

}

int Interface::stop() {

logger->info("Stopping interface");

if (affinity)

setAffinity(-1L);

if (tc_qdisc)

tc::reset(this);

return 0;

}

std::string Interface::getName() const {

auto str = rtnl_link_get_name(nl_link);

return std::string(str);

}

Interface *Interface::getEgress(struct sockaddr *sa, SuperNode *sn) {

struct rtnl_link *link;

Logger logger = Log::get("kernel:if");

auto &interfaces = sn->getInterfaces();

auto affinity = sn->getAffinity();

// Determine outgoing interface

link = nl::get_egress_link(sa);

if (!link)

throw RuntimeError("Failed to get interface for socket address '{}'",

socket_print_addr(sa));

// Search of existing interface with correct ifindex

for (auto *i : interfaces) {

if (rtnl_link_get_ifindex(i->nl_link) == rtnl_link_get_ifindex(link))

return i;

}

// If not found, create a new interface

auto *i = new Interface(link, affinity);

if (!i)

throw MemoryAllocationError();

interfaces.push_back(i);

return i;

}

int Interface::getIRQs() {

int irq;

auto dirname = fmt::format("/sys/class/net/{}/device/msi_irqs/", getName());

DIR *dir = opendir(dirname.c_str());

if (dir) {

irqs.clear();

struct dirent *entry;

while ((entry = readdir(dir))) {

irq = atoi(entry->d_name);

if (irq)

irqs.push_back(irq);

}

closedir(dir);

}

return 0;

}

int Interface::setAffinity(int affinity) {

assert(affinity != 0);

if (getuid() != 0) {

logger->warn("Failed to tune IRQ affinity. Please run as super-user");

return 0;

}

FILE *file;

CpuSet cset_pin(affinity);

for (int irq : irqs) {

std::string filename = fmt::format("/proc/irq/{}/smp_affinity", irq);

file = fopen(filename.c_str(), "w");

if (file) {

if (fprintf(file, "%8lx", (unsigned long)cset_pin) < 0)

throw SystemError(

"Failed to set affinity for for IRQ {} on interface '{}'", irq,

getName());

fclose(file);

logger->debug("Set affinity of IRQ {} to {} {}", irq,

cset_pin.count() == 1 ? "core" : "cores",

(std::string)cset_pin);

} else

throw SystemError(

"Failed to set affinity for for IRQ {} on interface '{}'", irq,

getName());

}

return 0;

}