dmesg | grep -i -e DMAR -e IOMMU

sudo nano /etc/default/grub

GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt pci=realloc"
sudo grub-mkconfig -o /boot/grub/grub.cfg

mlxconfig -d /dev/mst/mt4119_pciconf0 q | grep SRIOV_EN

# Set the Number of VFs in the firmware
# Run mst start.

# Using the IDs obtained in the previous task, type the following:

sudo mlxconfig -d /dev/mst/mt4119_pciconf0set SRIOV_EN=1 NUM_OF_VFS=<number between 0-127>

sudo mlxconfig -d /dev/mst/mt4119_pciconf0 set SRIOV_EN=1 NUM_OF_VFS=8

sudo mlxconfig -d /dev/mst/mt4119_pciconf0 query

sudo mlxconfig -d /dev/mst/mt4119_pciconf0 query


#Update the firmware

sudo mst start
sudo mst status -v
sudo flint -d /dev/mst/mt4119_pciconf0 -i fw-download-from-official-website-based-on-psid-of-your-nic.bin burn

# Reboot for the settings to take effect.


cat /sys/class/net/enp10s0np0/device/mlx5_num_vfs
echo 8 > /sys/class/infiniband/mlx5_0/device/mlx5_num_vfs

wget https://git.dpdk.org/apps/pktgen-dpdk/snapshot/pktgen-dpdk-pktgen-23.06.1.tar.xz
wget httpstar xJf dpdk-23.03.tar.xz

If you wanna build 'igb_uio', follow below.  In my case, it did not work for Mellanox Connect X5 (mlx_core driver was enough)

git clone git://fast.dpdk.org/rel/dpdk-kmods
cp -r dpdk-kmods/linux/igb_uio dpdk-23.03/kernel/linux/
cd dpdk-23.03/kernel/linux/igb_uio
vi meson.build

""""""""""

# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2017 Intel Corporation

mkfile = custom_target('igb_uio_makefile',
        output: 'Makefile',
        command: ['touch', '@OUTPUT@'])

kernel_dir = '/lib/modules/' + kernel_version
custom_target('igb_uio',
        input: ['igb_uio.c', 'Kbuild'],
        output: 'igb_uio.ko',
        command: ['make', '-C', kernel_dir + '/build',
                'M=' + meson.current_build_dir(),
                'src=' + meson.current_source_dir(),
                'EXTRA_CFLAGS=-I' + meson.current_source_dir() +
                        '/../../../lib/librte_eal/include',
                'modules'],
        depends: mkfile,
        install: true,
        install_dir: kernel_dir + '/extra/dpdk',
        build_by_default: get_option('enable_kmods'))

""""""

cd .. (in meson.build -> dpdk-23.03/kernel/linux)
add the following:
subdirs = ['kni', 'igb_uio']

.tar.xz
tar xJf dpdk-23.03.tar.xz
tar xJf pktgen-dpdk-pktgen-23.06.1.tar.xz

cd dpdk-23.03
meson setup build
cd build
ninja
ninja install
ldconfig

export RTE_SDK=/

wget https://fast.dpdk.org/relroot/dpdk-23.03
export RTE_TARGET=x86_64-native-linux-gcc


mlxconfig -d /dev/mst/mt4119_pciconf0 q | grep SRIOV_EN

cat /sys/class/net/enp10s0np0/device/mlx5_num_vfs
echo 8.tar.xz
tar xJf dpdk-23.03.tar.xz

Set the hugepage to 1G and number of page as per your requirements (just increase 'nr_hugepages' if the eval mem allocation failed).

There are three ways to do it as follows.

# 1. Using dpdk based python program 
/usr/local/bin/dpdk-hugepages.py -p 1G --setup 32G

# 2. Insert 'nr_hugepages'
echo 4 > /sys/classkernel/infinibandmm/hugepages/hugepages-1048576kB/nr_hugepages

# 3. The permanent option (but I won't recommend this, this makes the the grub penic attack in rocky linux and goes to emergency mode)
vi /etc/fstab

export RTE_SDK=/root/dpdk-23.03
export RTE_TARGET=x86_64-native-linux-gcc 

wget https://git.dpdk.org/apps/pktgen-dpdk/snapshot/pktgen-dpdk-pktgen-23.06.1.tar.xz
tar xJf pktgen-dpdk-pktgen-23.06.1.tar.xz

mlx5_0/device/mlx5_num_vfs

dmesg | grep -i -e DMAR -e IOMMU

sudo nano /etc/default/grub

GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt pci=realloc"
sudo grub-mkconfig -o /boot/grub/grub.cfg