You are not logged in.

#1 2020-12-12 20:42:47

kinru
Member
From: East Coast USA
Registered: 2019-03-23
Posts: 99

Libvirt with nftables and NO firewalld

Hello,
Recently I have been creating a virtual machine following the PCI passthrough via OVMF wiki page. I had success up until I had installed the guest os, at which point I found there was no internet connection, or more accurately Windows 10 (guest os) says I am connected to a local network but not the greater internet. Of course, host machine has internet access and (very importantly I think) uses NFTables without firewalld.

I have installed the NetKVM driver on the guest. In my virt-manager, under the VM's "Show hardware details" there is a "Virtual Network Interface". The network source is Virtual Network 'Default': Nat, the device model is virtio, the ip address is Unknown and the link state active box is checked.

Output of brctl show:

 bridge name	bridge id		STP enabled	interfaces
docker0		8000.0242a5c58577	no		
virbr0		8000.525400bb4309	yes		virbr0-nic
							vnet0

Which I think is normal but under interfaces I am not sure if virbr0-nic should be virbr0 instead.

Output of sudo virsh net-list --all

 Name      State    Autostart   Persistent
--------------------------------------------
 default   active   yes         yes

Output of sudo nft list ruleset

table inet filter {
	chain input {
		type filter hook input priority filter; policy accept;
		ct state { established, related } accept
		ct state invalid drop
		iifname "lo" accept
		ip protocol icmp accept
		meta l4proto ipv6-icmp accept
		tcp dport 22 accept
		reject
	}

	chain forward {
		type filter hook forward priority filter; policy accept;
		drop
	}

	chain output {
		type filter hook output priority filter; policy accept;
	}
}
table ip filter {
	chain INPUT {
		type filter hook input priority filter; policy accept;
		counter packets 103274 bytes 140547087 jump LIBVIRT_INP
	}

	chain FORWARD {
		type filter hook forward priority filter; policy accept;
		counter packets 0 bytes 0 jump DOCKER-USER
		counter packets 0 bytes 0 jump DOCKER-ISOLATION-STAGE-1
		oifname "docker0" # xt_conntrack counter packets 0 bytes 0 accept
		oifname "docker0" counter packets 0 bytes 0 jump DOCKER
		iifname "docker0" oifname != "docker0" counter packets 0 bytes 0 accept
		iifname "docker0" oifname "docker0" counter packets 0 bytes 0 accept
		counter packets 0 bytes 0 jump LIBVIRT_FWX
		counter packets 0 bytes 0 jump LIBVIRT_FWI
		counter packets 0 bytes 0 jump LIBVIRT_FWO
	}

	chain OUTPUT {
		type filter hook output priority filter; policy accept;
		counter packets 61142 bytes 4186432 jump LIBVIRT_OUT
	}

	chain LIBVIRT_INP {
		iifname "virbr0" meta l4proto udp # xt_udp counter packets 0 bytes 0 accept
		iifname "virbr0" meta l4proto tcp # xt_tcp counter packets 0 bytes 0 accept
		iifname "virbr0" meta l4proto udp # xt_udp counter packets 119 bytes 39173 accept
		iifname "virbr0" meta l4proto tcp # xt_tcp counter packets 0 bytes 0 accept
	}

	chain LIBVIRT_OUT {
		oifname "virbr0" meta l4proto udp # xt_udp counter packets 0 bytes 0 accept
		oifname "virbr0" meta l4proto tcp # xt_tcp counter packets 0 bytes 0 accept
		oifname "virbr0" meta l4proto udp # xt_udp counter packets 0 bytes 0 accept
		oifname "virbr0" meta l4proto tcp # xt_tcp counter packets 0 bytes 0 accept
	}

	chain LIBVIRT_FWO {
		iifname "virbr0" ip saddr 192.168.122.0/24 counter packets 0 bytes 0 accept
		iifname "virbr0" counter packets 0 bytes 0 # xt_REJECT
	}

	chain LIBVIRT_FWI {
		oifname "virbr0" ip daddr 192.168.122.0/24 # xt_conntrack counter packets 0 bytes 0 accept
		oifname "virbr0" counter packets 0 bytes 0 # xt_REJECT
	}

	chain LIBVIRT_FWX {
		iifname "virbr0" oifname "virbr0" counter packets 0 bytes 0 accept
	}

	chain DOCKER {
	}

	chain DOCKER-ISOLATION-STAGE-1 {
		iifname "docker0" oifname != "docker0" counter packets 0 bytes 0 jump DOCKER-ISOLATION-STAGE-2
		counter packets 0 bytes 0 return
	}

	chain DOCKER-ISOLATION-STAGE-2 {
		oifname "docker0" counter packets 0 bytes 0 drop
		counter packets 0 bytes 0 return
	}

	chain DOCKER-USER {
		counter packets 0 bytes 0 return
	}
}
table ip6 filter {
	chain INPUT {
		type filter hook input priority filter; policy accept;
		counter packets 294 bytes 114110 jump LIBVIRT_INP
	}

	chain FORWARD {
		type filter hook forward priority filter; policy accept;
		counter packets 0 bytes 0 jump LIBVIRT_FWX
		counter packets 0 bytes 0 jump LIBVIRT_FWI
		counter packets 0 bytes 0 jump LIBVIRT_FWO
	}

	chain OUTPUT {
		type filter hook output priority filter; policy accept;
		counter packets 456 bytes 209729 jump LIBVIRT_OUT
	}

	chain LIBVIRT_INP {
	}

	chain LIBVIRT_OUT {
	}

	chain LIBVIRT_FWO {
	}

	chain LIBVIRT_FWI {
	}

	chain LIBVIRT_FWX {
	}
}
table bridge filter {
	chain INPUT {
		type filter hook input priority filter; policy accept;
	}

	chain FORWARD {
		type filter hook forward priority filter; policy accept;
	}

	chain OUTPUT {
		type filter hook output priority filter; policy accept;
	}
}
table ip nat {
	chain PREROUTING {
		type nat hook prerouting priority dstnat; policy accept;
		# xt_addrtype counter packets 0 bytes 0 jump DOCKER
	}

	chain INPUT {
		type nat hook input priority 100; policy accept;
	}

	chain POSTROUTING {
		type nat hook postrouting priority srcnat; policy accept;
		oifname != "docker0" ip saddr 172.17.0.0/16 counter packets 0 bytes 0 # xt_MASQUERADE
		counter packets 1966 bytes 321854 jump LIBVIRT_PRT
	}

	chain OUTPUT {
		type nat hook output priority -100; policy accept;
		ip daddr != 127.0.0.0/8 # xt_addrtype counter packets 0 bytes 0 jump DOCKER
	}

	chain LIBVIRT_PRT {
		ip saddr 192.168.122.0/24 ip daddr 224.0.0.0/24 counter packets 11 bytes 1206 return
		ip saddr 192.168.122.0/24 ip daddr 255.255.255.255 counter packets 0 bytes 0 return
		meta l4proto tcp ip saddr 192.168.122.0/24 ip daddr != 192.168.122.0/24 counter packets 0 bytes 0 # xt_MASQUERADE
		meta l4proto udp ip saddr 192.168.122.0/24 ip daddr != 192.168.122.0/24 counter packets 8 bytes 1560 # xt_MASQUERADE
		ip saddr 192.168.122.0/24 ip daddr != 192.168.122.0/24 counter packets 0 bytes 0 # xt_MASQUERADE
	}

	chain DOCKER {
		iifname "docker0" counter packets 0 bytes 0 return
	}
}
table ip mangle {
	chain PREROUTING {
		type filter hook prerouting priority mangle; policy accept;
	}

	chain INPUT {
		type filter hook input priority mangle; policy accept;
	}

	chain FORWARD {
		type filter hook forward priority mangle; policy accept;
	}

	chain OUTPUT {
		type route hook output priority mangle; policy accept;
	}

	chain POSTROUTING {
		type filter hook postrouting priority mangle; policy accept;
		counter packets 61902 bytes 4510318 jump LIBVIRT_PRT
	}

	chain LIBVIRT_PRT {
		oifname "virbr0" meta l4proto udp # xt_udp counter packets 0 bytes 0 # xt_CHECKSUM
	}
}
table ip6 nat {
	chain PREROUTING {
		type nat hook prerouting priority dstnat; policy accept;
	}

	chain INPUT {
		type nat hook input priority 100; policy accept;
	}

	chain POSTROUTING {
		type nat hook postrouting priority srcnat; policy accept;
		counter packets 276 bytes 147647 jump LIBVIRT_PRT
	}

	chain OUTPUT {
		type nat hook output priority -100; policy accept;
	}

	chain LIBVIRT_PRT {
	}
}
table ip6 mangle {
	chain PREROUTING {
		type filter hook prerouting priority mangle; policy accept;
	}

	chain INPUT {
		type filter hook input priority mangle; policy accept;
	}

	chain FORWARD {
		type filter hook forward priority mangle; policy accept;
	}

	chain OUTPUT {
		type route hook output priority mangle; policy accept;
	}

	chain POSTROUTING {
		type filter hook postrouting priority mangle; policy accept;
		counter packets 670 bytes 317237 jump LIBVIRT_PRT
	}

	chain LIBVIRT_PRT {
	}
}

I have little knowledge about firewalls in Linux in general (it seems all very complicated sad) and as such I don't know how to do persistent configurations, but sometimes I invoke this script I made

#!/bin/bash
nft flush ruleset
nft -f $1

what just makes it less hassle to apply one of the .conf's when I need to use my printer. I know it is very 'janky'. One such conf is

table inet filter {
	chain input {
		type filter hook input priority 0;
		ct state {established, related} accept
		ct state invalid drop
		iifname "lo" accept
		ip protocol 1 accept
		meta l4proto 58 accept
		tcp dport 25565 accept
		udp dport 25565
		ip saddr 192.168.0.44 udp dport 5353 accept
		ip saddr 192.168.0.44 tcp dport 8612 accept
		ip saddr 192.168.0.44 udp dport 8612 accept 
		ip saddr 192.168.0.44 tcp dport 22 accept
		iif virbr0 accept
		reject
	}

	chain forward {
		type filter hook forward priority security; policy drop;
		mark 1 accept
	}

	chain output {
		type filter hook output priority 0; policy accept;
	}
}

and judging by the line

iif virbr0 accept

I did some vm-related config here in the past (but applying the .conf does not help me now).

I did a traceroute (tracert on the guest) to archlinux.org and strangely the ip was found (dns must be working right?) but the second hop (which was after 192.168.122.1) timed out. I also tried with 1.1.1.1 with identical result.

The vhost_net module is loaded. I think that has effect on the routing of packets through the machine.

I would very much appreciate some help with this if it is even possible to use NFTables and libvirt together.


P.S. I am trying to achieve minimum latency, maximum performance as this will be a gaming vm, as I understand virtio is right for this purpose but please let me know if I am wrong here. Of course I will provide logs, command outputs, etc that I have missed here.

EDIT: I have backed up then flushed the ruleset, and rebooted. Still no luck.

Last edited by kinru (2020-12-13 00:24:57)

Offline

#2 2020-12-13 01:32:27

kinru
Member
From: East Coast USA
Registered: 2019-03-23
Posts: 99

Re: Libvirt with nftables and NO firewalld

So, I did systemctl disable nftables and rebooted and the guest has internet. Great... however I would like to keep nftables and would rather not just operate with no firewall. I don't like the whole iptables system and would hate to be forced to use it. Maybe it is not so bad though...
Still, anything to get this working with NFTables is appreciated. smile

Offline

#3 2020-12-13 11:10:41

nl6720
Wiki Admin
Registered: 2016-07-02
Posts: 262

Re: Libvirt with nftables and NO firewalld

Install iptables-nft. It's an implementation of iptables/arptables/ebtables utilities that uses nftables.

Offline

#4 2020-12-13 16:32:02

kinru
Member
From: East Coast USA
Registered: 2019-03-23
Posts: 99

Re: Libvirt with nftables and NO firewalld

Hello, I already had iptables-nft installed when I had this problem.

Offline

#5 2020-12-13 20:40:10

Zod
Member
From: Hoosiertucky
Registered: 2019-03-10
Posts: 451

Re: Libvirt with nftables and NO firewalld

Disclaimer: Can't read firewall rules, didn't try. I'm just giving my experience migrating from (disabled) iptables, (installed but not configured by me) ebtables > nftables & firewalld. I use Qemu/kvm and virt-manager.

I installed iptables-nft which removed iptables and ebtables. iptables-nft provides iptables, arptables and ebtables.

I then installed firewalld which brings along nftables.

Looking at the firewalld GUI the firewall is set to allow *all* out and allow all inbound in the context that is a response to a request from inside the firewall. It is set to allow ssh and ip6 client (solicitation ?) in.

If I were you, I might install firewalld, check that the firewall is configured as above (using the GUI), and see if your vm's work.

If so, you could output the rules and see how that jibes with what you had without firewalld.

Last edited by Zod (2020-12-13 20:46:30)

Offline

#6 2020-12-13 21:59:52

loqs
Member
Registered: 2014-03-06
Posts: 13,005

Re: Libvirt with nftables and NO firewalld

I suspect you need to manually add the equivalent of https://github.com/libvirt/libvirt/comm … 68874da088 to your nftables rules.

Offline

#7 2020-12-14 02:58:41

kinru
Member
From: East Coast USA
Registered: 2019-03-23
Posts: 99

Re: Libvirt with nftables and NO firewalld

What I think the link loqs has sent (https://github.com/libvirt/libvirt/comm … 68874da088) is talking about creating a new firewalld zone which will be default accept (since it can't accept forwarding packets? not sure) just for libvirt to use. Therefore I don't think Zod suggestion of exporting the rules will work since the features are only present in firewalld.
Of course, I am not even close to being a firewall expert, they always give me problems! So I could be wrong about not being able to export those rules. In conclusion I might just have to install firewalld... not exactly happy but if it works it works.

P.S. It is too bad, I will have online classes all week and can't risk messing up my internet beforehand... so I probably will not get to try this until later tomorrow, maybe even Wednesday. Sorry for delay and thank you for your help.

Offline

#8 2020-12-14 17:04:52

Lone_Wolf
Member
From: Netherlands, Europe
Registered: 2005-10-04
Posts: 8,908

Re: Libvirt with nftables and NO firewalld

docker0		8000.0242a5c58577	no	

docker also adds firewall rules, try disabling docker and check if things work then.


Disliking systemd intensely, but not satisfied with alternatives so focusing on taming systemd.

(A works at time B)  && (time C > time B ) ≠  (A works at time C)

Offline

#9 2020-12-26 17:00:23

kinru
Member
From: East Coast USA
Registered: 2019-03-23
Posts: 99

Re: Libvirt with nftables and NO firewalld

Sorry for the very late reply. I discovered that what I needed the VM for was discouraged (by the game manufacturer, they don't want their games be played in a vm) and I forgot about my post. I don't want to just abandon this topic with no notification, as it has happened to me before and is rather annoying. Certainly I will pick this up in short order (for resource to future problem holders) however I am doing another project at the moment but it should not be long. Thank you for help.

Offline

#10 2020-12-31 18:07:16

kinru
Member
From: East Coast USA
Registered: 2019-03-23
Posts: 99

Re: Libvirt with nftables and NO firewalld

I removed docker and reinstalled nftables and brige-utils. No luck.
However, I noticed that when I change the Network Source field to NAT, the IP Address field becomes populated, and that ip matches the subnet masks in the nft ruleset. Still no internet, though.
I want to confirm that it is correct, when using Network Source "Bridge device" that the device name is set to "virbr0" or do I need to create another bridge.
The current output of sudo nft list ruleset is

table ip filter {
	chain INPUT {
		type filter hook input priority filter; policy accept;
		counter packets 15241 bytes 20532558 jump LIBVIRT_INP
	}

	chain FORWARD {
		type filter hook forward priority filter; policy accept;
		counter packets 0 bytes 0 jump LIBVIRT_FWX
		counter packets 0 bytes 0 jump LIBVIRT_FWI
		counter packets 0 bytes 0 jump LIBVIRT_FWO
	}

	chain OUTPUT {
		type filter hook output priority filter; policy accept;
		counter packets 9333 bytes 635544 jump LIBVIRT_OUT
	}

	chain LIBVIRT_INP {
		iifname "virbr0" meta l4proto udp # xt_udp counter packets 0 bytes 0 accept
		iifname "virbr0" meta l4proto tcp # xt_tcp counter packets 0 bytes 0 accept
		iifname "virbr0" meta l4proto udp # xt_udp counter packets 0 bytes 0 accept
		iifname "virbr0" meta l4proto tcp # xt_tcp counter packets 0 bytes 0 accept
	}

	chain LIBVIRT_OUT {
		oifname "virbr0" meta l4proto udp # xt_udp counter packets 0 bytes 0 accept
		oifname "virbr0" meta l4proto tcp # xt_tcp counter packets 0 bytes 0 accept
		oifname "virbr0" meta l4proto udp # xt_udp counter packets 0 bytes 0 accept
		oifname "virbr0" meta l4proto tcp # xt_tcp counter packets 0 bytes 0 accept
	}

	chain LIBVIRT_FWO {
		iifname "virbr0" ip saddr 192.168.122.0/24 counter packets 0 bytes 0 accept
		iifname "virbr0" counter packets 0 bytes 0 # xt_REJECT
	}

	chain LIBVIRT_FWI {
		oifname "virbr0" ip daddr 192.168.122.0/24 # xt_conntrack counter packets 0 bytes 0 accept
		oifname "virbr0" counter packets 0 bytes 0 # xt_REJECT
	}

	chain LIBVIRT_FWX {
		iifname "virbr0" oifname "virbr0" counter packets 0 bytes 0 accept
	}
}
table ip6 filter {
	chain INPUT {
		type filter hook input priority filter; policy accept;
		counter packets 103 bytes 18394 jump LIBVIRT_INP
	}

	chain FORWARD {
		type filter hook forward priority filter; policy accept;
		counter packets 0 bytes 0 jump LIBVIRT_FWX
		counter packets 0 bytes 0 jump LIBVIRT_FWI
		counter packets 0 bytes 0 jump LIBVIRT_FWO
	}

	chain OUTPUT {
		type filter hook output priority filter; policy accept;
		counter packets 172 bytes 39347 jump LIBVIRT_OUT
	}

	chain LIBVIRT_INP {
	}

	chain LIBVIRT_OUT {
	}

	chain LIBVIRT_FWO {
	}

	chain LIBVIRT_FWI {
	}

	chain LIBVIRT_FWX {
	}
}
table bridge filter {
	chain INPUT {
		type filter hook input priority filter; policy accept;
	}

	chain FORWARD {
		type filter hook forward priority filter; policy accept;
	}

	chain OUTPUT {
		type filter hook output priority filter; policy accept;
	}
}
table ip nat {
	chain PREROUTING {
		type nat hook prerouting priority dstnat; policy accept;
	}

	chain INPUT {
		type nat hook input priority 100; policy accept;
	}

	chain POSTROUTING {
		type nat hook postrouting priority srcnat; policy accept;
		counter packets 256 bytes 24320 jump LIBVIRT_PRT
	}

	chain OUTPUT {
		type nat hook output priority -100; policy accept;
	}

	chain LIBVIRT_PRT {
		ip saddr 192.168.122.0/24 ip daddr 224.0.0.0/24 counter packets 3 bytes 142 return
		ip saddr 192.168.122.0/24 ip daddr 255.255.255.255 counter packets 0 bytes 0 return
		meta l4proto tcp ip saddr 192.168.122.0/24 ip daddr != 192.168.122.0/24 counter packets 0 bytes 0 # xt_MASQUERADE
		meta l4proto udp ip saddr 192.168.122.0/24 ip daddr != 192.168.122.0/24 counter packets 2 bytes 390 # xt_MASQUERADE
		ip saddr 192.168.122.0/24 ip daddr != 192.168.122.0/24 counter packets 0 bytes 0 # xt_MASQUERADE
	}
}
table ip mangle {
	chain PREROUTING {
		type filter hook prerouting priority mangle; policy accept;
	}

	chain INPUT {
		type filter hook input priority mangle; policy accept;
	}

	chain FORWARD {
		type filter hook forward priority mangle; policy accept;
	}

	chain OUTPUT {
		type route hook output priority mangle; policy accept;
	}

	chain POSTROUTING {
		type filter hook postrouting priority mangle; policy accept;
		counter packets 9389 bytes 648056 jump LIBVIRT_PRT
	}

	chain LIBVIRT_PRT {
		oifname "virbr0" meta l4proto udp # xt_udp counter packets 0 bytes 0 # xt_CHECKSUM
	}
}
table ip6 nat {
	chain PREROUTING {
		type nat hook prerouting priority dstnat; policy accept;
	}

	chain INPUT {
		type nat hook input priority 100; policy accept;
	}

	chain POSTROUTING {
		type nat hook postrouting priority srcnat; policy accept;
		counter packets 0 bytes 0 jump LIBVIRT_PRT
	}

	chain OUTPUT {
		type nat hook output priority -100; policy accept;
	}

	chain LIBVIRT_PRT {
	}
}
table ip6 mangle {
	chain PREROUTING {
		type filter hook prerouting priority mangle; policy accept;
	}

	chain INPUT {
		type filter hook input priority mangle; policy accept;
	}

	chain FORWARD {
		type filter hook forward priority mangle; policy accept;
	}

	chain OUTPUT {
		type route hook output priority mangle; policy accept;
	}

	chain POSTROUTING {
		type filter hook postrouting priority mangle; policy accept;
		counter packets 199 bytes 46012 jump LIBVIRT_PRT
	}

	chain LIBVIRT_PRT {
	}
}

Offline

Board footer

Powered by FluxBB