git.iko.soy/typeable/nixpkgs
on master help sign in
Merge pull request #53033 from netixx/openvswitch-improved-systemd

openvswitch: better integration with systemd

commit1ddb140d95c6a1ab914f1d9f3d5169b12074c333 authorJörg Thalheim <Mic92@users.noreply.github.com> dateFri 21 Feb 2020, 08:24 (7 years ago) parentc38529dcd3597b
6 files changed+151−34
nixos/modules/services/networking/dhcpcd.nix+1−1 nixos/modules/tasks/network-interfaces-scripted.nix+31−18 nixos/modules/tasks/network-interfaces-systemd.nix+62−7 nixos/modules/tasks/network-interfaces.nix+50−6 nixos/modules/virtualisation/libvirtd.nix+1−1 nixos/modules/virtualisation/openvswitch.nix+6−1
nixos/modules/services/networking/dhcpcd.nix+1−1
@@ -19,7 +19,7 @@ let
1919 map (i: i.name) (filter (i: if i.useDHCP != null then !i.useDHCP else i.ipv4.addresses != [ ]) interfaces)
2020 ++ mapAttrsToList (i: _: i) config.networking.sits
2121 ++ concatLists (attrValues (mapAttrs (n: v: v.interfaces) config.networking.bridges))
22− ++ concatLists (attrValues (mapAttrs (n: v: v.interfaces) config.networking.vswitches))
22+ ++ flatten (concatMap (i: attrNames (filterAttrs (_: config: config.type != "internal") i.interfaces)) (attrValues config.networking.vswitches))
2323 ++ concatLists (attrValues (mapAttrs (n: v: v.interfaces) config.networking.bonds))
2424 ++ config.networking.dhcpcd.denyInterfaces;
2525
nixos/modules/tasks/network-interfaces-scripted.nix+31−18
@@ -10,7 +10,7 @@ let
1010
1111 slaves = concatMap (i: i.interfaces) (attrValues cfg.bonds)
1212 ++ concatMap (i: i.interfaces) (attrValues cfg.bridges)
13− ++ concatMap (i: i.interfaces) (attrValues cfg.vswitches)
13+ ++ concatMap (i: attrNames (filterAttrs (_: config: config.type != "internal") i.interfaces)) (attrValues cfg.vswitches)
1414 ++ concatMap (i: [i.interface]) (attrValues cfg.macvlans)
1515 ++ concatMap (i: [i.interface]) (attrValues cfg.vlans);
1616
@@ -336,34 +336,47 @@ let
336336
337337 createVswitchDevice = n: v: nameValuePair "${n}-netdev"
338338 (let
339− deps = concatLists (map deviceDependency v.interfaces);
339+ deps = concatLists (map deviceDependency (attrNames (filterAttrs (_: config: config.type != "internal") v.interfaces)));
340+ internalConfigs = concatMap (i: ["network-link-${i}.service" "network-addresses-${i}.service"]) (attrNames (filterAttrs (_: config: config.type == "internal") v.interfaces));
340341 ofRules = pkgs.writeText "vswitch-${n}-openFlowRules" v.openFlowRules;
341342 in
342343 { description = "Open vSwitch Interface ${n}";
343− wantedBy = [ "network-setup.service" "vswitchd.service" ] ++ deps;
344− bindsTo = [ "vswitchd.service" (subsystemDevice n) ] ++ deps;
345− partOf = [ "network-setup.service" "vswitchd.service" ];
346− after = [ "network-pre.target" "vswitchd.service" ] ++ deps;
347− before = [ "network-setup.service" ];
344+ wantedBy = [ "network-setup.service" (subsystemDevice n) ] ++ internalConfigs;
345+ # before = [ "network-setup.service" ];
346+ # should work without internalConfigs dependencies because address/link configuration depends
347+ # on the device, which is created by ovs-vswitchd with type=internal, but it does not...
348+ before = [ "network-setup.service" ] ++ internalConfigs;
349+ partOf = [ "network-setup.service" ]; # shutdown the bridge when network is shutdown
350+ bindsTo = [ "ovs-vswitchd.service" ]; # requires ovs-vswitchd to be alive at all times
351+ after = [ "network-pre.target" "ovs-vswitchd.service" ] ++ deps; # start switch after physical interfaces and vswitch daemon
352+ wants = deps; # if one or more interface fails, the switch should continue to run
348353 serviceConfig.Type = "oneshot";
349354 serviceConfig.RemainAfterExit = true;
350355 path = [ pkgs.iproute config.virtualisation.vswitch.package ];
356+ preStart = ''
357+ echo "Resetting Open vSwitch ${n}..."
358+ ovs-vsctl --if-exists del-br ${n} -- add-br ${n} \
359+ -- set bridge ${n} protocols=${concatStringsSep "," v.supportedOpenFlowVersions}
360+ '';
351361 script = ''
352− echo "Removing old Open vSwitch ${n}..."
353− ovs-vsctl --if-exists del-br ${n}
354−
355− echo "Adding Open vSwitch ${n}..."
356− ovs-vsctl -- add-br ${n} ${concatMapStrings (i: " -- add-port ${n} ${i}") v.interfaces} \
362+ echo "Configuring Open vSwitch ${n}..."
363+ ovs-vsctl ${concatStrings (mapAttrsToList (name: config: " -- add-port ${n} ${name}" + optionalString (config.vlan != null) " tag=${toString config.vlan}") v.interfaces)} \
364+ ${concatStrings (mapAttrsToList (name: config: optionalString (config.type != null) " -- set interface ${name} type=${config.type}") v.interfaces)} \
357365 ${concatMapStrings (x: " -- set-controller ${n} " + x) v.controllers} \
358366 ${concatMapStrings (x: " -- " + x) (splitString "\n" v.extraOvsctlCmds)}
359367
368+
360369 echo "Adding OpenFlow rules for Open vSwitch ${n}..."
361− ovs-ofctl add-flows ${n} ${ofRules}
370+ ovs-ofctl --protocols=${v.openFlowVersion} add-flows ${n} ${ofRules}
362371 '';
363372 postStop = ''
373+ echo "Cleaning Open vSwitch ${n}"
374+ echo "Shuting down internal ${n} interface"
364375 ip link set ${n} down || true
365− ovs-ofctl del-flows ${n} || true
366− ovs-vsctl --if-exists del-br ${n}
376+ echo "Deleting flows for ${n}"
377+ ovs-ofctl --protocols=${v.openFlowVersion} del-flows ${n} || true
378+ echo "Deleting Open vSwitch ${n}"
379+ ovs-vsctl --if-exists del-br ${n} || true
367380 '';
368381 });
369382
@@ -476,9 +489,9 @@ let
476489 # Remove Dead Interfaces
477490 ip link show "${n}" >/dev/null 2>&1 && ip link delete "${n}"
478491 ip link add link "${v.interface}" name "${n}" type vlan id "${toString v.id}"
479−
480− # We try to bring up the logical VLAN interface. If the master
481− # interface the logical interface is dependent upon is not up yet we will
492+
493+ # We try to bring up the logical VLAN interface. If the master
494+ # interface the logical interface is dependent upon is not up yet we will
482495 # fail to immediately bring up the logical interface. The resulting logical
483496 # interface will brought up later when the master interface is up.
484497 ip link set "${n}" up || true
nixos/modules/tasks/network-interfaces-systemd.nix+62−7
@@ -1,4 +1,4 @@
1−{ config, lib, utils, ... }:
1+{ config, lib, utils, pkgs, ... }:
22
33with utils;
44with lib;
@@ -18,7 +18,10 @@ let
1818 concatLists (map (bond: bond.interfaces) (attrValues cfg.bonds))
1919 ++ concatLists (map (bridge: bridge.interfaces) (attrValues cfg.bridges))
2020 ++ map (sit: sit.dev) (attrValues cfg.sits)
21− ++ map (vlan: vlan.interface) (attrValues cfg.vlans);
21+ ++ map (vlan: vlan.interface) (attrValues cfg.vlans)
22+ # add dependency to physical or independently created vswitch member interface
23+ # TODO: warn the user that any address configured on those interfaces will be useless
24+ ++ concatMap (i: attrNames (filterAttrs (_: config: config.type != "internal") i.interfaces)) (attrValues cfg.vswitches);
2225
2326in
2427
@@ -51,11 +54,6 @@ in
5154
5255 networking.dhcpcd.enable = mkDefault false;
5356
54− systemd.services.network-local-commands = {
55− after = [ "systemd-networkd.service" ];
56− bindsTo = [ "systemd-networkd.service" ];
57− };
58−
5957 systemd.network =
6058 let
6159 domains = cfg.search ++ (optional (cfg.domain != null) cfg.domain);
@@ -233,6 +231,63 @@ in
233231 # This forces the network interface creator to initialize slaves.
234232 networking.interfaces = listToAttrs (map (i: nameValuePair i { }) slaves);
235233
234+ systemd.services = let
235+ # We must escape interfaces due to the systemd interpretation
236+ subsystemDevice = interface:
237+ "sys-subsystem-net-devices-${escapeSystemdPath interface}.device";
238+ # support for creating openvswitch switches
239+ createVswitchDevice = n: v: nameValuePair "${n}-netdev"
240+ (let
241+ deps = map subsystemDevice (attrNames (filterAttrs (_: config: config.type != "internal") v.interfaces));
242+ ofRules = pkgs.writeText "vswitch-${n}-openFlowRules" v.openFlowRules;
243+ in
244+ { description = "Open vSwitch Interface ${n}";
245+ wantedBy = [ "network.target" (subsystemDevice n) ];
246+ # and create bridge before systemd-networkd starts because it might create internal interfaces
247+ before = [ "systemd-networkd.service" ];
248+ # shutdown the bridge when network is shutdown
249+ partOf = [ "network.target" ];
250+ # requires ovs-vswitchd to be alive at all times
251+ bindsTo = [ "ovs-vswitchd.service" ];
252+ # start switch after physical interfaces and vswitch daemon
253+ after = [ "network-pre.target" "ovs-vswitchd.service" ] ++ deps;
254+ wants = deps; # if one or more interface fails, the switch should continue to run
255+ serviceConfig.Type = "oneshot";
256+ serviceConfig.RemainAfterExit = true;
257+ path = [ pkgs.iproute config.virtualisation.vswitch.package ];
258+ preStart = ''
259+ echo "Resetting Open vSwitch ${n}..."
260+ ovs-vsctl --if-exists del-br ${n} -- add-br ${n} \
261+ -- set bridge ${n} protocols=${concatStringsSep "," v.supportedOpenFlowVersions}
262+ '';
263+ script = ''
264+ echo "Configuring Open vSwitch ${n}..."
265+ ovs-vsctl ${concatStrings (mapAttrsToList (name: config: " -- add-port ${n} ${name}" + optionalString (config.vlan != null) " tag=${toString config.vlan}") v.interfaces)} \
266+ ${concatStrings (mapAttrsToList (name: config: optionalString (config.type != null) " -- set interface ${name} type=${config.type}") v.interfaces)} \
267+ ${concatMapStrings (x: " -- set-controller ${n} " + x) v.controllers} \
268+ ${concatMapStrings (x: " -- " + x) (splitString "\n" v.extraOvsctlCmds)}
269+
270+
271+ echo "Adding OpenFlow rules for Open vSwitch ${n}..."
272+ ovs-ofctl --protocols=${v.openFlowVersion} add-flows ${n} ${ofRules}
273+ '';
274+ postStop = ''
275+ echo "Cleaning Open vSwitch ${n}"
276+ echo "Shuting down internal ${n} interface"
277+ ip link set ${n} down || true
278+ echo "Deleting flows for ${n}"
279+ ovs-ofctl --protocols=${v.openFlowVersion} del-flows ${n} || true
280+ echo "Deleting Open vSwitch ${n}"
281+ ovs-vsctl --if-exists del-br ${n} || true
282+ '';
283+ });
284+ in mapAttrs' createVswitchDevice cfg.vswitches
285+ // {
286+ "network-local-commands" = {
287+ after = [ "systemd-networkd.service" ];
288+ bindsTo = [ "systemd-networkd.service" ];
289+ };
290+ };
236291 };
237292
238293}
nixos/modules/tasks/network-interfaces.nix+50−6
@@ -13,7 +13,7 @@ let
1313
1414 slaves = concatMap (i: i.interfaces) (attrValues cfg.bonds)
1515 ++ concatMap (i: i.interfaces) (attrValues cfg.bridges)
16− ++ concatMap (i: i.interfaces) (attrValues cfg.vswitches);
16+ ++ concatMap (i: attrNames (filterAttrs (name: config: ! (config.type == "internal" || hasAttr name cfg.interfaces)) i.interfaces)) (attrValues cfg.vswitches);
1717
1818 slaveIfs = map (i: cfg.interfaces.${i}) (filter (i: cfg.interfaces ? ${i}) slaves);
1919
@@ -336,6 +336,32 @@ let
336336
337337 };
338338
339+ vswitchInterfaceOpts = {name, ...}: {
340+
341+ options = {
342+
343+ name = mkOption {
344+ description = "Name of the interface";
345+ example = "eth0";
346+ type = types.str;
347+ };
348+
349+ vlan = mkOption {
350+ description = "Vlan tag to apply to interface";
351+ example = 10;
352+ type = types.nullOr types.int;
353+ default = null;
354+ };
355+
356+ type = mkOption {
357+ description = "Openvswitch type to assign to interface";
358+ example = "internal";
359+ type = types.nullOr types.str;
360+ default = null;
361+ };
362+ };
363+ };
364+
339365 hexChars = stringToCharacters "0123456789abcdef";
340366
341367 isHexString = s: all (c: elem c hexChars) (stringToCharacters (toLower s));
@@ -486,8 +512,8 @@ in
486512 networking.vswitches = mkOption {
487513 default = { };
488514 example =
489− { vs0.interfaces = [ "eth0" "eth1" ];
490− vs1.interfaces = [ "eth2" "wlan0" ];
515+ { vs0.interfaces = { eth0 = { }; lo1 = { type="internal"; }; };
516+ vs1.interfaces = [ { name = "eth2"; } { name = "lo2"; type="internal"; } ];
491517 };
492518 description =
493519 ''
@@ -504,9 +530,8 @@ in
504530
505531 interfaces = mkOption {
506532 example = [ "eth0" "eth1" ];
507− type = types.listOf types.str;
508− description =
509− "The physical network interfaces connected by the vSwitch.";
533+ description = "The physical network interfaces connected by the vSwitch.";
534+ type = with types; loaOf (submodule vswitchInterfaceOpts);
510535 };
511536
512537 controllers = mkOption {
@@ -530,6 +555,25 @@ in
530555 '';
531556 };
532557
558+ # TODO: custom "openflow version" type, with list from existing openflow protocols
559+ supportedOpenFlowVersions = mkOption {
560+ type = types.listOf types.str;
561+ example = [ "OpenFlow10" "OpenFlow13" "OpenFlow14" ];
562+ default = [ "OpenFlow13" ];
563+ description = ''
564+ Supported versions to enable on this switch.
565+ '';
566+ };
567+
568+ # TODO: use same type as elements from supportedOpenFlowVersions
569+ openFlowVersion = mkOption {
570+ type = types.str;
571+ default = "OpenFlow13";
572+ description = ''
573+ Version of OpenFlow protocol to use when communicating with the switch internally (e.g. with <literal>openFlowRules</literal>).
574+ '';
575+ };
576+
533577 extraOvsctlCmds = mkOption {
534578 type = types.lines;
535579 default = "";
nixos/modules/virtualisation/libvirtd.nix+1−1
@@ -219,7 +219,7 @@ in {
219219 wantedBy = [ "multi-user.target" ];
220220 requires = [ "libvirtd-config.service" ];
221221 after = [ "systemd-udev-settle.service" "libvirtd-config.service" ]
222− ++ optional vswitch.enable "vswitchd.service";
222+ ++ optional vswitch.enable "ovs-vswitchd.service";
223223
224224 environment.LIBVIRTD_ARGS = ''--config "${configFile}" ${concatStringsSep " " cfg.extraOptions}'';
225225
nixos/modules/virtualisation/openvswitch.nix+6−1
@@ -124,7 +124,7 @@ in {
124124 '';
125125 };
126126
127− systemd.services.vswitchd = {
127+ systemd.services.ovs-vswitchd = {
128128 description = "Open_vSwitch Daemon";
129129 wantedBy = [ "multi-user.target" ];
130130 bindsTo = [ "ovsdb.service" ];
@@ -139,6 +139,8 @@ in {
139139 PIDFile = "/run/openvswitch/ovs-vswitchd.pid";
140140 # Use service type 'forking' to correctly determine when vswitchd is ready.
141141 Type = "forking";
142+ Restart = "always";
143+ RestartSec = 3;
142144 };
143145 };
144146
@@ -182,4 +184,7 @@ in {
182184 '';
183185 };
184186 })]));
187+
188+ meta.maintainers = with maintainers; [ netixx ];
189+
185190}