Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

kubelet: add initial support for cgroupv2 #85218

Merged
merged 1 commit into from Mar 26, 2020
Merged
Changes from all commits
Commits
File filter...
Filter file types
Jump to…
Jump to file
Failed to load files.

Always

Just for now

@@ -21,6 +21,7 @@ import (
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"

@@ -153,10 +154,11 @@ func (l *libcontainerAdapter) newManager(cgroups *libcontainerconfigs.Cgroup, pa
if !cgroupsystemd.UseSystemd() {
panic("systemd cgroup manager not available")
}
return &cgroupsystemd.LegacyManager{
Cgroups: cgroups,
Paths: paths,
}, nil
f, err := cgroupsystemd.NewSystemdCgroupsManager()

This comment has been minimized.

Copy link
@derekwaynecarr

derekwaynecarr Mar 12, 2020

Member

does this manager work on v1 and v2 hosts?

This comment has been minimized.

Copy link
@derekwaynecarr

derekwaynecarr Mar 12, 2020

Member

i need to check the runc code, but is this abstracing legacy from unified manager?

This comment has been minimized.

Copy link
@giuseppe

giuseppe Mar 12, 2020

Author Member

yes, NewSystemdCgroupsManager() checks for the cgroup version used:

https://github.com/opencontainers/runc/blob/master/libcontainer/cgroups/systemd/apply_systemd.go#L118-L136

If we are running on cgroup v1, it will use cgroupsystemd.LegacyManager

This comment has been minimized.

Copy link
@derekwaynecarr

derekwaynecarr Mar 26, 2020

Member

thanks for pointer, and noting that i confirmed this is in the vendored version.

if err != nil {
return nil, err
}
return f(cgroups, paths), nil
}
return nil, fmt.Errorf("invalid cgroup manager configuration")
}
@@ -254,6 +256,7 @@ func (m *cgroupManagerImpl) Exists(name CgroupName) bool {
// in https://github.com/opencontainers/runc/issues/1440
// once resolved, we can remove this code.
whitelistControllers := sets.NewString("cpu", "cpuacct", "cpuset", "memory", "systemd")

if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportPodPidsLimit) || utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportNodePidsLimit) {
whitelistControllers.Insert("pids")
}
@@ -369,14 +372,31 @@ func (m *cgroupManagerImpl) toResources(resourceConfig *ResourceConfig) *libcont
if resourceConfig.Memory != nil {
resources.Memory = *resourceConfig.Memory
}
if resourceConfig.CpuShares != nil {
resources.CpuShares = *resourceConfig.CpuShares
}
if resourceConfig.CpuQuota != nil {
resources.CpuQuota = *resourceConfig.CpuQuota
}
if resourceConfig.CpuPeriod != nil {
resources.CpuPeriod = *resourceConfig.CpuPeriod
if libcontainercgroups.IsCgroup2UnifiedMode() {
if resourceConfig.CpuShares != nil {
// Convert from the range [2-262144] to [1-10000]
resources.CpuWeight = (1 + ((*resourceConfig.CpuShares-2)*9999)/262142)

This comment has been minimized.

Copy link
@derekwaynecarr

derekwaynecarr Mar 26, 2020

Member

minor nit: may be useful to move this conversion into a helper that we can then unit test in a follow-on.

This comment has been minimized.

Copy link
@giuseppe

giuseppe Mar 27, 2020

Author Member

PR here: #89567

}

quota := "max"
period := "100000"
if resourceConfig.CpuQuota != nil {
quota = strconv.FormatInt(*resourceConfig.CpuQuota, 10)
}
if resourceConfig.CpuPeriod != nil {
period = strconv.FormatUint(*resourceConfig.CpuPeriod, 10)
}
resources.CpuMax = fmt.Sprintf("%s %s", quota, period)
} else {
if resourceConfig.CpuShares != nil {
resources.CpuShares = *resourceConfig.CpuShares
}
if resourceConfig.CpuQuota != nil {
resources.CpuQuota = *resourceConfig.CpuQuota
}
if resourceConfig.CpuPeriod != nil {
resources.CpuPeriod = *resourceConfig.CpuPeriod
}
}
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportPodPidsLimit) || utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportNodePidsLimit) {
if resourceConfig.PidsLimit != nil {
@@ -161,6 +161,10 @@ func validateSystemRequirements(mountUtil mount.Interface) (features, error) {
return f, fmt.Errorf("%s - %v", localErr, err)
}

if cgroups.IsCgroup2UnifiedMode() {
return f, nil
}

expectedCgroups := sets.NewString("cpu", "cpuacct", "cpuset", "memory")
for _, mountPoint := range mountPoints {
if mountPoint.Type == cgroupMountType {
@@ -884,6 +888,11 @@ func getContainer(pid int) (string, error) {
return "", err
}

if cgroups.IsCgroup2UnifiedMode() {
c, _ := cgs[""]
return c, nil
}

cpu, found := cgs["cpu"]
if !found {
return "", cgroups.NewNotFoundError("cpu")
@@ -19,9 +19,11 @@ package cm
import (
"bufio"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"

libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"

@@ -32,6 +34,7 @@ import (
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
kubefeatures "k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/cm/util"
)

const (
@@ -181,8 +184,8 @@ func ResourceConfigForPod(pod *v1.Pod, enforceCPULimits bool, cpuPeriod uint64)
return result
}

// GetCgroupSubsystems returns information about the mounted cgroup subsystems
func GetCgroupSubsystems() (*CgroupSubsystems, error) {
// getCgroupSubsystemsV1 returns information about the mounted cgroup v1 subsystems
func getCgroupSubsystemsV1() (*CgroupSubsystems, error) {
// get all cgroup mounts.
allCgroups, err := libcontainercgroups.GetCgroupMounts(true)
if err != nil {
@@ -203,6 +206,41 @@ func GetCgroupSubsystems() (*CgroupSubsystems, error) {
}, nil
}

// getCgroupSubsystemsV2 returns information about the enabled cgroup v2 subsystems
func getCgroupSubsystemsV2() (*CgroupSubsystems, error) {
content, err := ioutil.ReadFile(filepath.Join(util.CgroupRoot, "cgroup.controllers"))
if err != nil {
return nil, err
}

mounts := []libcontainercgroups.Mount{}
controllers := strings.Fields(string(content))
mountPoints := make(map[string]string, len(controllers))
for _, controller := range controllers {
mountPoints[controller] = util.CgroupRoot
m := libcontainercgroups.Mount{
Mountpoint: util.CgroupRoot,
Root: util.CgroupRoot,
Subsystems: []string{controller},
}
mounts = append(mounts, m)
}

return &CgroupSubsystems{
This conversation was marked as resolved by giuseppe

This comment has been minimized.

Copy link
@AkihiroSuda

AkihiroSuda Nov 13, 2019

Contributor

can we ditch "subsystem mountpoint" concept for v2 codes?

runc and containerd are trying to eliminate all "subsystem" concept: opencontainers/runc#2169 https://godoc.org/github.com/containerd/cgroups/v2

This comment has been minimized.

Copy link
@AkihiroSuda

This comment has been minimized.

Copy link
@giuseppe

giuseppe Mar 9, 2020

Author Member

is this refactoring something that could be addressed later? I'd like to get the current version merged as soon as possible, so that we can already start using it

This comment has been minimized.

Copy link
@AkihiroSuda

AkihiroSuda Mar 9, 2020

Contributor

SGTM

Mounts: mounts,
MountPoints: mountPoints,
}, nil
}

// GetCgroupSubsystems returns information about the mounted cgroup subsystems
func GetCgroupSubsystems() (*CgroupSubsystems, error) {
if libcontainercgroups.IsCgroup2UnifiedMode() {
return getCgroupSubsystemsV2()
}

return getCgroupSubsystemsV1()
}

// getCgroupProcs takes a cgroup directory name as an argument
// reads through the cgroup's procs file and returns a list of tgid's.
// It returns an empty list if a procs file doesn't exists
@@ -23,19 +23,35 @@ import (
libcontainerutils "github.com/opencontainers/runc/libcontainer/utils"
)

const (
// CgroupRoot is the base path where cgroups are mounted
CgroupRoot = "/sys/fs/cgroup"
)

// GetPids gets pids of the desired cgroup
// Forked from opencontainers/runc/libcontainer/cgroup/fs.Manager.GetPids()
func GetPids(cgroupPath string) ([]int, error) {
dir, err := getCgroupPath(cgroupPath)
if err != nil {
return nil, err
dir := ""

if libcontainercgroups.IsCgroup2UnifiedMode() {
path, err := filepath.Rel("/", cgroupPath)
if err != nil {
return nil, err
}
dir = filepath.Join(CgroupRoot, path)
} else {
var err error
dir, err = getCgroupV1Path(cgroupPath)
if err != nil {
return nil, err
}
}
return libcontainercgroups.GetPids(dir)
}

// getCgroupPath gets the file path to the "devices" subsystem of the desired cgroup.
// getCgroupV1Path gets the file path to the "devices" subsystem of the desired cgroup.
// cgroupPath is the path in the cgroup hierarchy.
func getCgroupPath(cgroupPath string) (string, error) {
func getCgroupV1Path(cgroupPath string) (string, error) {
cgroupPath = libcontainerutils.CleanPath(cgroupPath)

mnt, root, err := libcontainercgroups.FindCgroupMountpointAndRoot(cgroupPath, "devices")
@@ -50,16 +66,16 @@ func getCgroupPath(cgroupPath string) (string, error) {
return filepath.Join(root, mnt, cgroupPath), nil
}

parentPath, err := getCgroupParentPath(mnt, root)
parentPath, err := getCgroupV1ParentPath(mnt, root)
if err != nil {
return "", err
}

return filepath.Join(parentPath, cgroupPath), nil
}

// getCgroupParentPath gets the parent filepath to this cgroup, for resolving relative cgroup paths.
func getCgroupParentPath(mountpoint, root string) (string, error) {
// getCgroupV1ParentPath gets the parent filepath to this cgroup, for resolving relative cgroup paths.
func getCgroupV1ParentPath(mountpoint, root string) (string, error) {
// Use GetThisCgroupDir instead of GetInitCgroupDir, because the creating
// process could in container and shared pid namespace with host, and
// /proc/1/cgroup could point to whole other world of cgroups.
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.