Add missing container filtering + Address PR Comments by removing regexp and making more idiomatic choices

This commit is contained in:
Praneeth Bodduluri 2016-01-06 20:59:10 +05:30
parent b28e6c7648
commit fb47f73907
4 changed files with 72 additions and 84 deletions

View File

@ -38,35 +38,35 @@ func startApi(listenAddress string, router *mux.Router) {
}
}
func startOOMProtection(hostproc string, dockerSocket string, ticker *time.Ticker) {
log.Println("Changing OOMScore Adjust Value for this container to -800")
err := psutils.AdjustDockerOOMPriority(hostproc, "unix://"+dockerSocket, "resin-supervisor", -800, false)
if err != nil {
log.Printf(err.Error())
func startOOMProtectionTimer(hostproc string, dockerSocket string) *time.Ticker {
ticker := time.NewTicker(time.Minute * 5) //Timer runs every 5 minutes
procs := &psutils.Procs{hostproc}
log.Println("Changing oom_score_adj for the supervisor container to -800")
if err := procs.AdjustDockerOOMPriority("unix://"+dockerSocket, "resin_supervisor", -800, false); err != nil {
log.Printf("FAILED to OOM protect supervisor container: %s\n", err)
}
// Code below this could be eventually deprecated after all the devices are > 5 Jan 2016 deployment.
log.Println("Changing OOMScore Adjust Value for openvpn and connmand to -1000 if 0, every 5 minutes")
// Code below this could be eventually deprecated after all the devices are > 5 Jan 2016 deployment as this will be handled in the HOST OS.
log.Println("Changing oom_score_adj for openvpn and connmand to -1000 if 0, every 5 minutes")
// Errors are not being caught here as users could have openvpn and connmand disabled.
psutils.AdjustOOMPriorityByName(hostproc, "openvpn", -1000, true)
psutils.AdjustOOMPriorityByName(hostproc, "connmand", -1000, true)
procs.AdjustOOMPriorityByName("openvpn", -1000, true)
procs.AdjustOOMPriorityByName("connmand", -1000, true)
go func() {
for _ = range ticker.C {
psutils.AdjustOOMPriorityByName(hostproc, "openvpn", -1000, true)
psutils.AdjustOOMPriorityByName(hostproc, "connmand", -1000, true)
procs.AdjustOOMPriorityByName("openvpn", -1000, true)
procs.AdjustOOMPriorityByName("connmand", -1000, true)
}
}()
return ticker
}
func main() {
log.SetFlags(log.Lshortfile | log.LstdFlags)
log.Println("Resin Go Supervisor starting")
// Start ticker for protecting Openvpn/Connman every 5 minutes
ticker := time.NewTicker(time.Minute * 5)
defer ticker.Stop()
// Start OOMProtectionTimer for protecting Openvpn/Connman
dockerSocket := os.Getenv("DOCKER_SOCKET")
hostproc := os.Getenv("HOST_PROC")
startOOMProtection(hostproc, dockerSocket, ticker)
defer startOOMProtectionTimer(hostproc, dockerSocket).Stop()
listenAddress := os.Getenv("GOSUPER_SOCKET")
router := mux.NewRouter()

View File

@ -1,94 +1,82 @@
package psutils
import (
"bufio"
"fmt"
"os"
"io/ioutil"
"log"
"path"
"regexp"
"strconv"
"strings"
"resin-supervisor/gosuper/Godeps/_workspace/src/github.com/samalba/dockerclient"
"resin-supervisor/gosuper/Godeps/_workspace/src/github.com/shirou/gopsutil/process"
)
// Procs - psutils functions associated with the ProcfsPath
type Procs struct {
ProcfsPath string
}
// parseInt - Parse integer from string.
func parseInt(str string) (int, error) {
// strconv chokes on whitespace, go figure.
trimmed := strings.TrimSpace(str)
return strconv.Atoi(trimmed)
}
//AdjustOOMPriorityByName Adjust the OOM adj value for the process' with the given name regexp
func AdjustOOMPriorityByName(procPath string, processName string, value int64, ignoreIfNonZero bool) error {
runningProcess, err := process.Pids()
func (procs *Procs) AdjustOOMPriorityByName(processName string, value int, ignoreIfNonZero bool) error {
found := false
pids, err := process.Pids()
if err != nil {
return err
}
var processMatch = regexp.MustCompile(processName)
processFound := false
for _, pid := range runningProcess {
for _, pid := range pids {
// Find the process with the given name
pidProcess, _ := process.NewProcess(pid)
pidName, _ := pidProcess.Name()
if processMatch.MatchString(pidName) {
processFound = true
err := AdjustOOMPriority(procPath, int64(pid), value, ignoreIfNonZero)
if err != nil {
return err
}
if currProcess, err := process.NewProcess(pid); err != nil {
continue
} else if name, err := currProcess.Name(); err != nil && name != processName {
continue
} else if err := procs.AdjustOOMPriority(int(pid), value, ignoreIfNonZero); err == nil {
found = true
} else {
// Not an error but logging these for debugging.
log.Printf("Error adjusting OOM for process %s (pid %d): %s", processName, pid, err)
}
}
if processFound {
if found {
return nil
}
return fmt.Errorf("No process matches: %s\n", processName)
}
//AdjustOOMPriority Adjust the OOM adj value for the process with the given pid.
func AdjustOOMPriority(procPath string, pid int64, value int64, ignoreIfNonZero bool) error {
// Open the oom_score_adj file for the pid
oomAdjFile, err := os.OpenFile(path.Clean(procPath)+"/"+strconv.FormatInt(pid, 10)+"/oom_score_adj", os.O_RDWR, os.ModeType)
if err != nil {
return fmt.Errorf("Unable to open OOM adjust proc file for pid: %d\n", pid)
}
defer oomAdjFile.Close()
// Read the oom_score_adj value currently set
scanner := bufio.NewScanner(oomAdjFile)
scanner.Split(bufio.ScanLines)
var currentOOMString string
for scanner.Scan() {
currentOOMString = scanner.Text() // Read the OOMString
}
currentOOMValue, err := strconv.ParseInt(currentOOMString, 10, 64)
if err != nil {
func (procs *Procs) AdjustOOMPriority(pid int, value int, ignoreIfNonZero bool) error {
valueBytes := []byte(strconv.Itoa(value))
oomFile := fmt.Sprintf("%s/%d/oom_score_adj", path.Clean(procs.ProcfsPath), pid)
if currentOOMBytes, err := ioutil.ReadFile(oomFile); err != nil {
return err
} else if currentOOMValue, err := parseInt(string(currentOOMBytes)); err != nil {
return fmt.Errorf("Unable to read OOM adjust for pid: %d\n", pid)
}
if ignoreIfNonZero && currentOOMValue != 0 {
} else if ignoreIfNonZero && currentOOMValue != 0 {
return nil
}
// Write to the procfile to adjust the OOM adj value.
_, err = oomAdjFile.WriteString(strconv.FormatInt(value, 10))
if err != nil {
} else if err = ioutil.WriteFile(oomFile, valueBytes, 0644); err != nil {
return fmt.Errorf("Unable to OOM adjust for pid: %d\n", pid)
}
return nil
}
//AdjustDockerOOMPriority Adjusts the OOM Adj value for the entire docker container specified by the name. This should point to root proc filesystem
func AdjustDockerOOMPriority(procPath string, connection string, containerName string, value int64, ignoreIfNonZero bool) error {
// Connect to the docker host with the connection string
docker, err := dockerclient.NewDockerClient(connection, nil)
if err != nil {
func (procs *Procs) AdjustDockerOOMPriority(connection string, containerName string, value int, ignoreIfNonZero bool) error {
if docker, err := dockerclient.NewDockerClient(connection, nil); err != nil {
return err
}
// Get the running containers
containers, err := docker.ListContainers(false, false, "")
if err != nil {
} else if containers, err := docker.ListContainers(false, false, fmt.Sprintf(`{"name":["^/%s$"]}`, containerName)); err != nil {
return err
} else if containerInfo, err := docker.InspectContainer(containers[0].Id); err != nil {
return err
} else if err := procs.AdjustOOMPriority(containerInfo.State.Pid, value, ignoreIfNonZero); err != nil {
return err
} else {
return nil
}
for _, container := range containers {
containerInfo, err := docker.InspectContainer(container.Id)
if err != nil {
return err
}
err = AdjustOOMPriority(procPath, int64(containerInfo.State.Pid), value, ignoreIfNonZero)
if err != nil {
return err
}
}
return nil
}

21
gosuper/test_formatting.sh Normal file → Executable file
View File

@ -1,15 +1,14 @@
#!/bin/bash
DIRS=`ls -l . | egrep '^d' | awk '{print $9}'`
DIRS=`echo $DIRS | sed "s/Godeps//g"`
# and finally loop over the cleaned up directory list.
for DIR in $DIRS
do
if [[ -n "$(gofmt -l ${DIR})" ]]; then
echo "Bad formatting, run make format-gosuper to fix it."
exit 1
for dir in $(find ./* -path ./Godeps -prune -or -type d -print); do
errormessage=$(gofmt -l $dir)
if [ -n "$errormessage" ]; then
echo "$errormessage"
failed=1
fi
done
echo "Formatting test passed."
if [ -n "$failed" ]; then
echo "Bad formatting, run make format-gosuper to fix above errors."
exit 1
fi

View File

@ -13,6 +13,7 @@ checkValidKey = (s) ->
return
return s
# Defaults needed for both gosuper and node supervisor are declared in entry.sh
module.exports = config =
apiEndpoint: process.env.API_ENDPOINT ? 'https://api.resin.io'
listenPort: process.env.LISTEN_PORT ? 80