diff --git a/gosuper/gosuper/main.go b/gosuper/gosuper/main.go index cbb22c18..270a9e25 100644 --- a/gosuper/gosuper/main.go +++ b/gosuper/gosuper/main.go @@ -38,35 +38,35 @@ func startApi(listenAddress string, router *mux.Router) { } } -func startOOMProtection(hostproc string, dockerSocket string, ticker *time.Ticker) { - log.Println("Changing OOMScore Adjust Value for this container to -800") - err := psutils.AdjustDockerOOMPriority(hostproc, "unix://"+dockerSocket, "resin-supervisor", -800, false) - if err != nil { - log.Printf(err.Error()) +func startOOMProtectionTimer(hostproc string, dockerSocket string) *time.Ticker { + ticker := time.NewTicker(time.Minute * 5) //Timer runs every 5 minutes + procs := &psutils.Procs{hostproc} + log.Println("Changing oom_score_adj for the supervisor container to -800") + if err := procs.AdjustDockerOOMPriority("unix://"+dockerSocket, "resin_supervisor", -800, false); err != nil { + log.Printf("FAILED to OOM protect supervisor container: %s\n", err) } - // Code below this could be eventually deprecated after all the devices are > 5 Jan 2016 deployment. - log.Println("Changing OOMScore Adjust Value for openvpn and connmand to -1000 if 0, every 5 minutes") + // Code below this could be eventually deprecated after all the devices are > 5 Jan 2016 deployment as this will be handled in the HOST OS. + log.Println("Changing oom_score_adj for openvpn and connmand to -1000 if 0, every 5 minutes") // Errors are not being caught here as users could have openvpn and connmand disabled. - psutils.AdjustOOMPriorityByName(hostproc, "openvpn", -1000, true) - psutils.AdjustOOMPriorityByName(hostproc, "connmand", -1000, true) + procs.AdjustOOMPriorityByName("openvpn", -1000, true) + procs.AdjustOOMPriorityByName("connmand", -1000, true) go func() { for _ = range ticker.C { - psutils.AdjustOOMPriorityByName(hostproc, "openvpn", -1000, true) - psutils.AdjustOOMPriorityByName(hostproc, "connmand", -1000, true) + procs.AdjustOOMPriorityByName("openvpn", -1000, true) + procs.AdjustOOMPriorityByName("connmand", -1000, true) } }() + return ticker } func main() { log.SetFlags(log.Lshortfile | log.LstdFlags) log.Println("Resin Go Supervisor starting") - // Start ticker for protecting Openvpn/Connman every 5 minutes - ticker := time.NewTicker(time.Minute * 5) - defer ticker.Stop() + // Start OOMProtectionTimer for protecting Openvpn/Connman dockerSocket := os.Getenv("DOCKER_SOCKET") hostproc := os.Getenv("HOST_PROC") - startOOMProtection(hostproc, dockerSocket, ticker) + defer startOOMProtectionTimer(hostproc, dockerSocket).Stop() listenAddress := os.Getenv("GOSUPER_SOCKET") router := mux.NewRouter() diff --git a/gosuper/psutils/psutils.go b/gosuper/psutils/psutils.go index 7a5d0429..be7945da 100644 --- a/gosuper/psutils/psutils.go +++ b/gosuper/psutils/psutils.go @@ -1,94 +1,82 @@ package psutils import ( - "bufio" "fmt" - "os" + "io/ioutil" + "log" "path" - "regexp" "strconv" + "strings" "resin-supervisor/gosuper/Godeps/_workspace/src/github.com/samalba/dockerclient" "resin-supervisor/gosuper/Godeps/_workspace/src/github.com/shirou/gopsutil/process" ) +// Procs - psutils functions associated with the ProcfsPath +type Procs struct { + ProcfsPath string +} + +// parseInt - Parse integer from string. +func parseInt(str string) (int, error) { + // strconv chokes on whitespace, go figure. + trimmed := strings.TrimSpace(str) + return strconv.Atoi(trimmed) +} + //AdjustOOMPriorityByName Adjust the OOM adj value for the process' with the given name regexp -func AdjustOOMPriorityByName(procPath string, processName string, value int64, ignoreIfNonZero bool) error { - runningProcess, err := process.Pids() +func (procs *Procs) AdjustOOMPriorityByName(processName string, value int, ignoreIfNonZero bool) error { + found := false + pids, err := process.Pids() if err != nil { return err } - var processMatch = regexp.MustCompile(processName) - processFound := false - for _, pid := range runningProcess { + for _, pid := range pids { // Find the process with the given name - pidProcess, _ := process.NewProcess(pid) - pidName, _ := pidProcess.Name() - if processMatch.MatchString(pidName) { - processFound = true - err := AdjustOOMPriority(procPath, int64(pid), value, ignoreIfNonZero) - if err != nil { - return err - } + if currProcess, err := process.NewProcess(pid); err != nil { + continue + } else if name, err := currProcess.Name(); err != nil && name != processName { + continue + } else if err := procs.AdjustOOMPriority(int(pid), value, ignoreIfNonZero); err == nil { + found = true + } else { + // Not an error but logging these for debugging. + log.Printf("Error adjusting OOM for process %s (pid %d): %s", processName, pid, err) } } - if processFound { + if found { return nil } return fmt.Errorf("No process matches: %s\n", processName) } //AdjustOOMPriority Adjust the OOM adj value for the process with the given pid. -func AdjustOOMPriority(procPath string, pid int64, value int64, ignoreIfNonZero bool) error { - // Open the oom_score_adj file for the pid - oomAdjFile, err := os.OpenFile(path.Clean(procPath)+"/"+strconv.FormatInt(pid, 10)+"/oom_score_adj", os.O_RDWR, os.ModeType) - if err != nil { - return fmt.Errorf("Unable to open OOM adjust proc file for pid: %d\n", pid) - } - defer oomAdjFile.Close() - // Read the oom_score_adj value currently set - scanner := bufio.NewScanner(oomAdjFile) - scanner.Split(bufio.ScanLines) - var currentOOMString string - for scanner.Scan() { - currentOOMString = scanner.Text() // Read the OOMString - } - currentOOMValue, err := strconv.ParseInt(currentOOMString, 10, 64) - if err != nil { +func (procs *Procs) AdjustOOMPriority(pid int, value int, ignoreIfNonZero bool) error { + valueBytes := []byte(strconv.Itoa(value)) + oomFile := fmt.Sprintf("%s/%d/oom_score_adj", path.Clean(procs.ProcfsPath), pid) + if currentOOMBytes, err := ioutil.ReadFile(oomFile); err != nil { + return err + } else if currentOOMValue, err := parseInt(string(currentOOMBytes)); err != nil { return fmt.Errorf("Unable to read OOM adjust for pid: %d\n", pid) - } - if ignoreIfNonZero && currentOOMValue != 0 { + } else if ignoreIfNonZero && currentOOMValue != 0 { return nil - } - // Write to the procfile to adjust the OOM adj value. - _, err = oomAdjFile.WriteString(strconv.FormatInt(value, 10)) - if err != nil { + } else if err = ioutil.WriteFile(oomFile, valueBytes, 0644); err != nil { return fmt.Errorf("Unable to OOM adjust for pid: %d\n", pid) } return nil } //AdjustDockerOOMPriority Adjusts the OOM Adj value for the entire docker container specified by the name. This should point to root proc filesystem -func AdjustDockerOOMPriority(procPath string, connection string, containerName string, value int64, ignoreIfNonZero bool) error { - // Connect to the docker host with the connection string - docker, err := dockerclient.NewDockerClient(connection, nil) - if err != nil { +func (procs *Procs) AdjustDockerOOMPriority(connection string, containerName string, value int, ignoreIfNonZero bool) error { + if docker, err := dockerclient.NewDockerClient(connection, nil); err != nil { return err - } - // Get the running containers - containers, err := docker.ListContainers(false, false, "") - if err != nil { + } else if containers, err := docker.ListContainers(false, false, fmt.Sprintf(`{"name":["^/%s$"]}`, containerName)); err != nil { return err + } else if containerInfo, err := docker.InspectContainer(containers[0].Id); err != nil { + return err + } else if err := procs.AdjustOOMPriority(containerInfo.State.Pid, value, ignoreIfNonZero); err != nil { + return err + } else { + return nil } - for _, container := range containers { - containerInfo, err := docker.InspectContainer(container.Id) - if err != nil { - return err - } - err = AdjustOOMPriority(procPath, int64(containerInfo.State.Pid), value, ignoreIfNonZero) - if err != nil { - return err - } - } - return nil } diff --git a/gosuper/test_formatting.sh b/gosuper/test_formatting.sh old mode 100644 new mode 100755 index 0aa9a75b..1b0fc7bc --- a/gosuper/test_formatting.sh +++ b/gosuper/test_formatting.sh @@ -1,15 +1,14 @@ #!/bin/bash -DIRS=`ls -l . | egrep '^d' | awk '{print $9}'` - -DIRS=`echo $DIRS | sed "s/Godeps//g"` - -# and finally loop over the cleaned up directory list. -for DIR in $DIRS -do - if [[ -n "$(gofmt -l ${DIR})" ]]; then - echo "Bad formatting, run make format-gosuper to fix it." - exit 1 +for dir in $(find ./* -path ./Godeps -prune -or -type d -print); do + errormessage=$(gofmt -l $dir) + if [ -n "$errormessage" ]; then + echo "$errormessage" + failed=1 fi done -echo "Formatting test passed." + +if [ -n "$failed" ]; then + echo "Bad formatting, run make format-gosuper to fix above errors." + exit 1 +fi diff --git a/src/config.coffee b/src/config.coffee index b93c9d3c..f73d6403 100644 --- a/src/config.coffee +++ b/src/config.coffee @@ -13,6 +13,7 @@ checkValidKey = (s) -> return return s +# Defaults needed for both gosuper and node supervisor are declared in entry.sh module.exports = config = apiEndpoint: process.env.API_ENDPOINT ? 'https://api.resin.io' listenPort: process.env.LISTEN_PORT ? 80