/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "os"
        "path/filepath"

        "github.com/spf13/cobra"
)

type commonFlags struct {
        Master     string
        Kubeconfig string
}

func initFlags(cmd *cobra.Command, cf *commonFlags) {
        cmd.Flags().StringVarP(&cf.Master, "master", "s", "", "the address of apiserver")

        kubeConfFile := os.Getenv("KUBECONFIG")
        if kubeConfFile == "" {
                if home := homeDir(); home != "" {
                        kubeConfFile = filepath.Join(home, ".kube", "config")
                }
        }
        cmd.Flags().StringVarP(&cf.Kubeconfig, "kubeconfig", "k", kubeConfFile, "(optional) absolute path to the kubeconfig file")
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/volcano/pkg/cli/util"
)

type deleteFlags struct {
        commonFlags

        Namespace string
        JobName   string
}

var deleteJobFlags = &deleteFlags{}

// InitDeleteFlags init the delete command flags.
func InitDeleteFlags(cmd *cobra.Command) {
        initFlags(cmd, &deleteJobFlags.commonFlags)

        cmd.Flags().StringVarP(&deleteJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&deleteJobFlags.JobName, "name", "N", "", "the name of job")
}

// DeleteJob delete the job.
func DeleteJob() error {
        config, err := util.BuildConfig(deleteJobFlags.Master, deleteJobFlags.Kubeconfig)
        if err != nil {
                return err
        }

        if deleteJobFlags.JobName == "" {
                err := fmt.Errorf("job name is mandatory to delete a particular job")
                return err
        }

        jobClient := versioned.NewForConfigOrDie(config)
        err = jobClient.BatchV1alpha1().Jobs(deleteJobFlags.Namespace).Delete(context.TODO(), deleteJobFlags.JobName, metav1.DeleteOptions{})
        if err != nil {
                return err
        }
        fmt.Printf("delete job %v successfully\n", deleteJobFlags.JobName)
        return nil
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "io"
        "os"
        "strings"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/volcano/pkg/cli/util"
)

type listFlags struct {
        commonFlags

        Namespace     string
        SchedulerName string
        allNamespace  bool
        selector      string
}

const (

        // Name  name etc below key words are used in job print format
        Name string = "Name"
        // Creation create
        Creation string = "Creation"
        // Phase phase
        Phase string = "Phase"
        // Replicas  replicas
        Replicas string = "Replicas"
        // Min  minimum
        Min string = "Min"
        // Scheduler scheduler
        Scheduler string = "Scheduler"
        // Pending  pending
        Pending string = "Pending"
        // Running running
        Running string = "Running"
        // Succeeded success
        Succeeded string = "Succeeded"
        // Terminating terminating
        Terminating string = "Terminating"
        // Version version
        Version string = "Version"
        // Failed  failed
        Failed string = "Failed"
        // Unknown pod
        Unknown string = "Unknown"
        // RetryCount retry count
        RetryCount string = "RetryCount"
        // JobType  job type
        JobType string = "JobType"
        // Namespace job namespace
        Namespace string = "Namespace"
)

var listJobFlags = &listFlags{}

// InitListFlags init list command flags.
func InitListFlags(cmd *cobra.Command) {
        initFlags(cmd, &listJobFlags.commonFlags)

        cmd.Flags().StringVarP(&listJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&listJobFlags.SchedulerName, "scheduler", "S", "", "list job with specified scheduler name")
        cmd.Flags().BoolVarP(&listJobFlags.allNamespace, "all-namespaces", "", false, "list jobs in all namespaces")
        cmd.Flags().StringVarP(&listJobFlags.selector, "selector", "", "", "fuzzy matching jobName")
}

// ListJobs lists all jobs details.
func ListJobs() error {
        config, err := util.BuildConfig(listJobFlags.Master, listJobFlags.Kubeconfig)
        if err != nil {
                return err
        }
        if listJobFlags.allNamespace {
                listJobFlags.Namespace = ""
        }
        jobClient := versioned.NewForConfigOrDie(config)
        jobs, err := jobClient.BatchV1alpha1().Jobs(listJobFlags.Namespace).List(context.TODO(), metav1.ListOptions{})
        if err != nil {
                return err
        }

        if len(jobs.Items) == 0 {
                fmt.Printf("No resources found\n")
                return nil
        }
        PrintJobs(jobs, os.Stdout)

        return nil
}

// PrintJobs prints all jobs details.
func PrintJobs(jobs *v1alpha1.JobList, writer io.Writer) {
        maxLenInfo := getMaxLen(jobs)

        titleFormat := "%%-%ds%%-15s%%-12s%%-12s%%-12s%%-6s%%-10s%%-10s%%-12s%%-10s%%-12s%%-10s\n"
        contentFormat := "%%-%ds%%-15s%%-12s%%-12s%%-12d%%-6d%%-10d%%-10d%%-12d%%-10d%%-12d%%-10d\n"

        var err error
        if listJobFlags.allNamespace {
                _, err = fmt.Fprintf(writer, fmt.Sprintf("%%-%ds"+titleFormat, maxLenInfo[1], maxLenInfo[0]),
                        Namespace, Name, Creation, Phase, JobType, Replicas, Min, Pending, Running, Succeeded, Failed, Unknown, RetryCount)
        } else {
                _, err = fmt.Fprintf(writer, fmt.Sprintf(titleFormat, maxLenInfo[0]),
                        Name, Creation, Phase, JobType, Replicas, Min, Pending, Running, Succeeded, Failed, Unknown, RetryCount)
        }
        if err != nil {
                fmt.Printf("Failed to print list command result: %s.\n", err)
        }

        for _, job := range jobs.Items {
                if listJobFlags.SchedulerName != "" && listJobFlags.SchedulerName != job.Spec.SchedulerName {
                        continue
                }
                if !strings.Contains(job.Name, listJobFlags.selector) {
                        continue
                }
                replicas := int32(0)
                for _, ts := range job.Spec.Tasks {
                        replicas += ts.Replicas
                }
                jobType := job.ObjectMeta.Labels[v1alpha1.JobTypeKey]
                if jobType == "" {
                        jobType = "Batch"
                }

                if listJobFlags.allNamespace {
                        _, err = fmt.Fprintf(writer, fmt.Sprintf("%%-%ds"+contentFormat, maxLenInfo[1], maxLenInfo[0]),
                                job.Namespace, job.Name, job.CreationTimestamp.Format("2006-01-02"), job.Status.State.Phase, jobType, replicas,
                                job.Status.MinAvailable, job.Status.Pending, job.Status.Running, job.Status.Succeeded, job.Status.Failed, job.Status.Unknown, job.Status.RetryCount)
                } else {
                        _, err = fmt.Fprintf(writer, fmt.Sprintf(contentFormat, maxLenInfo[0]),
                                job.Name, job.CreationTimestamp.Format("2006-01-02"), job.Status.State.Phase, jobType, replicas,
                                job.Status.MinAvailable, job.Status.Pending, job.Status.Running, job.Status.Succeeded, job.Status.Failed, job.Status.Unknown, job.Status.RetryCount)
                }
                if err != nil {
                        fmt.Printf("Failed to print list command result: %s.\n", err)
                }
        }
}

func getMaxLen(jobs *v1alpha1.JobList) []int {
        maxNameLen := len(Name)
        maxNamespaceLen := len(Namespace)
        for _, job := range jobs.Items {
                if len(job.Name) > maxNameLen {
                        maxNameLen = len(job.Name)
                }
                if len(job.Namespace) > maxNamespaceLen {
                        maxNamespaceLen = len(job.Namespace)
                }
        }

        return []int{maxNameLen + 3, maxNamespaceLen + 3}
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "fmt"

        "github.com/spf13/cobra"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/volcano/pkg/cli/util"
)

type resumeFlags struct {
        commonFlags

        Namespace string
        JobName   string
}

var resumeJobFlags = &resumeFlags{}

// InitResumeFlags init resume command flags.
func InitResumeFlags(cmd *cobra.Command) {
        initFlags(cmd, &resumeJobFlags.commonFlags)

        cmd.Flags().StringVarP(&resumeJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&resumeJobFlags.JobName, "name", "N", "", "the name of job")
}

// ResumeJob resumes the job.
func ResumeJob() error {
        config, err := util.BuildConfig(resumeJobFlags.Master, resumeJobFlags.Kubeconfig)
        if err != nil {
                return err
        }
        if resumeJobFlags.JobName == "" {
                err := fmt.Errorf("job name is mandatory to resume a particular job")
                return err
        }

        return createJobCommand(config,
                resumeJobFlags.Namespace, resumeJobFlags.JobName,
                v1alpha1.ResumeJobAction)
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "io/ioutil"
        "strings"

        "github.com/spf13/cobra"

        v1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "sigs.k8s.io/yaml"

        vcbatch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/volcano/pkg/cli/util"
)

type runFlags struct {
        commonFlags

        Name      string
        Namespace string
        Image     string

        MinAvailable  int
        Replicas      int
        Requests      string
        Limits        string
        SchedulerName string
        FileName      string
}

var launchJobFlags = &runFlags{}

// InitRunFlags init the run flags.
func InitRunFlags(cmd *cobra.Command) {
        initFlags(cmd, &launchJobFlags.commonFlags)

        cmd.Flags().StringVarP(&launchJobFlags.Image, "image", "i", "busybox", "the container image of job")
        cmd.Flags().StringVarP(&launchJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&launchJobFlags.Name, "name", "N", "", "the name of job")
        cmd.Flags().IntVarP(&launchJobFlags.MinAvailable, "min", "m", 1, "the minimal available tasks of job")
        cmd.Flags().IntVarP(&launchJobFlags.Replicas, "replicas", "r", 1, "the total tasks of job")
        cmd.Flags().StringVarP(&launchJobFlags.Requests, "requests", "R", "cpu=1000m,memory=100Mi", "the resource request of the task")
        cmd.Flags().StringVarP(&launchJobFlags.Limits, "limits", "L", "cpu=1000m,memory=100Mi", "the resource limit of the task")
        cmd.Flags().StringVarP(&launchJobFlags.SchedulerName, "scheduler", "S", "volcano", "the scheduler for this job")
        cmd.Flags().StringVarP(&launchJobFlags.FileName, "filename", "f", "", "the yaml file of job")
}

var jobName = "job.volcano.sh"

// RunJob creates the job.
func RunJob() error {
        config, err := util.BuildConfig(launchJobFlags.Master, launchJobFlags.Kubeconfig)
        if err != nil {
                return err
        }

        if launchJobFlags.Name == "" && launchJobFlags.FileName == "" {
                err = fmt.Errorf("job name cannot be left blank")
                return err
        }

        req, err := populateResourceListV1(launchJobFlags.Requests)
        if err != nil {
                return err
        }

        limit, err := populateResourceListV1(launchJobFlags.Limits)
        if err != nil {
                return err
        }

        job, err := readFile(launchJobFlags.FileName)
        if err != nil {
                return err
        }

        if job == nil {
                job = constructLaunchJobFlagsJob(launchJobFlags, req, limit)
        }

        jobClient := versioned.NewForConfigOrDie(config)
        newJob, err := jobClient.BatchV1alpha1().Jobs(launchJobFlags.Namespace).Create(context.TODO(), job, metav1.CreateOptions{})
        if err != nil {
                return err
        }

        if newJob.Spec.Queue == "" {
                newJob.Spec.Queue = "default"
        }

        fmt.Printf("run job %v successfully\n", newJob.Name)

        return nil
}

func readFile(filename string) (*vcbatch.Job, error) {
        if filename == "" {
                return nil, nil
        }

        if !strings.Contains(filename, ".yaml") && !strings.Contains(filename, ".yml") {
                return nil, fmt.Errorf("only support yaml file")
        }

        file, err := ioutil.ReadFile(filename)
        if err != nil {
                return nil, fmt.Errorf("failed to read file, err: %v", err)
        }

        var job vcbatch.Job
        if err := yaml.Unmarshal(file, &job); err != nil {
                return nil, fmt.Errorf("failed to unmarshal file, err:  %v", err)
        }

        return &job, nil
}

func constructLaunchJobFlagsJob(launchJobFlags *runFlags, req, limit v1.ResourceList) *vcbatch.Job {
        return &vcbatch.Job{
                ObjectMeta: metav1.ObjectMeta{
                        Name:      launchJobFlags.Name,
                        Namespace: launchJobFlags.Namespace,
                },
                Spec: vcbatch.JobSpec{
                        MinAvailable:  int32(launchJobFlags.MinAvailable),
                        SchedulerName: launchJobFlags.SchedulerName,
                        Tasks: []vcbatch.TaskSpec{
                                {
                                        Replicas: int32(launchJobFlags.Replicas),

                                        Template: v1.PodTemplateSpec{
                                                ObjectMeta: metav1.ObjectMeta{
                                                        Name:   launchJobFlags.Name,
                                                        Labels: map[string]string{jobName: launchJobFlags.Name},
                                                },
                                                Spec: v1.PodSpec{
                                                        RestartPolicy: v1.RestartPolicyNever,
                                                        Containers: []v1.Container{
                                                                {
                                                                        Image:           launchJobFlags.Image,
                                                                        Name:            launchJobFlags.Name,
                                                                        ImagePullPolicy: v1.PullIfNotPresent,
                                                                        Resources: v1.ResourceRequirements{
                                                                                Limits:   limit,
                                                                                Requests: req,
                                                                        },
                                                                },
                                                        },
                                                },
                                        },
                                },
                        },
                },
        }
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "fmt"

        "github.com/spf13/cobra"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/volcano/pkg/cli/util"
)

type suspendFlags struct {
        commonFlags

        Namespace string
        JobName   string
}

var suspendJobFlags = &suspendFlags{}

// InitSuspendFlags init suspend related flags.
func InitSuspendFlags(cmd *cobra.Command) {
        initFlags(cmd, &suspendJobFlags.commonFlags)

        cmd.Flags().StringVarP(&suspendJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&suspendJobFlags.JobName, "name", "N", "", "the name of job")
}

// SuspendJob suspends the job.
func SuspendJob() error {
        config, err := util.BuildConfig(suspendJobFlags.Master, suspendJobFlags.Kubeconfig)
        if err != nil {
                return err
        }

        if suspendJobFlags.JobName == "" {
                err := fmt.Errorf("job name is mandatory to suspend a particular job")
                return err
        }

        return createJobCommand(config,
                suspendJobFlags.Namespace, suspendJobFlags.JobName,
                v1alpha1.AbortJobAction)
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "os"
        "strings"
        "time"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/resource"
        "k8s.io/client-go/rest"

        vcbus "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

func homeDir() string {
        if h := os.Getenv("HOME"); h != "" {
                return h
        }
        return os.Getenv("USERPROFILE") // windows
}

// populateResourceListV1 takes strings of form <resourceName1>=<value1>,<resourceName1>=<value2>
// and returns ResourceList.
func populateResourceListV1(spec string) (v1.ResourceList, error) {
        // empty input gets a nil response to preserve generator test expected behaviors
        if spec == "" {
                return nil, nil
        }

        result := v1.ResourceList{}
        resourceStatements := strings.Split(spec, ",")
        for _, resourceStatement := range resourceStatements {
                parts := strings.Split(resourceStatement, "=")
                if len(parts) != 2 {
                        return nil, fmt.Errorf("invalid argument syntax %v, expected <resource>=<value>", resourceStatement)
                }
                resourceName := v1.ResourceName(parts[0])
                resourceQuantity, err := resource.ParseQuantity(parts[1])
                if err != nil {
                        return nil, err
                }
                result[resourceName] = resourceQuantity
        }
        return result, nil
}

func createJobCommand(config *rest.Config, ns, name string, action vcbus.Action) error {
        jobClient := versioned.NewForConfigOrDie(config)
        job, err := jobClient.BatchV1alpha1().Jobs(ns).Get(context.TODO(), name, metav1.GetOptions{})
        if err != nil {
                return err
        }

        ctrlRef := metav1.NewControllerRef(job, helpers.JobKind)
        cmd := &vcbus.Command{
                ObjectMeta: metav1.ObjectMeta{
                        GenerateName: fmt.Sprintf("%s-%s-",
                                job.Name, strings.ToLower(string(action))),
                        Namespace: job.Namespace,
                        OwnerReferences: []metav1.OwnerReference{
                                *ctrlRef,
                        },
                },
                TargetObject: ctrlRef,
                Action:       string(action),
        }

        if _, err := jobClient.BusV1alpha1().Commands(ns).Create(context.TODO(), cmd, metav1.CreateOptions{}); err != nil {
                return err
        }

        return nil
}

func translateTimestampSince(timestamp metav1.Time) string {
        if timestamp.IsZero() {
                return "<unknown>"
        }
        return HumanDuration(time.Since(timestamp.Time))
}

// HumanDuration translate time.Duration to human readable time string.
func HumanDuration(d time.Duration) string {
        // Allow deviation no more than 2 seconds(excluded) to tolerate machine time
        // inconsistence, it can be considered as almost now.
        if seconds := int(d.Seconds()); seconds < -1 {
                return "<invalid>"
        } else if seconds < 0 {
                return "0s"
        } else if seconds < 60*2 {
                return fmt.Sprintf("%ds", seconds)
        }
        minutes := int(d / time.Minute)
        if minutes < 10 {
                s := int(d/time.Second) % 60
                if s == 0 {
                        return fmt.Sprintf("%dm", minutes)
                }
                return fmt.Sprintf("%dm%ds", minutes, s)
        } else if minutes < 60*3 {
                return fmt.Sprintf("%dm", minutes)
        }
        hours := int(d / time.Hour)
        if hours < 8 {
                m := int(d/time.Minute) % 60
                if m == 0 {
                        return fmt.Sprintf("%dh", hours)
                }
                return fmt.Sprintf("%dh%dm", hours, m)
        } else if hours < 48 {
                return fmt.Sprintf("%dh", hours)
        } else if hours < 24*8 {
                h := hours % 24
                if h == 0 {
                        return fmt.Sprintf("%dd", hours/24)
                }
                return fmt.Sprintf("%dd%dh", hours/24, h)
        } else if hours < 24*365*2 {
                return fmt.Sprintf("%dd", hours/24)
        } else if hours < 24*365*8 {
                return fmt.Sprintf("%dy%dd", hours/24/365, (hours/24)%365)
        }
        return fmt.Sprintf("%dy", hours/24/365)
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "encoding/json"
        "fmt"
        "io"
        "os"
        "strings"

        "github.com/spf13/cobra"

        coreV1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/kubernetes"
        "k8s.io/client-go/rest"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/volcano/pkg/cli/util"
)

type viewFlags struct {
        commonFlags

        Namespace string
        JobName   string
}

// level of print indent.
const (
        Level0 = iota
        Level1
        Level2
)

var viewJobFlags = &viewFlags{}

// InitViewFlags init the view command flags.
func InitViewFlags(cmd *cobra.Command) {
        initFlags(cmd, &viewJobFlags.commonFlags)

        cmd.Flags().StringVarP(&viewJobFlags.Namespace, "namespace", "n", "default", "the namespace of job")
        cmd.Flags().StringVarP(&viewJobFlags.JobName, "name", "N", "", "the name of job")
}

// ViewJob gives full details of the job.
func ViewJob() error {
        config, err := util.BuildConfig(viewJobFlags.Master, viewJobFlags.Kubeconfig)
        if err != nil {
                return err
        }
        if viewJobFlags.JobName == "" {
                err := fmt.Errorf("job name (specified by --name or -N) is mandatory to view a particular job")
                return err
        }

        jobClient := versioned.NewForConfigOrDie(config)
        job, err := jobClient.BatchV1alpha1().Jobs(viewJobFlags.Namespace).Get(context.TODO(), viewJobFlags.JobName, metav1.GetOptions{})
        if err != nil {
                return err
        }
        if job == nil {
                fmt.Printf("No resources found\n")
                return nil
        }
        PrintJobInfo(job, os.Stdout)
        PrintEvents(GetEvents(config, job), os.Stdout)
        return nil
}

// PrintJobInfo print the job detailed info into writer.
func PrintJobInfo(job *v1alpha1.Job, writer io.Writer) {
        WriteLine(writer, Level0, "Name:       \t%s\n", job.Name)
        WriteLine(writer, Level0, "Namespace:  \t%s\n", job.Namespace)
        if len(job.Labels) > 0 {
                label, _ := json.Marshal(job.Labels)
                WriteLine(writer, Level0, "Labels:     \t%s\n", string(label))
        } else {
                WriteLine(writer, Level0, "Labels:     \t<none>\n")
        }
        if len(job.Annotations) > 0 {
                annotation, _ := json.Marshal(job.Annotations)
                WriteLine(writer, Level0, "Annotations:\t%s\n", string(annotation))
        } else {
                WriteLine(writer, Level0, "Annotations:\t<none>\n")
        }
        WriteLine(writer, Level0, "API Version:\t%s\n", job.APIVersion)
        WriteLine(writer, Level0, "Kind:       \t%s\n", job.Kind)

        WriteLine(writer, Level0, "Metadata:\n")
        WriteLine(writer, Level1, "Creation Timestamp:\t%s\n", job.CreationTimestamp)
        WriteLine(writer, Level1, "Generate Name:     \t%s\n", job.GenerateName)
        WriteLine(writer, Level1, "Generation:        \t%d\n", job.Generation)
        WriteLine(writer, Level1, "Resource Version:  \t%s\n", job.ResourceVersion)
        WriteLine(writer, Level1, "Self Link:         \t%s\n", job.SelfLink)
        WriteLine(writer, Level1, "UID:               \t%s\n", job.UID)

        WriteLine(writer, Level0, "Spec:\n")
        WriteLine(writer, Level1, "Min Available:     \t%d\n", job.Spec.MinAvailable)
        WriteLine(writer, Level1, "Plugins:\n")
        WriteLine(writer, Level2, "Env:\t%v\n", job.Spec.Plugins["env"])
        WriteLine(writer, Level2, "Ssh:\t%v\n", job.Spec.Plugins["ssh"])
        WriteLine(writer, Level1, "Scheduler Name:    \t%s\n", job.Spec.SchedulerName)
        WriteLine(writer, Level1, "Tasks:\n")
        for i := 0; i < len(job.Spec.Tasks); i++ {
                WriteLine(writer, Level2, "Name:\t%s\n", job.Spec.Tasks[i].Name)
                WriteLine(writer, Level2, "Replicas:\t%d\n", job.Spec.Tasks[i].Replicas)
                WriteLine(writer, Level2, "Template:\n")
                WriteLine(writer, Level2+1, "Metadata:\n")
                WriteLine(writer, Level2+2, "Annotations:\n")
                WriteLine(writer, Level2+3, "Cri . Cci . Io / Container - Type:          \t%s\n", job.Spec.Tasks[i].Template.ObjectMeta.Annotations["cri.cci.io/container-type"])
                WriteLine(writer, Level2+3, "Kubernetes . Io / Availablezone:            \t%s\n", job.Spec.Tasks[i].Template.ObjectMeta.Annotations["kubernetes.io/availablezone"])
                WriteLine(writer, Level2+3, "Network . Alpha . Kubernetes . Io / Network:\t%s\n", job.Spec.Tasks[i].Template.ObjectMeta.Annotations["network.alpha.kubernetes.io/network"])
                WriteLine(writer, Level2+2, "Creation Timestamp:\t%s\n", job.Spec.Tasks[i].Template.ObjectMeta.CreationTimestamp)

                WriteLine(writer, Level2+1, "Spec:\n")
                WriteLine(writer, Level2+2, "Containers:\n")
                for j := 0; j < len(job.Spec.Tasks[i].Template.Spec.Containers); j++ {
                        WriteLine(writer, Level2+3, "Command:\n")
                        for k := 0; k < len(job.Spec.Tasks[i].Template.Spec.Containers[j].Command); k++ {
                                WriteLine(writer, Level2+4, "%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Command[k])
                        }
                        WriteLine(writer, Level2+3, "Image:\t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Image)
                        WriteLine(writer, Level2+3, "Name: \t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Name)
                        WriteLine(writer, Level2+3, "Ports:\n")
                        for k := 0; k < len(job.Spec.Tasks[i].Template.Spec.Containers[j].Ports); k++ {
                                WriteLine(writer, Level2+4, "Container Port:\t%d\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Ports[k].ContainerPort)
                                WriteLine(writer, Level2+4, "Name:          \t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Ports[k].Name)
                        }
                        WriteLine(writer, Level2+3, "Resources:\n")
                        WriteLine(writer, Level2+4, "Limits:\n")
                        WriteLine(writer, Level2+5, "Cpu:   \t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Resources.Limits.Cpu())
                        WriteLine(writer, Level2+5, "Memory:\t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Resources.Limits.Memory())
                        WriteLine(writer, Level2+4, "Requests:\n")
                        WriteLine(writer, Level2+5, "Cpu:   \t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Resources.Requests.Cpu())
                        WriteLine(writer, Level2+5, "Memory:\t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].Resources.Requests.Memory())
                        WriteLine(writer, Level2+4, "Working Dir:\t%s\n", job.Spec.Tasks[i].Template.Spec.Containers[j].WorkingDir)
                }
                WriteLine(writer, Level2+2, "Image Pull Secrets:\n")
                for j := 0; j < len(job.Spec.Tasks[i].Template.Spec.ImagePullSecrets); j++ {
                        WriteLine(writer, Level2+3, "Name:     \t%s\n", job.Spec.Tasks[i].Template.Spec.ImagePullSecrets[j].Name)
                }
                WriteLine(writer, Level2+2, "Restart Policy:   \t%s\n", job.Spec.Tasks[i].Template.Spec.RestartPolicy)
        }

        WriteLine(writer, Level0, "Status:\n")
        if job.Status.Succeeded > 0 {
                WriteLine(writer, Level1, "Succeeded:    \t%d\n", job.Status.Succeeded)
        }
        if job.Status.Pending > 0 {
                WriteLine(writer, Level1, "Pending:      \t%d\n", job.Status.Pending)
        }
        if job.Status.Running > 0 {
                WriteLine(writer, Level1, "Running:      \t%d\n", job.Status.Running)
        }
        if job.Status.Failed > 0 {
                WriteLine(writer, Level1, "Failed:       \t%d\n", job.Status.Failed)
        }
        if job.Status.Terminating > 0 {
                WriteLine(writer, Level1, "Terminating:  \t%d\n", job.Status.Terminating)
        }
        if job.Status.Unknown > 0 {
                WriteLine(writer, Level1, "Unknown:      \t%d\n", job.Status.Unknown)
        }
        if job.Status.RetryCount > 0 {
                WriteLine(writer, Level1, "RetryCount:   \t%d\n", job.Status.RetryCount)
        }
        if job.Status.MinAvailable > 0 {
                WriteLine(writer, Level1, "Min Available:\t%d\n", job.Status.MinAvailable)
        }
        if job.Status.Version > 0 {
                WriteLine(writer, Level1, "Version:      \t%d\n", job.Status.Version)
        }

        WriteLine(writer, Level1, "State:\n")
        WriteLine(writer, Level2, "Phase:\t%s\n", job.Status.State.Phase)
        if len(job.Status.ControlledResources) > 0 {
                WriteLine(writer, Level1, "Controlled Resources:\n")
                for key, value := range job.Status.ControlledResources {
                        WriteLine(writer, Level2, "%s: \t%s\n", key, value)
                }
        }
}

// PrintEvents print event info to writer.
func PrintEvents(events []coreV1.Event, writer io.Writer) {
        if len(events) > 0 {
                WriteLine(writer, Level0, "%s:\n%-15s\t%-40s\t%-30s\t%-40s\t%s\n", "Events", "Type", "Reason", "Age", "Form", "Message")
                WriteLine(writer, Level0, "%-15s\t%-40s\t%-30s\t%-40s\t%s\n", "-------", "-------", "-------", "-------", "-------")
                for _, e := range events {
                        var interval string
                        if e.Count > 1 {
                                interval = fmt.Sprintf("%s (x%d over %s)", translateTimestampSince(e.LastTimestamp), e.Count, translateTimestampSince(e.FirstTimestamp))
                        } else {
                                interval = translateTimestampSince(e.FirstTimestamp)
                        }
                        EventSourceString := []string{e.Source.Component}
                        if len(e.Source.Host) > 0 {
                                EventSourceString = append(EventSourceString, e.Source.Host)
                        }
                        WriteLine(writer, Level0, "%-15v\t%-40v\t%-30s\t%-40s\t%v\n",
                                e.Type,
                                e.Reason,
                                interval,
                                strings.Join(EventSourceString, ", "),
                                strings.TrimSpace(e.Message),
                        )
                }
        } else {
                WriteLine(writer, Level0, "Events: \t<none>\n")
        }
}

// GetEvents get the job event by config.
func GetEvents(config *rest.Config, job *v1alpha1.Job) []coreV1.Event {
        kubernetes, err := kubernetes.NewForConfig(config)
        if err != nil {
                fmt.Printf("%v\n", err)
                return nil
        }
        events, _ := kubernetes.CoreV1().Events(viewJobFlags.Namespace).List(context.TODO(), metav1.ListOptions{})
        var jobEvents []coreV1.Event
        for _, v := range events.Items {
                if strings.HasPrefix(v.ObjectMeta.Name, job.Name+".") {
                        jobEvents = append(jobEvents, v)
                }
        }
        return jobEvents
}

// WriteLine write lines with specified indent.
func WriteLine(writer io.Writer, spaces int, content string, params ...interface{}) {
        prefix := ""
        for i := 0; i < spaces; i++ {
                prefix += "  "
        }
        fmt.Fprintf(writer, prefix+content, params...)
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "os"
        "path/filepath"

        "github.com/spf13/cobra"
)

type commonFlags struct {
        Master        string
        Kubeconfig    string
        SchedulerName string
}

func initFlags(cmd *cobra.Command, cf *commonFlags) {
        cmd.Flags().StringVarP(&cf.SchedulerName, "scheduler", "", "volcano", "the scheduler for this job")
        cmd.Flags().StringVarP(&cf.Master, "master", "s", "", "the address of apiserver")

        kubeConfFile := os.Getenv("KUBECONFIG")
        if kubeConfFile == "" {
                if home := homeDir(); home != "" {
                        kubeConfFile = filepath.Join(home, ".kube", "config")
                }
        }
        cmd.Flags().StringVarP(&cf.Kubeconfig, "kubeconfig", "k", kubeConfFile, "(optional) absolute path to the kubeconfig file")
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

type createFlags struct {
        commonFlags

        Name   string
        Weight int32
        // State is state of Queue
        State string
}

var createQueueFlags = &createFlags{}

// InitCreateFlags is used to init all flags during queue creating.
func InitCreateFlags(cmd *cobra.Command) {
        initFlags(cmd, &createQueueFlags.commonFlags)

        cmd.Flags().StringVarP(&createQueueFlags.Name, "name", "n", "test", "the name of queue")
        cmd.Flags().Int32VarP(&createQueueFlags.Weight, "weight", "w", 1, "the weight of the queue")

        cmd.Flags().StringVarP(&createQueueFlags.State, "state", "S", "Open", "the state of queue")
}

// CreateQueue create queue.
func CreateQueue() error {
        config, err := buildConfig(createQueueFlags.Master, createQueueFlags.Kubeconfig)
        if err != nil {
                return err
        }

        queue := &schedulingv1beta1.Queue{
                ObjectMeta: metav1.ObjectMeta{
                        Name: createQueueFlags.Name,
                },
                Spec: schedulingv1beta1.QueueSpec{
                        Weight: createQueueFlags.Weight,
                },
                Status: schedulingv1beta1.QueueStatus{
                        State: schedulingv1beta1.QueueState(createQueueFlags.State),
                },
        }

        queueClient := versioned.NewForConfigOrDie(config)
        if _, err := queueClient.SchedulingV1beta1().Queues().Create(context.TODO(), queue, metav1.CreateOptions{}); err != nil {
                return err
        }

        return nil
}

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"

        "volcano.sh/apis/pkg/client/clientset/versioned"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type deleteFlags struct {
        commonFlags

        // Name is name of queue
        Name string
}

var deleteQueueFlags = &deleteFlags{}

// InitDeleteFlags is used to init all flags during queue deleting.
func InitDeleteFlags(cmd *cobra.Command) {
        initFlags(cmd, &deleteQueueFlags.commonFlags)

        cmd.Flags().StringVarP(&deleteQueueFlags.Name, "name", "n", "", "the name of queue")
}

// DeleteQueue delete queue.
func DeleteQueue() error {
        config, err := buildConfig(deleteQueueFlags.Master, deleteQueueFlags.Kubeconfig)
        if err != nil {
                return err
        }

        if len(deleteQueueFlags.Name) == 0 {
                return fmt.Errorf("queue name must be specified")
        }

        queueClient := versioned.NewForConfigOrDie(config)
        return queueClient.SchedulingV1beta1().Queues().Delete(context.TODO(), deleteQueueFlags.Name, metav1.DeleteOptions{})
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"
        "io"
        "os"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

type getFlags struct {
        commonFlags

        Name string
}

var getQueueFlags = &getFlags{}

// InitGetFlags is used to init all flags.
func InitGetFlags(cmd *cobra.Command) {
        initFlags(cmd, &getQueueFlags.commonFlags)

        cmd.Flags().StringVarP(&getQueueFlags.Name, "name", "n", "", "the name of queue")
}

// GetQueue gets a queue.
func GetQueue() error {
        config, err := buildConfig(getQueueFlags.Master, getQueueFlags.Kubeconfig)
        if err != nil {
                return err
        }

        if getQueueFlags.Name == "" {
                err := fmt.Errorf("name is mandatory to get the particular queue details")
                return err
        }

        queueClient := versioned.NewForConfigOrDie(config)
        queue, err := queueClient.SchedulingV1beta1().Queues().Get(context.TODO(), getQueueFlags.Name, metav1.GetOptions{})
        if err != nil {
                return err
        }

        PrintQueue(queue, os.Stdout)

        return nil
}

// PrintQueue prints queue information.
func PrintQueue(queue *v1beta1.Queue, writer io.Writer) {
        _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s%-8s%-8s\n",
                Name, Weight, State, Inqueue, Pending, Running, Unknown)
        if err != nil {
                fmt.Printf("Failed to print queue command result: %s.\n", err)
        }
        _, err = fmt.Fprintf(writer, "%-25s%-8d%-8s%-8d%-8d%-8d%-8d\n",
                queue.Name, queue.Spec.Weight, queue.Status.State, queue.Status.Inqueue,
                queue.Status.Pending, queue.Status.Running, queue.Status.Unknown)
        if err != nil {
                fmt.Printf("Failed to print queue command result: %s.\n", err)
        }
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"
        "io"
        "os"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

type listFlags struct {
        commonFlags
}

const (
        // Weight of the queue
        Weight string = "Weight"

        // Name of queue
        Name string = "Name"

        // Pending status of the queue
        Pending string = "Pending"

        // Running status of the queue
        Running string = "Running"

        // Unknown status of the queue
        Unknown string = "Unknown"

        // Inqueue status of queue
        Inqueue string = "Inqueue"

        // State is state of queue
        State string = "State"
)

var listQueueFlags = &listFlags{}

// InitListFlags inits all flags.
func InitListFlags(cmd *cobra.Command) {
        initFlags(cmd, &listQueueFlags.commonFlags)
}

// ListQueue lists all the queue.
func ListQueue() error {
        config, err := buildConfig(listQueueFlags.Master, listQueueFlags.Kubeconfig)
        if err != nil {
                return err
        }

        jobClient := versioned.NewForConfigOrDie(config)
        queues, err := jobClient.SchedulingV1beta1().Queues().List(context.TODO(), metav1.ListOptions{})
        if err != nil {
                return err
        }

        if len(queues.Items) == 0 {
                fmt.Printf("No resources found\n")
                return nil
        }
        PrintQueues(queues, os.Stdout)

        return nil
}

// PrintQueues prints queue information.
func PrintQueues(queues *v1beta1.QueueList, writer io.Writer) {
        _, err := fmt.Fprintf(writer, "%-25s%-8s%-8s%-8s%-8s%-8s%-8s\n",
                Name, Weight, State, Inqueue, Pending, Running, Unknown)
        if err != nil {
                fmt.Printf("Failed to print queue command result: %s.\n", err)
        }
        for _, queue := range queues.Items {
                _, err = fmt.Fprintf(writer, "%-25s%-8d%-8s%-8d%-8d%-8d%-8d\n",
                        queue.Name, queue.Spec.Weight, queue.Status.State, queue.Status.Inqueue,
                        queue.Status.Pending, queue.Status.Running, queue.Status.Unknown)
                if err != nil {
                        fmt.Printf("Failed to print queue command result: %s.\n", err)
                }
        }
}

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"

        "github.com/spf13/cobra"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "k8s.io/apimachinery/pkg/types"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

const (
        // ActionOpen is `open` action
        ActionOpen = "open"
        // ActionClose is `close` action
        ActionClose = "close"
        // ActionUpdate is `update` action
        ActionUpdate = "update"
)

type operateFlags struct {
        commonFlags

        // Name is name of queue
        Name string
        // Weight is weight of queue
        Weight int32
        // Action is operation action of queue
        Action string
}

var operateQueueFlags = &operateFlags{}

// InitOperateFlags is used to init all flags during queue operating
func InitOperateFlags(cmd *cobra.Command) {
        initFlags(cmd, &operateQueueFlags.commonFlags)

        cmd.Flags().StringVarP(&operateQueueFlags.Name, "name", "n", "", "the name of queue")
        cmd.Flags().Int32VarP(&operateQueueFlags.Weight, "weight", "w", 0, "the weight of the queue")
        cmd.Flags().StringVarP(&operateQueueFlags.Action, "action", "a", "",
                "operate action to queue, valid actions are open, close, update")
}

// OperateQueue operates queue
func OperateQueue() error {
        config, err := buildConfig(operateQueueFlags.Master, operateQueueFlags.Kubeconfig)
        if err != nil {
                return err
        }

        if len(operateQueueFlags.Name) == 0 {
                return fmt.Errorf("queue name must be specified")
        }

        var action v1alpha1.Action

        switch operateQueueFlags.Action {
        case ActionOpen:
                action = v1alpha1.OpenQueueAction
        case ActionClose:
                action = v1alpha1.CloseQueueAction
        case ActionUpdate:
                if operateQueueFlags.Weight == 0 {
                        return fmt.Errorf("when %s queue %s, weight must be specified, "+
                                "the value must be greater than 0", ActionUpdate, operateQueueFlags.Name)
                }

                queueClient := versioned.NewForConfigOrDie(config)
                patchBytes := []byte(fmt.Sprintf(`{"spec":{"weight":%d}}`, operateQueueFlags.Weight))
                _, err := queueClient.SchedulingV1beta1().Queues().Patch(context.TODO(),
                        operateQueueFlags.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})

                return err
        case "":
                return fmt.Errorf("action can not be null")
        default:
                return fmt.Errorf("action %s invalid, valid actions are %s, %s and %s",
                        operateQueueFlags.Action, ActionOpen, ActionClose, ActionUpdate)
        }

        return createQueueCommand(config, action)
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"
        "os"
        "strings"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        // Initialize client auth plugin.
        _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
        "k8s.io/client-go/rest"
        "k8s.io/client-go/tools/clientcmd"

        busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

func homeDir() string {
        if h := os.Getenv("HOME"); h != "" {
                return h
        }
        return os.Getenv("USERPROFILE") // windows
}

func buildConfig(master, kubeconfig string) (*rest.Config, error) {
        return clientcmd.BuildConfigFromFlags(master, kubeconfig)
}

func createQueueCommand(config *rest.Config, action busv1alpha1.Action) error {
        queueClient := versioned.NewForConfigOrDie(config)
        queue, err := queueClient.SchedulingV1beta1().Queues().Get(context.TODO(), operateQueueFlags.Name, metav1.GetOptions{})
        if err != nil {
                return err
        }

        ctrlRef := metav1.NewControllerRef(queue, helpers.V1beta1QueueKind)
        cmd := &busv1alpha1.Command{
                ObjectMeta: metav1.ObjectMeta{
                        GenerateName: fmt.Sprintf("%s-%s-",
                                queue.Name, strings.ToLower(string(action))),
                        OwnerReferences: []metav1.OwnerReference{
                                *ctrlRef,
                        },
                },
                TargetObject: ctrlRef,
                Action:       string(action),
        }

        if _, err := queueClient.BusV1alpha1().Commands("default").Create(context.TODO(), cmd, metav1.CreateOptions{}); err != nil {
                return err
        }

        return nil
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
        "context"
        "fmt"
        "os"
        "path/filepath"
        "strings"
        "time"

        "github.com/spf13/cobra"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/resource"
        "k8s.io/client-go/rest"
        "k8s.io/client-go/tools/clientcmd"

        vcbus "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        "volcano.sh/apis/pkg/client/clientset/versioned"
)

// CommonFlags are the flags that most command lines have.
type CommonFlags struct {
        Master     string
        Kubeconfig string
}

// InitFlags initializes the common flags for most command lines.
func InitFlags(cmd *cobra.Command, cf *CommonFlags) {
        cmd.Flags().StringVarP(&cf.Master, "master", "s", "", "the address of apiserver")

        kubeConfFile := os.Getenv("KUBECONFIG")
        if kubeConfFile == "" {
                if home := HomeDir(); home != "" {
                        kubeConfFile = filepath.Join(home, ".kube", "config")
                }
        }
        cmd.Flags().StringVarP(&cf.Kubeconfig, "kubeconfig", "k", kubeConfFile, "(optional) absolute path to the kubeconfig file")
}

// HomeDir gets the env $HOME.
func HomeDir() string {
        if h := os.Getenv("HOME"); h != "" {
                return h
        }
        return os.Getenv("USERPROFILE") // windows
}

// BuildConfig builds the configure file for command lines.
func BuildConfig(master, kubeconfig string) (*rest.Config, error) {
        return clientcmd.BuildConfigFromFlags(master, kubeconfig)
}

// PopulateResourceListV1 takes strings of form <resourceName1>=<value1>,<resourceName1>=<value2> and returns ResourceList.
func PopulateResourceListV1(spec string) (v1.ResourceList, error) {
        // empty input gets a nil response to preserve generator test expected behaviors
        if spec == "" {
                return nil, nil
        }

        result := v1.ResourceList{}
        resourceStatements := strings.Split(spec, ",")
        for _, resourceStatement := range resourceStatements {
                parts := strings.Split(resourceStatement, "=")
                if len(parts) != 2 {
                        return nil, fmt.Errorf("invalid argument syntax %v, expected <resource>=<value>", resourceStatement)
                }
                resourceName := v1.ResourceName(parts[0])
                resourceQuantity, err := resource.ParseQuantity(parts[1])
                if err != nil {
                        return nil, err
                }
                result[resourceName] = resourceQuantity
        }
        return result, nil
}

// CreateQueueCommand executes a command such as open/close
func CreateQueueCommand(vcClient *versioned.Clientset, ns, name string, action vcbus.Action) error {
        queue, err := vcClient.SchedulingV1beta1().Queues().Get(context.TODO(), name, metav1.GetOptions{})
        if err != nil {
                return err
        }
        ctrlRef := metav1.NewControllerRef(queue, helpers.V1beta1QueueKind)
        cmd := &vcbus.Command{
                ObjectMeta: metav1.ObjectMeta{
                        GenerateName: fmt.Sprintf("%s-%s-",
                                queue.Name, strings.ToLower(string(action))),
                        Namespace: queue.Namespace,
                        OwnerReferences: []metav1.OwnerReference{
                                *ctrlRef,
                        },
                },
                TargetObject: ctrlRef,
                Action:       string(action),
        }

        if _, err := vcClient.BusV1alpha1().Commands(ns).Create(context.TODO(), cmd, metav1.CreateOptions{}); err != nil {
                return err
        }

        return nil
}

// CreateJobCommand executes a command such as resume/suspend.
func CreateJobCommand(config *rest.Config, ns, name string, action vcbus.Action) error {
        jobClient := versioned.NewForConfigOrDie(config)
        job, err := jobClient.BatchV1alpha1().Jobs(ns).Get(context.TODO(), name, metav1.GetOptions{})
        if err != nil {
                return err
        }

        ctrlRef := metav1.NewControllerRef(job, helpers.JobKind)
        cmd := &vcbus.Command{
                ObjectMeta: metav1.ObjectMeta{
                        GenerateName: fmt.Sprintf("%s-%s-",
                                job.Name, strings.ToLower(string(action))),
                        Namespace: job.Namespace,
                        OwnerReferences: []metav1.OwnerReference{
                                *ctrlRef,
                        },
                },
                TargetObject: ctrlRef,
                Action:       string(action),
        }

        if _, err := jobClient.BusV1alpha1().Commands(ns).Create(context.TODO(), cmd, metav1.CreateOptions{}); err != nil {
                return err
        }

        return nil
}

// TranslateTimestampSince translates the time stamp.
func TranslateTimestampSince(timestamp metav1.Time) string {
        if timestamp.IsZero() {
                return "<unknown>"
        }
        return HumanDuration(time.Since(timestamp.Time))
}

// HumanDuration translate time.Duration to human readable time string.
func HumanDuration(d time.Duration) string {
        // Allow deviation no more than 2 seconds(excluded) to tolerate machine time
        // inconsistence, it can be considered as almost now.
        if seconds := int(d.Seconds()); seconds < -1 {
                return "<invalid>"
        } else if seconds < 0 {
                return "0s"
        } else if seconds < 60*2 {
                return fmt.Sprintf("%ds", seconds)
        }
        minutes := int(d / time.Minute)
        if minutes < 10 {
                s := int(d/time.Second) % 60
                if s == 0 {
                        return fmt.Sprintf("%dm", minutes)
                }
                return fmt.Sprintf("%dm%ds", minutes, s)
        } else if minutes < 60*3 {
                return fmt.Sprintf("%dm", minutes)
        }
        hours := int(d / time.Hour)
        if hours < 8 {
                m := int(d/time.Minute) % 60
                if m == 0 {
                        return fmt.Sprintf("%dh", hours)
                }
                return fmt.Sprintf("%dh%dm", hours, m)
        } else if hours < 48 {
                return fmt.Sprintf("%dh", hours)
        } else if hours < 24*8 {
                h := hours % 24
                if h == 0 {
                        return fmt.Sprintf("%dd", hours/24)
                }
                return fmt.Sprintf("%dd%dh", hours/24, h)
        } else if hours < 24*365*2 {
                return fmt.Sprintf("%dd", hours/24)
        } else if hours < 24*365*8 {
                return fmt.Sprintf("%dy%dd", hours/24/365, (hours/24)%365)
        }
        return fmt.Sprintf("%dy", hours/24/365)
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vcancel

import (
        "context"
        "fmt"

        "github.com/spf13/cobra"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

        "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/volcano/pkg/cli/util"
)

type cancelFlags struct {
        util.CommonFlags

        Namespace string
        JobName   string
}

var cancelJobFlags = &cancelFlags{}

// InitCancelFlags init the cancel command flags.
func InitCancelFlags(cmd *cobra.Command) {
        util.InitFlags(cmd, &cancelJobFlags.CommonFlags)

        cmd.Flags().StringVarP(&cancelJobFlags.Namespace, "namespace", "N", "default", "the namespace of job")
        cmd.Flags().StringVarP(&cancelJobFlags.JobName, "name", "n", "", "the name of job")
}

// CancelJob cancel the job.
func CancelJob() error {
        config, err := util.BuildConfig(cancelJobFlags.Master, cancelJobFlags.Kubeconfig)
        if err != nil {
                return err
        }

        if cancelJobFlags.JobName == "" {
                err := fmt.Errorf("job name is mandatory to cancel a particular job")
                return err
        }

        jobClient := versioned.NewForConfigOrDie(config)
        err = jobClient.BatchV1alpha1().Jobs(cancelJobFlags.Namespace).Delete(context.TODO(), cancelJobFlags.JobName, metav1.DeleteOptions{})
        if err != nil {
                return err
        }
        fmt.Printf("cancel job %v successfully\n", cancelJobFlags.JobName)
        return nil
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vresume

import (
        "fmt"

        "github.com/spf13/cobra"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/volcano/pkg/cli/util"
)

type resumeFlags struct {
        util.CommonFlags

        Namespace string
        JobName   string
}

var resumeJobFlags = &resumeFlags{}

// InitResumeFlags init resume command flags.
func InitResumeFlags(cmd *cobra.Command) {
        util.InitFlags(cmd, &resumeJobFlags.CommonFlags)

        cmd.Flags().StringVarP(&resumeJobFlags.Namespace, "namespace", "N", "default", "the namespace of job")
        cmd.Flags().StringVarP(&resumeJobFlags.JobName, "name", "n", "", "the name of job")
}

// ResumeJob resumes the job.
func ResumeJob() error {
        config, err := util.BuildConfig(resumeJobFlags.Master, resumeJobFlags.Kubeconfig)
        if err != nil {
                return err
        }
        if resumeJobFlags.JobName == "" {
                err := fmt.Errorf("job name is mandatory to resume a particular job")
                return err
        }

        return util.CreateJobCommand(config,
                resumeJobFlags.Namespace, resumeJobFlags.JobName,
                v1alpha1.ResumeJobAction)
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vsuspend

import (
        "fmt"

        "github.com/spf13/cobra"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/volcano/pkg/cli/util"
)

type suspendFlags struct {
        util.CommonFlags

        Namespace string
        JobName   string
}

var suspendJobFlags = &suspendFlags{}

// InitSuspendFlags init suspend related flags.
func InitSuspendFlags(cmd *cobra.Command) {
        util.InitFlags(cmd, &suspendJobFlags.CommonFlags)

        cmd.Flags().StringVarP(&suspendJobFlags.Namespace, "namespace", "N", "default", "the namespace of job")
        cmd.Flags().StringVarP(&suspendJobFlags.JobName, "name", "n", "", "the name of job")
}

// SuspendJob suspends the job.
func SuspendJob() error {
        config, err := util.BuildConfig(suspendJobFlags.Master, suspendJobFlags.Kubeconfig)
        if err != nil {
                return err
        }

        if suspendJobFlags.JobName == "" {
                err := fmt.Errorf("job name is mandatory to suspend a particular job")
                return err
        }

        return util.CreateJobCommand(config,
                suspendJobFlags.Namespace, suspendJobFlags.JobName,
                v1alpha1.AbortJobAction)
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apis

import (
        "fmt"

        v1 "k8s.io/api/core/v1"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
)

//JobInfo struct.
type JobInfo struct {
        Namespace string
        Name      string

        Job  *batch.Job
        Pods map[string]map[string]*v1.Pod
}

//Clone function clones the k8s pod values to the JobInfo struct.
func (ji *JobInfo) Clone() *JobInfo {
        job := &JobInfo{
                Namespace: ji.Namespace,
                Name:      ji.Name,
                Job:       ji.Job,

                Pods: make(map[string]map[string]*v1.Pod),
        }

        for key, pods := range ji.Pods {
                job.Pods[key] = make(map[string]*v1.Pod)
                for pn, pod := range pods {
                        job.Pods[key][pn] = pod
                }
        }

        return job
}

//SetJob sets the volcano jobs values to the JobInfo struct.
func (ji *JobInfo) SetJob(job *batch.Job) {
        ji.Name = job.Name
        ji.Namespace = job.Namespace
        ji.Job = job
}

//AddPod adds the k8s pod object values to the Pods field
//of JobStruct if it doesn't exist. Otherwise it throws error.
func (ji *JobInfo) AddPod(pod *v1.Pod) error {
        taskName, found := pod.Annotations[batch.TaskSpecKey]
        if !found {
                return fmt.Errorf("failed to taskName of Pod <%s/%s>",
                        pod.Namespace, pod.Name)
        }

        _, found = pod.Annotations[batch.JobVersion]
        if !found {
                return fmt.Errorf("failed to find jobVersion of Pod <%s/%s>",
                        pod.Namespace, pod.Name)
        }

        if _, found := ji.Pods[taskName]; !found {
                ji.Pods[taskName] = make(map[string]*v1.Pod)
        }
        if _, found := ji.Pods[taskName][pod.Name]; found {
                return fmt.Errorf("duplicated pod")
        }
        ji.Pods[taskName][pod.Name] = pod

        return nil
}

//UpdatePod updates the k8s pod object values to the existing pod.
func (ji *JobInfo) UpdatePod(pod *v1.Pod) error {
        taskName, found := pod.Annotations[batch.TaskSpecKey]
        if !found {
                return fmt.Errorf("failed to find taskName of Pod <%s/%s>",
                        pod.Namespace, pod.Name)
        }
        _, found = pod.Annotations[batch.JobVersion]
        if !found {
                return fmt.Errorf("failed to find jobVersion of Pod <%s/%s>",
                        pod.Namespace, pod.Name)
        }

        if _, found := ji.Pods[taskName]; !found {
                return fmt.Errorf("can not find task %s in cache", taskName)
        }
        if _, found := ji.Pods[taskName][pod.Name]; !found {
                return fmt.Errorf("can not find pod <%s/%s> in cache",
                        pod.Namespace, pod.Name)
        }
        ji.Pods[taskName][pod.Name] = pod

        return nil
}

//DeletePod deletes the given k8s pod from the JobInfo struct.
func (ji *JobInfo) DeletePod(pod *v1.Pod) error {
        taskName, found := pod.Annotations[batch.TaskSpecKey]
        if !found {
                return fmt.Errorf("failed to find taskName of Pod <%s/%s>",
                        pod.Namespace, pod.Name)
        }
        _, found = pod.Annotations[batch.JobVersion]
        if !found {
                return fmt.Errorf("failed to find jobVersion of Pod <%s/%s>",
                        pod.Namespace, pod.Name)
        }

        if pods, found := ji.Pods[taskName]; found {
                delete(pods, pod.Name)
                if len(pods) == 0 {
                        delete(ji.Pods, taskName)
                }
        }

        return nil
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apis

import (
        "fmt"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
)

//Request struct.
type Request struct {
        Namespace string
        JobName   string
        TaskName  string
        QueueName string

        Event      v1alpha1.Event
        ExitCode   int32
        Action     v1alpha1.Action
        JobVersion int32
}

// String function returns the request in string format.
func (r Request) String() string {
        return fmt.Sprintf(
                "Queue: %s, Job: %s/%s, Task:%s, Event:%s, ExitCode:%d, Action:%s, JobVersion: %d",
                r.QueueName, r.Namespace, r.JobName, r.TaskName, r.Event, r.ExitCode, r.Action, r.JobVersion)
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
        "fmt"
        "sync"
        "time"

        "golang.org/x/time/rate"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/util/wait"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/volcano/pkg/controllers/apis"
)

type jobCache struct {
        sync.Mutex

        jobs        map[string]*apis.JobInfo
        deletedJobs workqueue.RateLimitingInterface
}

func keyFn(ns, name string) string {
        return fmt.Sprintf("%s/%s", ns, name)
}

//JobKeyByName gets the key for the job name.
func JobKeyByName(namespace string, name string) string {
        return keyFn(namespace, name)
}

//JobKeyByReq gets the key for the job request.
func JobKeyByReq(req *apis.Request) string {
        return keyFn(req.Namespace, req.JobName)
}

//JobKey gets the "ns"/"name" format of the given job.
func JobKey(job *v1alpha1.Job) string {
        return keyFn(job.Namespace, job.Name)
}

func jobTerminated(job *apis.JobInfo) bool {
        return job.Job == nil && len(job.Pods) == 0
}

func jobKeyOfPod(pod *v1.Pod) (string, error) {
        jobName, found := pod.Annotations[v1alpha1.JobNameKey]
        if !found {
                return "", fmt.Errorf("failed to find job name of pod <%s/%s>",
                        pod.Namespace, pod.Name)
        }

        return keyFn(pod.Namespace, jobName), nil
}

// New gets the job Cache.
func New() Cache {
        queue := workqueue.NewMaxOfRateLimiter(
                workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 180*time.Second),
                // 10 qps, 100 bucket size.  This is only for retry speed and its only the overall factor (not per item)
                &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
        )

        return &jobCache{
                jobs:        map[string]*apis.JobInfo{},
                deletedJobs: workqueue.NewRateLimitingQueue(queue),
        }
}

func (jc *jobCache) Get(key string) (*apis.JobInfo, error) {
        jc.Lock()
        defer jc.Unlock()

        job, found := jc.jobs[key]
        if !found {
                return nil, fmt.Errorf("failed to find job <%s>", key)
        }

        if job.Job == nil {
                return nil, fmt.Errorf("job <%s> is not ready", key)
        }

        return job.Clone(), nil
}

func (jc *jobCache) GetStatus(key string) (*v1alpha1.JobStatus, error) {
        jc.Lock()
        defer jc.Unlock()

        job, found := jc.jobs[key]
        if !found {
                return nil, fmt.Errorf("failed to find job <%s>", key)
        }

        if job.Job == nil {
                return nil, fmt.Errorf("job <%s> is not ready", key)
        }

        status := job.Job.Status

        return &status, nil
}

func (jc *jobCache) Add(job *v1alpha1.Job) error {
        jc.Lock()
        defer jc.Unlock()
        key := JobKey(job)
        if jobInfo, found := jc.jobs[key]; found {
                if jobInfo.Job == nil {
                        jobInfo.SetJob(job)

                        return nil
                }
                return fmt.Errorf("duplicated jobInfo <%v>", key)
        }

        jc.jobs[key] = &apis.JobInfo{
                Name:      job.Name,
                Namespace: job.Namespace,

                Job:  job,
                Pods: make(map[string]map[string]*v1.Pod),
        }

        return nil
}

func (jc *jobCache) Update(obj *v1alpha1.Job) error {
        jc.Lock()
        defer jc.Unlock()

        key := JobKey(obj)
        job, found := jc.jobs[key]
        if !found {
                return fmt.Errorf("failed to find job <%v>", key)
        }
        job.Job = obj

        return nil
}

func (jc *jobCache) Delete(obj *v1alpha1.Job) error {
        jc.Lock()
        defer jc.Unlock()

        key := JobKey(obj)
        jobInfo, found := jc.jobs[key]
        if !found {
                return fmt.Errorf("failed to find job <%v>", key)
        }
        jobInfo.Job = nil
        jc.deleteJob(jobInfo)

        return nil
}

func (jc *jobCache) AddPod(pod *v1.Pod) error {
        jc.Lock()
        defer jc.Unlock()

        key, err := jobKeyOfPod(pod)
        if err != nil {
                return err
        }

        job, found := jc.jobs[key]
        if !found {
                job = &apis.JobInfo{
                        Pods: make(map[string]map[string]*v1.Pod),
                }
                jc.jobs[key] = job
        }

        return job.AddPod(pod)
}

func (jc *jobCache) UpdatePod(pod *v1.Pod) error {
        jc.Lock()
        defer jc.Unlock()

        key, err := jobKeyOfPod(pod)
        if err != nil {
                return err
        }

        job, found := jc.jobs[key]
        if !found {
                job = &apis.JobInfo{
                        Pods: make(map[string]map[string]*v1.Pod),
                }
                jc.jobs[key] = job
        }

        return job.UpdatePod(pod)
}

func (jc *jobCache) DeletePod(pod *v1.Pod) error {
        jc.Lock()
        defer jc.Unlock()

        key, err := jobKeyOfPod(pod)
        if err != nil {
                return err
        }

        job, found := jc.jobs[key]
        if !found {
                job = &apis.JobInfo{
                        Pods: make(map[string]map[string]*v1.Pod),
                }
                jc.jobs[key] = job
        }

        if err := job.DeletePod(pod); err != nil {
                return err
        }

        if jc.jobs[key].Job == nil {
                jc.deleteJob(job)
        }

        return nil
}

func (jc *jobCache) Run(stopCh <-chan struct{}) {
        wait.Until(jc.worker, 0, stopCh)
}

func (jc *jobCache) TaskCompleted(jobKey, taskName string) bool {
        jc.Lock()
        defer jc.Unlock()

        var taskReplicas, completed int32

        jobInfo, found := jc.jobs[jobKey]
        if !found {
                return false
        }

        taskPods, found := jobInfo.Pods[taskName]

        if !found {
                return false
        }

        if jobInfo.Job == nil {
                return false
        }

        for _, task := range jobInfo.Job.Spec.Tasks {
                if task.Name == taskName {
                        taskReplicas = task.Replicas
                        break
                }
        }
        if taskReplicas <= 0 {
                return false
        }

        for _, pod := range taskPods {
                if pod.Status.Phase == v1.PodSucceeded {
                        completed++
                }
        }
        return completed >= taskReplicas
}

func (jc *jobCache) TaskFailed(jobKey, taskName string) bool {
        jc.Lock()
        defer jc.Unlock()

        var taskReplicas, retried, maxRetry int32

        jobInfo, found := jc.jobs[jobKey]
        if !found {
                return false
        }

        taskPods, found := jobInfo.Pods[taskName]

        if !found || jobInfo.Job == nil {
                return false
        }

        for _, task := range jobInfo.Job.Spec.Tasks {
                if task.Name == taskName {
                        maxRetry = task.MaxRetry
                        taskReplicas = task.Replicas
                        break
                }
        }

        // maxRetry == -1 means no limit
        if taskReplicas == 0 || maxRetry == -1 {
                return false
        }

        // Compatible with existing job
        if maxRetry == 0 {
                maxRetry = 3
        }

        for _, pod := range taskPods {
                if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodPending {
                        for j := range pod.Status.InitContainerStatuses {
                                stat := pod.Status.InitContainerStatuses[j]
                                retried += stat.RestartCount
                        }
                        for j := range pod.Status.ContainerStatuses {
                                stat := pod.Status.ContainerStatuses[j]
                                retried += stat.RestartCount
                        }
                }
        }
        return retried > maxRetry
}

func (jc *jobCache) worker() {
        for jc.processCleanupJob() {
        }
}

func (jc *jobCache) processCleanupJob() bool {
        obj, shutdown := jc.deletedJobs.Get()
        if shutdown {
                return false
        }
        defer jc.deletedJobs.Done(obj)

        job, ok := obj.(*apis.JobInfo)
        if !ok {
                klog.Errorf("failed to convert %v to *apis.JobInfo", obj)
                return true
        }

        jc.Mutex.Lock()
        defer jc.Mutex.Unlock()

        if jobTerminated(job) {
                jc.deletedJobs.Forget(obj)
                key := keyFn(job.Namespace, job.Name)
                delete(jc.jobs, key)
                klog.V(3).Infof("Job <%s> was deleted.", key)
        } else {
                // Retry
                jc.deleteJob(job)
        }
        return true
}

func (jc *jobCache) deleteJob(job *apis.JobInfo) {
        klog.V(3).Infof("Try to delete Job <%v/%v>",
                job.Namespace, job.Name)

        jc.deletedJobs.AddRateLimited(job)
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package garbagecollector

import (
        "context"
        "fmt"
        "time"

        "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/util/wait"
        "k8s.io/client-go/tools/cache"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        vcclientset "volcano.sh/apis/pkg/client/clientset/versioned"
        informerfactory "volcano.sh/apis/pkg/client/informers/externalversions"
        batchinformers "volcano.sh/apis/pkg/client/informers/externalversions/batch/v1alpha1"
        batchlisters "volcano.sh/apis/pkg/client/listers/batch/v1alpha1"
        "volcano.sh/volcano/pkg/controllers/framework"
)

func init() {
        framework.RegisterController(&gccontroller{})
}

// gccontroller runs reflectors to watch for changes of managed API
// objects. Currently it only watches Jobs. Triggered by Job creation
// and updates, it enqueues Jobs that have non-nil `.spec.ttlSecondsAfterFinished`
// to the `queue`. The gccontroller has workers who consume `queue`, check whether
// the Job TTL has expired or not; if the Job TTL hasn't expired, it will add the
// Job to the queue after the TTL is expected to expire; if the TTL has expired, the
// worker will send requests to the API server to delete the Jobs accordingly.
// This is implemented outside of Job controller for separation of concerns, and
// because it will be extended to handle other finishable resource types.
type gccontroller struct {
        vcClient vcclientset.Interface

        jobInformer batchinformers.JobInformer

        // A store of jobs
        jobLister batchlisters.JobLister
        jobSynced func() bool

        // queues that need to be updated.
        queue workqueue.RateLimitingInterface
}

func (gc *gccontroller) Name() string {
        return "gc-controller"
}

// Initialize creates an instance of gccontroller.
func (gc *gccontroller) Initialize(opt *framework.ControllerOption) error {
        gc.vcClient = opt.VolcanoClient
        jobInformer := informerfactory.NewSharedInformerFactory(gc.vcClient, 0).Batch().V1alpha1().Jobs()

        gc.jobInformer = jobInformer
        gc.jobLister = jobInformer.Lister()
        gc.jobSynced = jobInformer.Informer().HasSynced
        gc.queue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())

        jobInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    gc.addJob,
                UpdateFunc: gc.updateJob,
        })

        return nil
}

// Run starts the worker to clean up Jobs.
func (gc *gccontroller) Run(stopCh <-chan struct{}) {
        defer gc.queue.ShutDown()

        klog.Infof("Starting garbage collector")
        defer klog.Infof("Shutting down garbage collector")

        go gc.jobInformer.Informer().Run(stopCh)
        if !cache.WaitForCacheSync(stopCh, gc.jobSynced) {
                return
        }

        go wait.Until(gc.worker, time.Second, stopCh)

        <-stopCh
}

func (gc *gccontroller) addJob(obj interface{}) {
        job := obj.(*v1alpha1.Job)
        klog.V(4).Infof("Adding job %s/%s", job.Namespace, job.Name)

        if job.DeletionTimestamp == nil && needsCleanup(job) {
                gc.enqueue(job)
        }
}

func (gc *gccontroller) updateJob(old, cur interface{}) {
        job := cur.(*v1alpha1.Job)
        klog.V(4).Infof("Updating job %s/%s", job.Namespace, job.Name)

        if job.DeletionTimestamp == nil && needsCleanup(job) {
                gc.enqueue(job)
        }
}

func (gc *gccontroller) enqueue(job *v1alpha1.Job) {
        klog.V(4).Infof("Add job %s/%s to cleanup", job.Namespace, job.Name)
        key, err := cache.MetaNamespaceKeyFunc(job)
        if err != nil {
                klog.Errorf("couldn't get key for object %#v: %v", job, err)
                return
        }

        gc.queue.Add(key)
}

func (gc *gccontroller) enqueueAfter(job *v1alpha1.Job, after time.Duration) {
        key, err := cache.MetaNamespaceKeyFunc(job)
        if err != nil {
                klog.Errorf("couldn't get key for object %#v: %v", job, err)
                return
        }

        gc.queue.AddAfter(key, after)
}

func (gc *gccontroller) worker() {
        for gc.processNextWorkItem() {
        }
}

func (gc *gccontroller) processNextWorkItem() bool {
        key, quit := gc.queue.Get()
        if quit {
                return false
        }
        defer gc.queue.Done(key)

        err := gc.processJob(key.(string))
        gc.handleErr(err, key)

        return true
}

func (gc *gccontroller) handleErr(err error, key interface{}) {
        if err == nil {
                gc.queue.Forget(key)
                return
        }

        klog.Errorf("error cleaning up Job %v, will retry: %v", key, err)
        gc.queue.AddRateLimited(key)
}

// processJob will check the Job's state and TTL and delete the Job when it
// finishes and its TTL after finished has expired. If the Job hasn't finished or
// its TTL hasn't expired, it will be added to the queue after the TTL is expected
// to expire.
// This function is not meant to be invoked concurrently with the same key.
func (gc *gccontroller) processJob(key string) error {
        namespace, name, err := cache.SplitMetaNamespaceKey(key)
        if err != nil {
                return err
        }

        klog.V(4).Infof("Checking if Job %s/%s is ready for cleanup", namespace, name)
        // Ignore the Jobs that are already deleted or being deleted, or the ones that don't need clean up.
        job, err := gc.jobLister.Jobs(namespace).Get(name)
        if errors.IsNotFound(err) {
                return nil
        }
        if err != nil {
                return err
        }

        if expired, err := gc.processTTL(job); err != nil {
                return err
        } else if !expired {
                return nil
        }

        // The Job's TTL is assumed to have expired, but the Job TTL might be stale.
        // Before deleting the Job, do a final sanity check.
        // If TTL is modified before we do this check, we cannot be sure if the TTL truly expires.
        // The latest Job may have a different UID, but it's fine because the checks will be run again.
        fresh, err := gc.vcClient.BatchV1alpha1().Jobs(namespace).Get(context.TODO(), name, metav1.GetOptions{})
        if errors.IsNotFound(err) {
                return nil
        }
        if err != nil {
                return err
        }
        // Use the latest Job TTL to see if the TTL truly expires.
        if expired, err := gc.processTTL(fresh); err != nil {
                return err
        } else if !expired {
                return nil
        }
        // Cascade deletes the Jobs if TTL truly expires.
        policy := metav1.DeletePropagationForeground
        options := metav1.DeleteOptions{
                PropagationPolicy: &policy,
                Preconditions:     &metav1.Preconditions{UID: &fresh.UID},
        }
        klog.V(4).Infof("Cleaning up Job %s/%s", namespace, name)
        return gc.vcClient.BatchV1alpha1().Jobs(fresh.Namespace).Delete(context.TODO(), fresh.Name, options)
}

// processTTL checks whether a given Job's TTL has expired, and add it to the queue after the TTL is expected to expire
// if the TTL will expire later.
func (gc *gccontroller) processTTL(job *v1alpha1.Job) (expired bool, err error) {
        // We don't care about the Jobs that are going to be deleted, or the ones that don't need clean up.
        if job.DeletionTimestamp != nil || !needsCleanup(job) {
                return false, nil
        }

        now := time.Now()
        t, err := timeLeft(job, &now)
        if err != nil {
                return false, err
        }

        // TTL has expired
        if *t <= 0 {
                return true, nil
        }

        gc.enqueueAfter(job, *t)
        return false, nil
}

// needsCleanup checks whether a Job has finished and has a TTL set.
func needsCleanup(j *v1alpha1.Job) bool {
        return j.Spec.TTLSecondsAfterFinished != nil && isJobFinished(j)
}

func isJobFinished(job *v1alpha1.Job) bool {
        return job.Status.State.Phase == v1alpha1.Completed ||
                job.Status.State.Phase == v1alpha1.Failed ||
                job.Status.State.Phase == v1alpha1.Terminated
}

func getFinishAndExpireTime(j *v1alpha1.Job) (*time.Time, *time.Time, error) {
        if !needsCleanup(j) {
                return nil, nil, fmt.Errorf("job %s/%s should not be cleaned up", j.Namespace, j.Name)
        }
        finishAt, err := jobFinishTime(j)
        if err != nil {
                return nil, nil, err
        }
        finishAtUTC := finishAt.UTC()
        expireAtUTC := finishAtUTC.Add(time.Duration(*j.Spec.TTLSecondsAfterFinished) * time.Second)
        return &finishAtUTC, &expireAtUTC, nil
}

func timeLeft(j *v1alpha1.Job, since *time.Time) (*time.Duration, error) {
        finishAt, expireAt, err := getFinishAndExpireTime(j)
        if err != nil {
                return nil, err
        }
        if finishAt.UTC().After(since.UTC()) {
                klog.Warningf("Warning: Found Job %s/%s finished in the future. This is likely due to time skew in the cluster. Job cleanup will be deferred.", j.Namespace, j.Name)
        }
        remaining := expireAt.UTC().Sub(since.UTC())
        klog.V(4).Infof("Found Job %s/%s finished at %v, remaining TTL %v since %v, TTL will expire at %v", j.Namespace, j.Name, finishAt.UTC(), remaining, since.UTC(), expireAt.UTC())
        return &remaining, nil
}

// jobFinishTime takes an already finished Job and returns the time it finishes.
func jobFinishTime(finishedJob *v1alpha1.Job) (metav1.Time, error) {
        if finishedJob.Status.State.LastTransitionTime.IsZero() {
                return metav1.Time{}, fmt.Errorf("unable to find the time when the Job %s/%s finished", finishedJob.Namespace, finishedJob.Name)
        }
        return finishedJob.Status.State.LastTransitionTime, nil
}

/*
Copyright 2017 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "fmt"
        "hash"
        "hash/fnv"
        "time"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/util/wait"
        coreinformers "k8s.io/client-go/informers/core/v1"
        kubeschedulinginformers "k8s.io/client-go/informers/scheduling/v1beta1"
        "k8s.io/client-go/kubernetes"
        corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
        corelisters "k8s.io/client-go/listers/core/v1"
        kubeschedulinglisters "k8s.io/client-go/listers/scheduling/v1beta1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/client-go/tools/record"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        batchv1alpha1 "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        vcclientset "volcano.sh/apis/pkg/client/clientset/versioned"
        vcscheme "volcano.sh/apis/pkg/client/clientset/versioned/scheme"
        informerfactory "volcano.sh/apis/pkg/client/informers/externalversions"
        batchinformer "volcano.sh/apis/pkg/client/informers/externalversions/batch/v1alpha1"
        businformer "volcano.sh/apis/pkg/client/informers/externalversions/bus/v1alpha1"
        schedulinginformers "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1"
        batchlister "volcano.sh/apis/pkg/client/listers/batch/v1alpha1"
        buslister "volcano.sh/apis/pkg/client/listers/bus/v1alpha1"
        schedulinglisters "volcano.sh/apis/pkg/client/listers/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
        jobcache "volcano.sh/volcano/pkg/controllers/cache"
        "volcano.sh/volcano/pkg/controllers/framework"
        "volcano.sh/volcano/pkg/controllers/job/state"
)

func init() {
        framework.RegisterController(&jobcontroller{})
}

// jobcontroller the Job jobcontroller type.
type jobcontroller struct {
        kubeClient kubernetes.Interface
        vcClient   vcclientset.Interface

        jobInformer   batchinformer.JobInformer
        podInformer   coreinformers.PodInformer
        pvcInformer   coreinformers.PersistentVolumeClaimInformer
        pgInformer    schedulinginformers.PodGroupInformer
        svcInformer   coreinformers.ServiceInformer
        cmdInformer   businformer.CommandInformer
        pcInformer    kubeschedulinginformers.PriorityClassInformer
        queueInformer schedulinginformers.QueueInformer

        // A store of jobs
        jobLister batchlister.JobLister
        jobSynced func() bool

        // A store of pods
        podLister corelisters.PodLister
        podSynced func() bool

        pvcLister corelisters.PersistentVolumeClaimLister
        pvcSynced func() bool

        // A store of podgroups
        pgLister schedulinglisters.PodGroupLister
        pgSynced func() bool

        // A store of service
        svcLister corelisters.ServiceLister
        svcSynced func() bool

        cmdLister buslister.CommandLister
        cmdSynced func() bool

        pcLister kubeschedulinglisters.PriorityClassLister
        pcSynced func() bool

        queueLister schedulinglisters.QueueLister
        queueSynced func() bool

        // queue that need to sync up
        queueList    []workqueue.RateLimitingInterface
        commandQueue workqueue.RateLimitingInterface
        cache        jobcache.Cache
        // Job Event recorder
        recorder record.EventRecorder

        errTasks      workqueue.RateLimitingInterface
        workers       uint32
        maxRequeueNum int
}

func (cc *jobcontroller) Name() string {
        return "job-controller"
}

// Initialize creates the new Job job controller.
func (cc *jobcontroller) Initialize(opt *framework.ControllerOption) error {
        cc.kubeClient = opt.KubeClient
        cc.vcClient = opt.VolcanoClient

        sharedInformers := opt.SharedInformerFactory
        workers := opt.WorkerNum
        // Initialize event client
        eventBroadcaster := record.NewBroadcaster()
        eventBroadcaster.StartLogging(klog.Infof)
        eventBroadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: cc.kubeClient.CoreV1().Events("")})
        recorder := eventBroadcaster.NewRecorder(vcscheme.Scheme, v1.EventSource{Component: "vc-controller-manager"})

        cc.queueList = make([]workqueue.RateLimitingInterface, workers)
        cc.commandQueue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
        cc.cache = jobcache.New()
        cc.errTasks = newRateLimitingQueue()
        cc.recorder = recorder
        cc.workers = workers
        cc.maxRequeueNum = opt.MaxRequeueNum
        if cc.maxRequeueNum < 0 {
                cc.maxRequeueNum = -1
        }

        var i uint32
        for i = 0; i < workers; i++ {
                cc.queueList[i] = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
        }

        cc.jobInformer = informerfactory.NewSharedInformerFactory(cc.vcClient, 0).Batch().V1alpha1().Jobs()
        cc.jobInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    cc.addJob,
                UpdateFunc: cc.updateJob,
                DeleteFunc: cc.deleteJob,
        })
        cc.jobLister = cc.jobInformer.Lister()
        cc.jobSynced = cc.jobInformer.Informer().HasSynced

        cc.cmdInformer = informerfactory.NewSharedInformerFactory(cc.vcClient, 0).Bus().V1alpha1().Commands()
        cc.cmdInformer.Informer().AddEventHandler(
                cache.FilteringResourceEventHandler{
                        FilterFunc: func(obj interface{}) bool {
                                switch v := obj.(type) {
                                case *busv1alpha1.Command:
                                        if v.TargetObject != nil &&
                                                v.TargetObject.APIVersion == batchv1alpha1.SchemeGroupVersion.String() &&
                                                v.TargetObject.Kind == "Job" {
                                                return true
                                        }

                                        return false
                                default:
                                        return false
                                }
                        },
                        Handler: cache.ResourceEventHandlerFuncs{
                                AddFunc: cc.addCommand,
                        },
                },
        )
        cc.cmdLister = cc.cmdInformer.Lister()
        cc.cmdSynced = cc.cmdInformer.Informer().HasSynced

        cc.podInformer = sharedInformers.Core().V1().Pods()
        cc.podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    cc.addPod,
                UpdateFunc: cc.updatePod,
                DeleteFunc: cc.deletePod,
        })

        cc.podLister = cc.podInformer.Lister()
        cc.podSynced = cc.podInformer.Informer().HasSynced

        cc.pvcInformer = sharedInformers.Core().V1().PersistentVolumeClaims()
        cc.pvcLister = cc.pvcInformer.Lister()
        cc.pvcSynced = cc.pvcInformer.Informer().HasSynced

        cc.svcInformer = sharedInformers.Core().V1().Services()
        cc.svcLister = cc.svcInformer.Lister()
        cc.svcSynced = cc.svcInformer.Informer().HasSynced

        cc.pgInformer = informerfactory.NewSharedInformerFactory(cc.vcClient, 0).Scheduling().V1beta1().PodGroups()
        cc.pgInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                UpdateFunc: cc.updatePodGroup,
        })
        cc.pgLister = cc.pgInformer.Lister()
        cc.pgSynced = cc.pgInformer.Informer().HasSynced

        cc.pcInformer = sharedInformers.Scheduling().V1beta1().PriorityClasses()
        cc.pcLister = cc.pcInformer.Lister()
        cc.pcSynced = cc.pcInformer.Informer().HasSynced

        cc.queueInformer = informerfactory.NewSharedInformerFactory(cc.vcClient, 0).Scheduling().V1beta1().Queues()
        cc.queueLister = cc.queueInformer.Lister()
        cc.queueSynced = cc.queueInformer.Informer().HasSynced

        // Register actions
        state.SyncJob = cc.syncJob
        state.KillJob = cc.killJob

        return nil
}

// Run start JobController.
func (cc *jobcontroller) Run(stopCh <-chan struct{}) {
        go cc.jobInformer.Informer().Run(stopCh)
        go cc.podInformer.Informer().Run(stopCh)
        go cc.pvcInformer.Informer().Run(stopCh)
        go cc.pgInformer.Informer().Run(stopCh)
        go cc.svcInformer.Informer().Run(stopCh)
        go cc.cmdInformer.Informer().Run(stopCh)
        go cc.pcInformer.Informer().Run(stopCh)
        go cc.queueInformer.Informer().Run(stopCh)

        cache.WaitForCacheSync(stopCh, cc.jobSynced, cc.podSynced, cc.pgSynced,
                cc.svcSynced, cc.cmdSynced, cc.pvcSynced, cc.pcSynced, cc.queueSynced)

        go wait.Until(cc.handleCommands, 0, stopCh)
        var i uint32
        for i = 0; i < cc.workers; i++ {
                go func(num uint32) {
                        wait.Until(
                                func() {
                                        cc.worker(num)
                                },
                                time.Second,
                                stopCh)
                }(i)
        }

        go cc.cache.Run(stopCh)

        // Re-sync error tasks.
        go wait.Until(cc.processResyncTask, 0, stopCh)

        klog.Infof("JobController is running ...... ")
}

func (cc *jobcontroller) worker(i uint32) {
        klog.Infof("worker %d start ...... ", i)

        for cc.processNextReq(i) {
        }
}

func (cc *jobcontroller) belongsToThisRoutine(key string, count uint32) bool {
        var hashVal hash.Hash32
        var val uint32

        hashVal = fnv.New32()
        hashVal.Write([]byte(key))

        val = hashVal.Sum32()

        return val%cc.workers == count
}

func (cc *jobcontroller) getWorkerQueue(key string) workqueue.RateLimitingInterface {
        var hashVal hash.Hash32
        var val uint32

        hashVal = fnv.New32()
        hashVal.Write([]byte(key))

        val = hashVal.Sum32()

        queue := cc.queueList[val%cc.workers]

        return queue
}

func (cc *jobcontroller) processNextReq(count uint32) bool {
        queue := cc.queueList[count]
        obj, shutdown := queue.Get()
        if shutdown {
                klog.Errorf("Fail to pop item from queue")
                return false
        }

        req := obj.(apis.Request)
        defer queue.Done(req)

        key := jobcache.JobKeyByReq(&req)
        if !cc.belongsToThisRoutine(key, count) {
                klog.Errorf("should not occur The job does not belongs to this routine key:%s, worker:%d...... ", key, count)
                queueLocal := cc.getWorkerQueue(key)
                queueLocal.Add(req)
                return true
        }

        klog.V(3).Infof("Try to handle request <%v>", req)

        jobInfo, err := cc.cache.Get(key)
        if err != nil {
                // TODO(k82cn): ignore not-ready error.
                klog.Errorf("Failed to get job by <%v> from cache: %v", req, err)
                return true
        }

        st := state.NewState(jobInfo)
        if st == nil {
                klog.Errorf("Invalid state <%s> of Job <%v/%v>",
                        jobInfo.Job.Status.State, jobInfo.Job.Namespace, jobInfo.Job.Name)
                return true
        }

        action := applyPolicies(jobInfo.Job, &req)
        klog.V(3).Infof("Execute <%v> on Job <%s/%s> in <%s> by <%T>.",
                action, req.Namespace, req.JobName, jobInfo.Job.Status.State.Phase, st)

        if action != busv1alpha1.SyncJobAction {
                cc.recordJobEvent(jobInfo.Job.Namespace, jobInfo.Job.Name, batchv1alpha1.ExecuteAction, fmt.Sprintf(
                        "Start to execute action %s ", action))
        }

        if err := st.Execute(action); err != nil {
                if cc.maxRequeueNum == -1 || queue.NumRequeues(req) < cc.maxRequeueNum {
                        klog.V(2).Infof("Failed to handle Job <%s/%s>: %v",
                                jobInfo.Job.Namespace, jobInfo.Job.Name, err)
                        // If any error, requeue it.
                        queue.AddRateLimited(req)
                        return true
                }
                cc.recordJobEvent(jobInfo.Job.Namespace, jobInfo.Job.Name, batchv1alpha1.ExecuteAction, fmt.Sprintf(
                        "Job failed on action %s for retry limit reached", action))
                klog.Warningf("Terminating Job <%s/%s> and releasing resources", jobInfo.Job.Namespace, jobInfo.Job.Name)
                if err = st.Execute(busv1alpha1.TerminateJobAction); err != nil {
                        klog.Errorf("Failed to terminate Job<%s/%s>: %v", jobInfo.Job.Namespace, jobInfo.Job.Name, err)
                }
                klog.Warningf("Dropping job<%s/%s> out of the queue: %v because max retries has reached", jobInfo.Job.Namespace, jobInfo.Job.Name, err)
        }

        // If no error, forget it.
        queue.Forget(req)

        return true
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "reflect"
        "sort"
        "sync"
        "sync/atomic"
        "time"

        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
        jobhelpers "volcano.sh/volcano/pkg/controllers/job/helpers"
        "volcano.sh/volcano/pkg/controllers/job/state"
)

var calMutex sync.Mutex

func (cc *jobcontroller) killJob(jobInfo *apis.JobInfo, podRetainPhase state.PhaseMap, updateStatus state.UpdateStatusFn) error {
        job := jobInfo.Job
        klog.V(3).Infof("Killing Job <%s/%s>, current version %d", job.Namespace, job.Name, job.Status.Version)
        defer klog.V(3).Infof("Finished Job <%s/%s> killing, current version %d", job.Namespace, job.Name, job.Status.Version)

        if job.DeletionTimestamp != nil {
                klog.Infof("Job <%s/%s> is terminating, skip management process.",
                        job.Namespace, job.Name)
                return nil
        }

        var pending, running, terminating, succeeded, failed, unknown int32
        taskStatusCount := make(map[string]batch.TaskState)

        var errs []error
        var total int

        for _, pods := range jobInfo.Pods {
                for _, pod := range pods {
                        total++

                        if pod.DeletionTimestamp != nil {
                                klog.Infof("Pod <%s/%s> is terminating", pod.Namespace, pod.Name)
                                terminating++
                                continue
                        }

                        maxRetry := job.Spec.MaxRetry
                        lastRetry := false
                        if job.Status.RetryCount > maxRetry-1 {
                                lastRetry = true
                        }

                        _, retain := podRetainPhase[pod.Status.Phase]

                        if !retain && !lastRetry {
                                err := cc.deleteJobPod(job.Name, pod)
                                if err == nil {
                                        terminating++
                                        continue
                                }
                                // record the err, and then collect the pod info like retained pod
                                errs = append(errs, err)
                                cc.resyncTask(pod)
                        }

                        classifyAndAddUpPodBaseOnPhase(pod, &pending, &running, &succeeded, &failed, &unknown)
                        calcPodStatus(pod, taskStatusCount)
                }
        }

        if len(errs) != 0 {
                klog.Errorf("failed to kill pods for job %s/%s, with err %+v", job.Namespace, job.Name, errs)
                cc.recorder.Event(job, v1.EventTypeWarning, FailedDeletePodReason,
                        fmt.Sprintf("Error deleting pods: %+v", errs))
                return fmt.Errorf("failed to kill %d pods of %d", len(errs), total)
        }

        job = job.DeepCopy()
        // Job version is bumped only when job is killed
        job.Status.Version++
        job.Status.Pending = pending
        job.Status.Running = running
        job.Status.Succeeded = succeeded
        job.Status.Failed = failed
        job.Status.Terminating = terminating
        job.Status.Unknown = unknown
        job.Status.TaskStatusCount = taskStatusCount

        // Update running duration
        klog.V(3).Infof("Running duration is %s", metav1.Duration{Duration: time.Since(jobInfo.Job.CreationTimestamp.Time)}.ToUnstructured())
        job.Status.RunningDuration = &metav1.Duration{Duration: time.Since(jobInfo.Job.CreationTimestamp.Time)}

        if updateStatus != nil {
                if updateStatus(&job.Status) {
                        job.Status.State.LastTransitionTime = metav1.Now()
                }
        }

        // must be called before update job status
        if err := cc.pluginOnJobDelete(job); err != nil {
                return err
        }

        // Update Job status
        newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(context.TODO(), job, metav1.UpdateOptions{})
        if err != nil {
                klog.Errorf("Failed to update status of Job %v/%v: %v",
                        job.Namespace, job.Name, err)
                return err
        }
        if e := cc.cache.Update(newJob); e != nil {
                klog.Errorf("KillJob - Failed to update Job %v/%v in cache:  %v",
                        newJob.Namespace, newJob.Name, e)
                return e
        }

        // Delete PodGroup
        if err := cc.vcClient.SchedulingV1beta1().PodGroups(job.Namespace).Delete(context.TODO(), job.Name, metav1.DeleteOptions{}); err != nil {
                if !apierrors.IsNotFound(err) {
                        klog.Errorf("Failed to delete PodGroup of Job %v/%v: %v",
                                job.Namespace, job.Name, err)
                        return err
                }
        }

        // NOTE(k82cn): DO NOT delete input/output until job is deleted.

        return nil
}

func (cc *jobcontroller) initiateJob(job *batch.Job) (*batch.Job, error) {
        klog.V(3).Infof("Starting to initiate Job <%s/%s>", job.Namespace, job.Name)
        jobInstance, err := cc.initJobStatus(job)
        if err != nil {
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.JobStatusError),
                        fmt.Sprintf("Failed to initialize job status, err: %v", err))
                return nil, err
        }

        if err := cc.pluginOnJobAdd(jobInstance); err != nil {
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.PluginError),
                        fmt.Sprintf("Execute plugin when job add failed, err: %v", err))
                return nil, err
        }

        newJob, err := cc.createJobIOIfNotExist(jobInstance)
        if err != nil {
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.PVCError),
                        fmt.Sprintf("Failed to create PVC, err: %v", err))
                return nil, err
        }

        if err := cc.createOrUpdatePodGroup(newJob); err != nil {
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.PodGroupError),
                        fmt.Sprintf("Failed to create PodGroup, err: %v", err))
                return nil, err
        }

        return newJob, nil
}

func (cc *jobcontroller) initOnJobUpdate(job *batch.Job) error {
        klog.V(3).Infof("Starting to initiate Job <%s/%s> on update", job.Namespace, job.Name)

        if err := cc.pluginOnJobUpdate(job); err != nil {
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.PluginError),
                        fmt.Sprintf("Execute plugin when job add failed, err: %v", err))
                return err
        }

        if err := cc.createOrUpdatePodGroup(job); err != nil {
                cc.recorder.Event(job, v1.EventTypeWarning, string(batch.PodGroupError),
                        fmt.Sprintf("Failed to create PodGroup, err: %v", err))
                return err
        }

        return nil
}

func (cc *jobcontroller) GetQueueInfo(queue string) (*scheduling.Queue, error) {
        queueInfo, err := cc.queueLister.Get(queue)
        if err != nil {
                klog.Errorf("Failed to get queue from queueLister, error: %s", err.Error())
        }

        return queueInfo, err
}

func (cc *jobcontroller) syncJob(jobInfo *apis.JobInfo, updateStatus state.UpdateStatusFn) error {
        job := jobInfo.Job
        klog.V(3).Infof("Starting to sync up Job <%s/%s>, current version %d", job.Namespace, job.Name, job.Status.Version)
        defer klog.V(3).Infof("Finished Job <%s/%s> sync up, current version %d", job.Namespace, job.Name, job.Status.Version)

        if jobInfo.Job.DeletionTimestamp != nil {
                klog.Infof("Job <%s/%s> is terminating, skip management process.",
                        jobInfo.Job.Namespace, jobInfo.Job.Name)
                return nil
        }

        // deep copy job to prevent mutate it
        job = job.DeepCopy()

        // Find queue that job belongs to, and check if the queue has forwarding metadata
        queueInfo, err := cc.GetQueueInfo(job.Spec.Queue)
        if err != nil {
                return err
        }

        var jobForwarding bool
        if len(queueInfo.Spec.ExtendClusters) != 0 {
                jobForwarding = true
                if len(job.Annotations) == 0 {
                        job.Annotations = make(map[string]string)
                }
                job.Annotations[batch.JobForwardingKey] = "true"
                job, err = cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).Update(context.TODO(), job, metav1.UpdateOptions{})
                if err != nil {
                        klog.Errorf("failed to update job: %s/%s, error: %s", job.Namespace, job.Name, err.Error())
                        return err
                }
        }

        // Skip job initiation if job is already initiated
        if !isInitiated(job) {
                if job, err = cc.initiateJob(job); err != nil {
                        return err
                }
        } else {
                // TODO: optimize this call it only when scale up/down
                if err = cc.initOnJobUpdate(job); err != nil {
                        return err
                }
        }

        if len(queueInfo.Spec.ExtendClusters) != 0 {
                jobForwarding = true
                job.Annotations[batch.JobForwardingKey] = "true"
                _, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).Update(context.TODO(), job, metav1.UpdateOptions{})
                if err != nil {
                        klog.Errorf("failed to update job: %s/%s, error: %s", job.Namespace, job.Name, err.Error())
                        return err
                }
        }

        var syncTask bool
        if pg, _ := cc.pgLister.PodGroups(job.Namespace).Get(job.Name); pg != nil {
                if pg.Status.Phase != "" && pg.Status.Phase != scheduling.PodGroupPending {
                        syncTask = true
                }

                for _, condition := range pg.Status.Conditions {
                        if condition.Type == scheduling.PodGroupUnschedulableType {
                                cc.recorder.Eventf(job, v1.EventTypeWarning, string(batch.PodGroupPending),
                                        fmt.Sprintf("PodGroup %s:%s unschedule,reason: %s", job.Namespace, job.Name, condition.Message))
                        }
                }
        }

        if !syncTask {
                if updateStatus != nil {
                        if updateStatus(&job.Status) {
                                job.Status.State.LastTransitionTime = metav1.Now()
                        }
                }
                newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(context.TODO(), job, metav1.UpdateOptions{})
                if err != nil {
                        klog.Errorf("Failed to update status of Job %v/%v: %v",
                                job.Namespace, job.Name, err)
                        return err
                }
                if e := cc.cache.Update(newJob); e != nil {
                        klog.Errorf("SyncJob - Failed to update Job %v/%v in cache:  %v",
                                newJob.Namespace, newJob.Name, e)
                        return e
                }
                return nil
        }

        var running, pending, terminating, succeeded, failed, unknown int32
        taskStatusCount := make(map[string]batch.TaskState)

        var podToCreate []*v1.Pod
        var podToDelete []*v1.Pod
        var creationErrs []error
        var deletionErrs []error
        appendMutex := sync.Mutex{}

        appendError := func(container *[]error, err error) {
                appendMutex.Lock()
                defer appendMutex.Unlock()
                *container = append(*container, err)
        }

        for _, ts := range job.Spec.Tasks {
                ts.Template.Name = ts.Name
                tc := ts.Template.DeepCopy()
                name := ts.Template.Name

                pods, found := jobInfo.Pods[name]
                if !found {
                        pods = map[string]*v1.Pod{}
                }

                for i := 0; i < int(ts.Replicas); i++ {
                        podName := fmt.Sprintf(jobhelpers.PodNameFmt, job.Name, name, i)
                        if pod, found := pods[podName]; !found {
                                newPod := createJobPod(job, tc, ts.TopologyPolicy, i, jobForwarding)
                                if err := cc.pluginOnPodCreate(job, newPod); err != nil {
                                        return err
                                }
                                podToCreate = append(podToCreate, newPod)
                        } else {
                                delete(pods, podName)
                                if pod.DeletionTimestamp != nil {
                                        klog.Infof("Pod <%s/%s> is terminating", pod.Namespace, pod.Name)
                                        atomic.AddInt32(&terminating, 1)
                                        continue
                                }

                                classifyAndAddUpPodBaseOnPhase(pod, &pending, &running, &succeeded, &failed, &unknown)
                                calcPodStatus(pod, taskStatusCount)
                        }
                }

                for _, pod := range pods {
                        podToDelete = append(podToDelete, pod)
                }
        }

        waitCreationGroup := sync.WaitGroup{}
        waitCreationGroup.Add(len(podToCreate))
        for _, pod := range podToCreate {
                go func(pod *v1.Pod) {
                        defer waitCreationGroup.Done()
                        newPod, err := cc.kubeClient.CoreV1().Pods(pod.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{})
                        if err != nil && !apierrors.IsAlreadyExists(err) {
                                // Failed to create Pod, waitCreationGroup a moment and then create it again
                                // This is to ensure all podsMap under the same Job created
                                // So gang-scheduling could schedule the Job successfully
                                klog.Errorf("Failed to create pod %s for Job %s, err %#v",
                                        pod.Name, job.Name, err)
                                appendError(&creationErrs, fmt.Errorf("failed to create pod %s, err: %#v", pod.Name, err))
                        } else {
                                classifyAndAddUpPodBaseOnPhase(newPod, &pending, &running, &succeeded, &failed, &unknown)
                                calcPodStatus(pod, taskStatusCount)
                                klog.V(3).Infof("Created Task <%s> of Job <%s/%s>",
                                        pod.Name, job.Namespace, job.Name)
                        }
                }(pod)
        }
        waitCreationGroup.Wait()

        if len(creationErrs) != 0 {
                cc.recorder.Event(job, v1.EventTypeWarning, FailedCreatePodReason,
                        fmt.Sprintf("Error creating pods: %+v", creationErrs))
                return fmt.Errorf("failed to create %d pods of %d", len(creationErrs), len(podToCreate))
        }

        // Delete pods when scale down.
        waitDeletionGroup := sync.WaitGroup{}
        waitDeletionGroup.Add(len(podToDelete))
        for _, pod := range podToDelete {
                go func(pod *v1.Pod) {
                        defer waitDeletionGroup.Done()
                        err := cc.deleteJobPod(job.Name, pod)
                        if err != nil {
                                // Failed to delete Pod, waitCreationGroup a moment and then create it again
                                // This is to ensure all podsMap under the same Job created
                                // So gang-scheduling could schedule the Job successfully
                                klog.Errorf("Failed to delete pod %s for Job %s, err %#v",
                                        pod.Name, job.Name, err)
                                appendError(&deletionErrs, err)
                                cc.resyncTask(pod)
                        } else {
                                klog.V(3).Infof("Deleted Task <%s> of Job <%s/%s>",
                                        pod.Name, job.Namespace, job.Name)
                                atomic.AddInt32(&terminating, 1)
                        }
                }(pod)
        }
        waitDeletionGroup.Wait()

        if len(deletionErrs) != 0 {
                cc.recorder.Event(job, v1.EventTypeWarning, FailedDeletePodReason,
                        fmt.Sprintf("Error deleting pods: %+v", deletionErrs))
                return fmt.Errorf("failed to delete %d pods of %d", len(deletionErrs), len(podToDelete))
        }
        job.Status = batch.JobStatus{
                State: job.Status.State,

                Pending:             pending,
                Running:             running,
                Succeeded:           succeeded,
                Failed:              failed,
                Terminating:         terminating,
                Unknown:             unknown,
                Version:             job.Status.Version,
                MinAvailable:        job.Spec.MinAvailable,
                TaskStatusCount:     taskStatusCount,
                ControlledResources: job.Status.ControlledResources,
                RetryCount:          job.Status.RetryCount,
        }

        if updateStatus != nil {
                if updateStatus(&job.Status) {
                        job.Status.State.LastTransitionTime = metav1.Now()
                }
        }
        newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(context.TODO(), job, metav1.UpdateOptions{})
        if err != nil {
                klog.Errorf("Failed to update status of Job %v/%v: %v",
                        job.Namespace, job.Name, err)
                return err
        }
        if e := cc.cache.Update(newJob); e != nil {
                klog.Errorf("SyncJob - Failed to update Job %v/%v in cache:  %v",
                        newJob.Namespace, newJob.Name, e)
                return e
        }

        return nil
}

func (cc *jobcontroller) createJobIOIfNotExist(job *batch.Job) (*batch.Job, error) {
        // If PVC does not exist, create them for Job.
        var needUpdate bool
        if job.Status.ControlledResources == nil {
                job.Status.ControlledResources = make(map[string]string)
        }
        for index, volume := range job.Spec.Volumes {
                vcName := volume.VolumeClaimName
                if len(vcName) == 0 {
                        // NOTE(k82cn): Ensure never have duplicated generated names.
                        for {
                                vcName = jobhelpers.GenPVCName(job.Name)
                                exist, err := cc.checkPVCExist(job, vcName)
                                if err != nil {
                                        return job, err
                                }
                                if exist {
                                        continue
                                }
                                job.Spec.Volumes[index].VolumeClaimName = vcName
                                needUpdate = true
                                break
                        }
                        // TODO: check VolumeClaim must be set if VolumeClaimName is empty
                        if volume.VolumeClaim != nil {
                                if err := cc.createPVC(job, vcName, volume.VolumeClaim); err != nil {
                                        return job, err
                                }
                        }
                } else {
                        exist, err := cc.checkPVCExist(job, vcName)
                        if err != nil {
                                return job, err
                        }
                        if !exist {
                                return job, fmt.Errorf("pvc %s is not found, the job will be in the Pending state until the PVC is created", vcName)
                        }
                }
                job.Status.ControlledResources["volume-pvc-"+vcName] = vcName
        }
        if needUpdate {
                newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).Update(context.TODO(), job, metav1.UpdateOptions{})
                if err != nil {
                        klog.Errorf("Failed to update Job %v/%v for volume claim name: %v ",
                                job.Namespace, job.Name, err)
                        return job, err
                }

                newJob.Status = job.Status
                return newJob, err
        }
        return job, nil
}

func (cc *jobcontroller) checkPVCExist(job *batch.Job, pvc string) (bool, error) {
        if _, err := cc.pvcLister.PersistentVolumeClaims(job.Namespace).Get(pvc); err != nil {
                if apierrors.IsNotFound(err) {
                        return false, nil
                }
                klog.V(3).Infof("Failed to get PVC %s for job <%s/%s>: %v",
                        pvc, job.Namespace, job.Name, err)
                return false, err
        }
        return true, nil
}

func (cc *jobcontroller) createPVC(job *batch.Job, vcName string, volumeClaim *v1.PersistentVolumeClaimSpec) error {
        pvc := &v1.PersistentVolumeClaim{
                ObjectMeta: metav1.ObjectMeta{
                        Namespace: job.Namespace,
                        Name:      vcName,
                        OwnerReferences: []metav1.OwnerReference{
                                *metav1.NewControllerRef(job, helpers.JobKind),
                        },
                },
                Spec: *volumeClaim,
        }

        klog.V(3).Infof("Try to create PVC: %v", pvc)

        if _, e := cc.kubeClient.CoreV1().PersistentVolumeClaims(job.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{}); e != nil {
                klog.V(3).Infof("Failed to create PVC for Job <%s/%s>: %v",
                        job.Namespace, job.Name, e)
                return e
        }
        return nil
}

func (cc *jobcontroller) createOrUpdatePodGroup(job *batch.Job) error {
        // If PodGroup does not exist, create one for Job.
        pg, err := cc.pgLister.PodGroups(job.Namespace).Get(job.Name)
        if err != nil {
                if !apierrors.IsNotFound(err) {
                        klog.Errorf("Failed to get PodGroup for Job <%s/%s>: %v",
                                job.Namespace, job.Name, err)
                        return err
                }

                minTaskMember := map[string]int32{}
                for _, task := range job.Spec.Tasks {
                        if task.MinAvailable != nil {
                                minTaskMember[task.Name] = *task.MinAvailable
                        } else {
                                minTaskMember[task.Name] = task.Replicas
                        }
                }

                pg := &scheduling.PodGroup{
                        ObjectMeta: metav1.ObjectMeta{
                                Namespace:   job.Namespace,
                                Name:        job.Name,
                                Annotations: job.Annotations,
                                Labels:      job.Labels,
                                OwnerReferences: []metav1.OwnerReference{
                                        *metav1.NewControllerRef(job, helpers.JobKind),
                                },
                        },
                        Spec: scheduling.PodGroupSpec{
                                MinMember:         job.Spec.MinAvailable,
                                MinTaskMember:     minTaskMember,
                                Queue:             job.Spec.Queue,
                                MinResources:      cc.calcPGMinResources(job),
                                PriorityClassName: job.Spec.PriorityClassName,
                        },
                }

                if _, err = cc.vcClient.SchedulingV1beta1().PodGroups(job.Namespace).Create(context.TODO(), pg, metav1.CreateOptions{}); err != nil {
                        if !apierrors.IsAlreadyExists(err) {
                                klog.Errorf("Failed to create PodGroup for Job <%s/%s>: %v",
                                        job.Namespace, job.Name, err)
                                return err
                        }
                }
                return nil
        }

        pgShouldUpdate := false
        if pg.Spec.PriorityClassName != job.Spec.PriorityClassName {
                pg.Spec.PriorityClassName = job.Spec.PriorityClassName
                pgShouldUpdate = true
        }

        minResources := cc.calcPGMinResources(job)
        if pg.Spec.MinMember != job.Spec.MinAvailable || !reflect.DeepEqual(pg.Spec.MinResources, minResources) {
                pg.Spec.MinMember = job.Spec.MinAvailable
                pg.Spec.MinResources = minResources
                pgShouldUpdate = true
        }

        if pg.Spec.MinTaskMember == nil {
                pgShouldUpdate = true
                pg.Spec.MinTaskMember = make(map[string]int32)
        }

        for _, task := range job.Spec.Tasks {
                if task.MinAvailable == nil {
                        continue
                }

                if taskMember, ok := pg.Spec.MinTaskMember[task.Name]; !ok {
                        pgShouldUpdate = true
                        pg.Spec.MinTaskMember[task.Name] = *task.MinAvailable
                } else {
                        if taskMember == *task.MinAvailable {
                                continue
                        }

                        pgShouldUpdate = true
                        pg.Spec.MinTaskMember[task.Name] = *task.MinAvailable
                }
        }

        if !pgShouldUpdate {
                return nil
        }

        _, err = cc.vcClient.SchedulingV1beta1().PodGroups(job.Namespace).Update(context.TODO(), pg, metav1.UpdateOptions{})
        if err != nil {
                klog.V(3).Infof("Failed to update PodGroup for Job <%s/%s>: %v",
                        job.Namespace, job.Name, err)
        }
        return err
}

func (cc *jobcontroller) deleteJobPod(jobName string, pod *v1.Pod) error {
        err := cc.kubeClient.CoreV1().Pods(pod.Namespace).Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
        if err != nil && !apierrors.IsNotFound(err) {
                klog.Errorf("Failed to delete pod %s/%s for Job %s, err %#v",
                        pod.Namespace, pod.Name, jobName, err)

                return fmt.Errorf("failed to delete pod %s, err %#v", pod.Name, err)
        }

        return nil
}

func (cc *jobcontroller) calcPGMinResources(job *batch.Job) *v1.ResourceList {
        // sort task by priorityClasses
        var tasksPriority TasksPriority
        for _, task := range job.Spec.Tasks {
                tp := TaskPriority{0, task}
                pc := task.Template.Spec.PriorityClassName

                priorityClass, err := cc.pcLister.Get(pc)
                if err != nil || priorityClass == nil {
                        klog.Warningf("Ignore task %s priority class %s: %v", task.Name, pc, err)
                } else {
                        tp.priority = priorityClass.Value
                }

                tasksPriority = append(tasksPriority, tp)
        }

        sort.Sort(tasksPriority)

        minAvailableTasksRes := v1.ResourceList{}
        podCnt := int32(0)
        for _, task := range tasksPriority {
                for i := int32(0); i < task.Replicas; i++ {
                        if podCnt >= job.Spec.MinAvailable {
                                break
                        }
                        podCnt++
                        for _, c := range task.Template.Spec.Containers {
                                addResourceList(minAvailableTasksRes, c.Resources.Requests, c.Resources.Limits)
                        }
                }
        }

        return &minAvailableTasksRes
}

func (cc *jobcontroller) initJobStatus(job *batch.Job) (*batch.Job, error) {
        if job.Status.State.Phase != "" {
                return job, nil
        }

        job.Status.State.LastTransitionTime = metav1.Now()
        job.Status.State.Phase = batch.Pending
        job.Status.State.LastTransitionTime = metav1.Now()
        job.Status.MinAvailable = job.Spec.MinAvailable
        newJob, err := cc.vcClient.BatchV1alpha1().Jobs(job.Namespace).UpdateStatus(context.TODO(), job, metav1.UpdateOptions{})
        if err != nil {
                klog.Errorf("Failed to update status of Job %v/%v: %v",
                        job.Namespace, job.Name, err)
                return nil, err
        }
        if err := cc.cache.Update(newJob); err != nil {
                klog.Errorf("CreateJob - Failed to update Job %v/%v in cache:  %v",
                        newJob.Namespace, newJob.Name, err)
                return nil, err
        }

        return newJob, nil
}

func classifyAndAddUpPodBaseOnPhase(pod *v1.Pod, pending, running, succeeded, failed, unknown *int32) {
        switch pod.Status.Phase {
        case v1.PodPending:
                atomic.AddInt32(pending, 1)
        case v1.PodRunning:
                atomic.AddInt32(running, 1)
        case v1.PodSucceeded:
                atomic.AddInt32(succeeded, 1)
        case v1.PodFailed:
                atomic.AddInt32(failed, 1)
        default:
                atomic.AddInt32(unknown, 1)
        }
}

func calcPodStatus(pod *v1.Pod, taskStatusCount map[string]batch.TaskState) {
        taskName, found := pod.Annotations[batch.TaskSpecKey]
        if !found {
                return
        }

        calMutex.Lock()
        defer calMutex.Unlock()
        if _, ok := taskStatusCount[taskName]; !ok {
                taskStatusCount[taskName] = batch.TaskState{
                        Phase: make(map[v1.PodPhase]int32),
                }
        }

        switch pod.Status.Phase {
        case v1.PodPending:
                taskStatusCount[taskName].Phase[v1.PodPending]++
        case v1.PodRunning:
                taskStatusCount[taskName].Phase[v1.PodRunning]++
        case v1.PodSucceeded:
                taskStatusCount[taskName].Phase[v1.PodSucceeded]++
        case v1.PodFailed:
                taskStatusCount[taskName].Phase[v1.PodFailed]++
        default:
                taskStatusCount[taskName].Phase[v1.PodUnknown]++
        }
}

func isInitiated(job *batch.Job) bool {
        if job.Status.State.Phase == "" || job.Status.State.Phase == batch.Pending {
                return false
        }

        return true
}

/*
Copyright 2017 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "reflect"
        "strconv"

        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        bus "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
        jobcache "volcano.sh/volcano/pkg/controllers/cache"
        jobhelpers "volcano.sh/volcano/pkg/controllers/job/helpers"
)

func (cc *jobcontroller) addCommand(obj interface{}) {
        cmd, ok := obj.(*bus.Command)
        if !ok {
                klog.Errorf("obj is not Command")
                return
        }

        cc.commandQueue.Add(cmd)
}

func (cc *jobcontroller) addJob(obj interface{}) {
        job, ok := obj.(*batch.Job)
        if !ok {
                klog.Errorf("obj is not Job")
                return
        }

        req := apis.Request{
                Namespace: job.Namespace,
                JobName:   job.Name,

                Event: bus.OutOfSyncEvent,
        }

        // TODO(k82cn): if failed to add job, the cache should be refresh
        if err := cc.cache.Add(job); err != nil {
                klog.Errorf("Failed to add job <%s/%s>: %v in cache",
                        job.Namespace, job.Name, err)
        }
        key := jobhelpers.GetJobKeyByReq(&req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)
}

func (cc *jobcontroller) updateJob(oldObj, newObj interface{}) {
        newJob, ok := newObj.(*batch.Job)
        if !ok {
                klog.Errorf("newObj is not Job")
                return
        }

        oldJob, ok := oldObj.(*batch.Job)
        if !ok {
                klog.Errorf("oldJob is not Job")
                return
        }

        // No need to update if ResourceVersion is not changed
        if newJob.ResourceVersion == oldJob.ResourceVersion {
                klog.V(6).Infof("No need to update because job is not modified.")
                return
        }

        if err := cc.cache.Update(newJob); err != nil {
                klog.Errorf("UpdateJob - Failed to update job <%s/%s>: %v in cache",
                        newJob.Namespace, newJob.Name, err)
        }

        // NOTE: Since we only reconcile job based on Spec, we will ignore other attributes
        // For Job status, it's used internally and always been updated via our controller.
        if reflect.DeepEqual(newJob.Spec, oldJob.Spec) && newJob.Status.State.Phase == oldJob.Status.State.Phase {
                klog.V(6).Infof("Job update event is ignored since no update in 'Spec'.")
                return
        }

        req := apis.Request{
                Namespace: newJob.Namespace,
                JobName:   newJob.Name,
                Event:     bus.OutOfSyncEvent,
        }
        key := jobhelpers.GetJobKeyByReq(&req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)
}

func (cc *jobcontroller) deleteJob(obj interface{}) {
        job, ok := obj.(*batch.Job)
        if !ok {
                // If we reached here it means the Job was deleted but its final state is unrecorded.
                tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
                if !ok {
                        klog.Errorf("Couldn't get object from tombstone %#v", obj)
                        return
                }
                job, ok = tombstone.Obj.(*batch.Job)
                if !ok {
                        klog.Errorf("Tombstone contained object that is not a volcano Job: %#v", obj)
                        return
                }
        }

        if err := cc.cache.Delete(job); err != nil {
                klog.Errorf("Failed to delete job <%s/%s>: %v in cache",
                        job.Namespace, job.Name, err)
        }
}

func (cc *jobcontroller) addPod(obj interface{}) {
        pod, ok := obj.(*v1.Pod)
        if !ok {
                klog.Errorf("Failed to convert %v to v1.Pod", obj)
                return
        }
        // Filter out pods that are not created from volcano job
        if !isControlledBy(pod, helpers.JobKind) {
                return
        }

        jobName, found := pod.Annotations[batch.JobNameKey]
        if !found {
                klog.Infof("Failed to find jobName of Pod <%s/%s>, skipping",
                        pod.Namespace, pod.Name)
                return
        }

        version, found := pod.Annotations[batch.JobVersion]
        if !found {
                klog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping",
                        pod.Namespace, pod.Name)
                return
        }

        dVersion, err := strconv.Atoi(version)
        if err != nil {
                klog.Infof("Failed to convert jobVersion of Pod <%s/%s> into number, skipping",
                        pod.Namespace, pod.Name)
                return
        }

        if pod.DeletionTimestamp != nil {
                cc.deletePod(pod)
                return
        }

        req := apis.Request{
                Namespace: pod.Namespace,
                JobName:   jobName,

                Event:      bus.OutOfSyncEvent,
                JobVersion: int32(dVersion),
        }

        if err := cc.cache.AddPod(pod); err != nil {
                klog.Errorf("Failed to add Pod <%s/%s>: %v to cache",
                        pod.Namespace, pod.Name, err)
        }
        key := jobhelpers.GetJobKeyByReq(&req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)
}

func (cc *jobcontroller) updatePod(oldObj, newObj interface{}) {
        oldPod, ok := oldObj.(*v1.Pod)
        if !ok {
                klog.Errorf("Failed to convert %v to v1.Pod", oldObj)
                return
        }

        newPod, ok := newObj.(*v1.Pod)
        if !ok {
                klog.Errorf("Failed to convert %v to v1.Pod", newObj)
                return
        }

        // Filter out pods that are not created from volcano job
        if !isControlledBy(newPod, helpers.JobKind) {
                return
        }

        if newPod.ResourceVersion == oldPod.ResourceVersion {
                return
        }

        if newPod.DeletionTimestamp != nil {
                cc.deletePod(newObj)
                return
        }

        taskName, found := newPod.Annotations[batch.TaskSpecKey]
        if !found {
                klog.Infof("Failed to find taskName of Pod <%s/%s>, skipping",
                        newPod.Namespace, newPod.Name)
                return
        }

        jobName, found := newPod.Annotations[batch.JobNameKey]
        if !found {
                klog.Infof("Failed to find jobName of Pod <%s/%s>, skipping",
                        newPod.Namespace, newPod.Name)
                return
        }

        version, found := newPod.Annotations[batch.JobVersion]
        if !found {
                klog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping",
                        newPod.Namespace, newPod.Name)
                return
        }

        dVersion, err := strconv.Atoi(version)
        if err != nil {
                klog.Infof("Failed to convert jobVersion of Pod into number <%s/%s>, skipping",
                        newPod.Namespace, newPod.Name)
                return
        }

        if err := cc.cache.UpdatePod(newPod); err != nil {
                klog.Errorf("Failed to update Pod <%s/%s>: %v in cache",
                        newPod.Namespace, newPod.Name, err)
        }

        event := bus.OutOfSyncEvent
        var exitCode int32

        switch newPod.Status.Phase {
        case v1.PodFailed:
                if oldPod.Status.Phase != v1.PodFailed {
                        event = bus.PodFailedEvent
                        // TODO: currently only one container pod is supported by volcano
                        // Once multi containers pod is supported, update accordingly.
                        if len(newPod.Status.ContainerStatuses) > 0 && newPod.Status.ContainerStatuses[0].State.Terminated != nil {
                                exitCode = newPod.Status.ContainerStatuses[0].State.Terminated.ExitCode
                        }
                }
        case v1.PodSucceeded:
                if oldPod.Status.Phase != v1.PodSucceeded &&
                        cc.cache.TaskCompleted(jobcache.JobKeyByName(newPod.Namespace, jobName), taskName) {
                        event = bus.TaskCompletedEvent
                }
        case v1.PodPending, v1.PodRunning:
                if cc.cache.TaskFailed(jobcache.JobKeyByName(newPod.Namespace, jobName), taskName) {
                        event = bus.TaskFailedEvent
                }
        }

        req := apis.Request{
                Namespace: newPod.Namespace,
                JobName:   jobName,
                TaskName:  taskName,

                Event:      event,
                ExitCode:   exitCode,
                JobVersion: int32(dVersion),
        }

        key := jobhelpers.GetJobKeyByReq(&req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)
}

func (cc *jobcontroller) deletePod(obj interface{}) {
        pod, ok := obj.(*v1.Pod)
        if !ok {
                // If we reached here it means the pod was deleted but its final state is unrecorded.
                tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
                if !ok {
                        klog.Errorf("Couldn't get object from tombstone %#v", obj)
                        return
                }
                pod, ok = tombstone.Obj.(*v1.Pod)
                if !ok {
                        klog.Errorf("Tombstone contained object that is not a Pod: %#v", obj)
                        return
                }
        }

        // Filter out pods that are not created from volcano job
        if !isControlledBy(pod, helpers.JobKind) {
                return
        }

        taskName, found := pod.Annotations[batch.TaskSpecKey]
        if !found {
                klog.Infof("Failed to find taskName of Pod <%s/%s>, skipping",
                        pod.Namespace, pod.Name)
                return
        }

        jobName, found := pod.Annotations[batch.JobNameKey]
        if !found {
                klog.Infof("Failed to find jobName of Pod <%s/%s>, skipping",
                        pod.Namespace, pod.Name)
                return
        }

        version, found := pod.Annotations[batch.JobVersion]
        if !found {
                klog.Infof("Failed to find jobVersion of Pod <%s/%s>, skipping",
                        pod.Namespace, pod.Name)
                return
        }

        dVersion, err := strconv.Atoi(version)
        if err != nil {
                klog.Infof("Failed to convert jobVersion of Pod <%s/%s> into number, skipping",
                        pod.Namespace, pod.Name)
                return
        }

        req := apis.Request{
                Namespace: pod.Namespace,
                JobName:   jobName,
                TaskName:  taskName,

                Event:      bus.PodEvictedEvent,
                JobVersion: int32(dVersion),
        }

        if err := cc.cache.DeletePod(pod); err != nil {
                klog.Errorf("Failed to delete Pod <%s/%s>: %v in cache",
                        pod.Namespace, pod.Name, err)
        }

        key := jobhelpers.GetJobKeyByReq(&req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)
}

func (cc *jobcontroller) recordJobEvent(namespace, name string, event batch.JobEvent, message string) {
        job, err := cc.cache.Get(jobcache.JobKeyByName(namespace, name))
        if err != nil {
                klog.Warningf("Failed to find job in cache when reporting job event <%s/%s>: %v",
                        namespace, name, err)
                return
        }
        cc.recorder.Event(job.Job, v1.EventTypeNormal, string(event), message)
}

func (cc *jobcontroller) handleCommands() {
        for cc.processNextCommand() {
        }
}

func (cc *jobcontroller) processNextCommand() bool {
        obj, shutdown := cc.commandQueue.Get()
        if shutdown {
                return false
        }
        cmd := obj.(*bus.Command)
        defer cc.commandQueue.Done(cmd)

        if err := cc.vcClient.BusV1alpha1().Commands(cmd.Namespace).Delete(context.TODO(), cmd.Name, metav1.DeleteOptions{}); err != nil {
                if !apierrors.IsNotFound(err) {
                        klog.Errorf("Failed to delete Command <%s/%s>.", cmd.Namespace, cmd.Name)
                        cc.commandQueue.AddRateLimited(cmd)
                }
                return true
        }
        cc.recordJobEvent(cmd.Namespace, cmd.TargetObject.Name,
                batch.CommandIssued,
                fmt.Sprintf(
                        "Start to execute command %s, and clean it up to make sure executed not more than once.", cmd.Action))
        req := apis.Request{
                Namespace: cmd.Namespace,
                JobName:   cmd.TargetObject.Name,
                Event:     bus.CommandIssuedEvent,
                Action:    bus.Action(cmd.Action),
        }

        key := jobhelpers.GetJobKeyByReq(&req)
        queue := cc.getWorkerQueue(key)
        queue.Add(req)

        return true
}

func (cc *jobcontroller) updatePodGroup(oldObj, newObj interface{}) {
        oldPG, ok := oldObj.(*scheduling.PodGroup)
        if !ok {
                klog.Errorf("Failed to convert %v to PodGroup", newObj)
                return
        }

        newPG, ok := newObj.(*scheduling.PodGroup)
        if !ok {
                klog.Errorf("Failed to convert %v to PodGroup", newObj)
                return
        }

        _, err := cc.cache.Get(jobcache.JobKeyByName(newPG.Namespace, newPG.Name))
        if err != nil && newPG.Annotations != nil {
                klog.Warningf(
                        "Failed to find job in cache by PodGroup, this may not be a PodGroup for volcano job.")
        }

        if newPG.Status.Phase != oldPG.Status.Phase {
                req := apis.Request{
                        Namespace: newPG.Namespace,
                        JobName:   newPG.Name,
                }
                switch newPG.Status.Phase {
                case scheduling.PodGroupUnknown:
                        req.Event = bus.JobUnknownEvent
                }
                key := jobhelpers.GetJobKeyByReq(&req)
                queue := cc.getWorkerQueue(key)
                queue.Add(req)
        }
}

// TODO(k82cn): add handler for PodGroup unschedulable event.

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/volcano/pkg/controllers/job/plugins"
        pluginsinterface "volcano.sh/volcano/pkg/controllers/job/plugins/interface"
)

func (cc *jobcontroller) pluginOnPodCreate(job *batch.Job, pod *v1.Pod) error {
        client := pluginsinterface.PluginClientset{KubeClients: cc.kubeClient}
        for name, args := range job.Spec.Plugins {
                pb, found := plugins.GetPluginBuilder(name)
                if !found {
                        err := fmt.Errorf("failed to get plugin %s", name)
                        klog.Error(err)
                        return err
                }
                klog.Infof("Starting to execute plugin at <pluginOnPodCreate>: %s on job: <%s/%s>", name, job.Namespace, job.Name)
                if err := pb(client, args).OnPodCreate(pod, job); err != nil {
                        klog.Errorf("Failed to process on pod create plugin %s, err %v.", name, err)
                        return err
                }
        }
        return nil
}

func (cc *jobcontroller) pluginOnJobAdd(job *batch.Job) error {
        client := pluginsinterface.PluginClientset{KubeClients: cc.kubeClient}
        if job.Status.ControlledResources == nil {
                job.Status.ControlledResources = make(map[string]string)
        }
        for name, args := range job.Spec.Plugins {
                pb, found := plugins.GetPluginBuilder(name)
                if !found {
                        err := fmt.Errorf("failed to get plugin %s", name)
                        klog.Error(err)
                        return err
                }
                klog.Infof("Starting to execute plugin at <pluginOnJobAdd>: %s on job: <%s/%s>", name, job.Namespace, job.Name)
                if err := pb(client, args).OnJobAdd(job); err != nil {
                        klog.Errorf("Failed to process on job add plugin %s, err %v.", name, err)
                        return err
                }
        }

        return nil
}

func (cc *jobcontroller) pluginOnJobDelete(job *batch.Job) error {
        if job.Status.ControlledResources == nil {
                job.Status.ControlledResources = make(map[string]string)
        }
        client := pluginsinterface.PluginClientset{KubeClients: cc.kubeClient}
        for name, args := range job.Spec.Plugins {
                pb, found := plugins.GetPluginBuilder(name)
                if !found {
                        err := fmt.Errorf("failed to get plugin %s", name)
                        klog.Error(err)
                        return err
                }
                klog.Infof("Starting to execute plugin at <pluginOnJobDelete>: %s on job: <%s/%s>", name, job.Namespace, job.Name)
                if err := pb(client, args).OnJobDelete(job); err != nil {
                        klog.Errorf("failed to process on job delete plugin %s, err %v.", name, err)
                        return err
                }
        }

        return nil
}

func (cc *jobcontroller) pluginOnJobUpdate(job *batch.Job) error {
        client := pluginsinterface.PluginClientset{KubeClients: cc.kubeClient}
        if job.Status.ControlledResources == nil {
                job.Status.ControlledResources = make(map[string]string)
        }
        for name, args := range job.Spec.Plugins {
                pb, found := plugins.GetPluginBuilder(name)
                if !found {
                        err := fmt.Errorf("failed to get plugin %s", name)
                        klog.Error(err)
                        return err
                }
                klog.Infof("Starting to execute plugin at <pluginOnJobUpdate>: %s on job: <%s/%s>", name, job.Namespace, job.Name)
                if err := pb(client, args).OnJobUpdate(job); err != nil {
                        klog.Errorf("Failed to process on job update plugin %s, err %v.", name, err)
                        return err
                }
        }

        return nil
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "context"
        "fmt"
        "time"

        "golang.org/x/time/rate"
        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"
)

func newRateLimitingQueue() workqueue.RateLimitingInterface {
        return workqueue.NewRateLimitingQueue(workqueue.NewMaxOfRateLimiter(
                workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 180*time.Second),
                // 10 qps, 100 bucket size.  This is only for retry speed and its only the overall factor (not per item)
                &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
        ))
}

func (cc *jobcontroller) processResyncTask() {
        obj, shutdown := cc.errTasks.Get()
        if shutdown {
                return
        }

        // one task only resync 10 times
        if cc.errTasks.NumRequeues(obj) > 10 {
                cc.errTasks.Forget(obj)
                return
        }

        defer cc.errTasks.Done(obj)

        task, ok := obj.(*v1.Pod)
        if !ok {
                klog.Errorf("failed to convert %v to *v1.Pod", obj)
                return
        }

        if err := cc.syncTask(task); err != nil {
                klog.Errorf("Failed to sync pod <%v/%v>, retry it, err %v", task.Namespace, task.Name, err)
                cc.resyncTask(task)
        }
}

func (cc *jobcontroller) syncTask(oldTask *v1.Pod) error {
        newPod, err := cc.kubeClient.CoreV1().Pods(oldTask.Namespace).Get(context.TODO(), oldTask.Name, metav1.GetOptions{})
        if err != nil {
                if errors.IsNotFound(err) {
                        if err := cc.cache.DeletePod(oldTask); err != nil {
                                klog.Errorf("failed to delete cache pod <%v/%v>, err %v.", oldTask.Namespace, oldTask.Name, err)
                                return err
                        }
                        klog.V(3).Infof("Pod <%v/%v> was deleted, removed from cache.", oldTask.Namespace, oldTask.Name)

                        return nil
                }
                return fmt.Errorf("failed to get Pod <%v/%v>: err %v", oldTask.Namespace, oldTask.Name, err)
        }

        return cc.cache.UpdatePod(newPod)
}

func (cc *jobcontroller) resyncTask(task *v1.Pod) {
        cc.errTasks.AddRateLimited(task)
}

/*
Copyright 2017 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package job

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/runtime/schema"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        schedulingv2 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
        jobhelpers "volcano.sh/volcano/pkg/controllers/job/helpers"
)

// MakePodName append podname,jobname,taskName and index and returns the string.
func MakePodName(jobName string, taskName string, index int) string {
        return fmt.Sprintf(jobhelpers.PodNameFmt, jobName, taskName, index)
}

func createJobPod(job *batch.Job, template *v1.PodTemplateSpec, topologyPolicy batch.NumaPolicy, ix int, jobForwarding bool) *v1.Pod {
        templateCopy := template.DeepCopy()

        pod := &v1.Pod{
                ObjectMeta: metav1.ObjectMeta{
                        Name:      jobhelpers.MakePodName(job.Name, template.Name, ix),
                        Namespace: job.Namespace,
                        OwnerReferences: []metav1.OwnerReference{
                                *metav1.NewControllerRef(job, helpers.JobKind),
                        },
                        Labels:      templateCopy.Labels,
                        Annotations: templateCopy.Annotations,
                },
                Spec: templateCopy.Spec,
        }

        // If no scheduler name in Pod, use scheduler name from Job.
        if len(pod.Spec.SchedulerName) == 0 {
                pod.Spec.SchedulerName = job.Spec.SchedulerName
        }

        volumeMap := make(map[string]string)
        for _, volume := range job.Spec.Volumes {
                vcName := volume.VolumeClaimName
                name := fmt.Sprintf("%s-%s", job.Name, jobhelpers.GenRandomStr(12))
                if _, ok := volumeMap[vcName]; !ok {
                        volume := v1.Volume{
                                Name: name,
                                VolumeSource: v1.VolumeSource{
                                        PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
                                                ClaimName: vcName,
                                        },
                                },
                        }
                        pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
                        volumeMap[vcName] = name
                } else {
                        // duplicate volumes, should be prevented
                        continue
                }

                for i, c := range pod.Spec.Containers {
                        vm := v1.VolumeMount{
                                MountPath: volume.MountPath,
                                Name:      name,
                        }
                        pod.Spec.Containers[i].VolumeMounts = append(c.VolumeMounts, vm)
                }
        }

        tsKey := templateCopy.Name
        if len(tsKey) == 0 {
                tsKey = batch.DefaultTaskSpec
        }

        if len(pod.Annotations) == 0 {
                pod.Annotations = make(map[string]string)
        }

        pod.Annotations[batch.TaskSpecKey] = tsKey
        pod.Annotations[schedulingv2.KubeGroupNameAnnotationKey] = job.Name
        pod.Annotations[batch.JobNameKey] = job.Name
        pod.Annotations[batch.QueueNameKey] = job.Spec.Queue
        pod.Annotations[batch.JobVersion] = fmt.Sprintf("%d", job.Status.Version)
        pod.Annotations[batch.PodTemplateKey] = fmt.Sprintf("%s-%s", job.Name, template.Name)

        if topologyPolicy != "" {
                pod.Annotations[schedulingv2.NumaPolicyKey] = string(topologyPolicy)
        }

        if len(job.Annotations) > 0 {
                if value, found := job.Annotations[schedulingv2.PodPreemptable]; found {
                        pod.Annotations[schedulingv2.PodPreemptable] = value
                }
                if value, found := job.Annotations[schedulingv2.RevocableZone]; found {
                        pod.Annotations[schedulingv2.RevocableZone] = value
                }

                if value, found := job.Annotations[schedulingv2.JDBMinAvailable]; found {
                        pod.Annotations[schedulingv2.JDBMinAvailable] = value
                } else if value, found := job.Annotations[schedulingv2.JDBMaxUnavailable]; found {
                        pod.Annotations[schedulingv2.JDBMaxUnavailable] = value
                }
        }

        if len(pod.Labels) == 0 {
                pod.Labels = make(map[string]string)
        }

        // Set pod labels for Service.
        pod.Labels[batch.JobNameKey] = job.Name
        pod.Labels[batch.TaskSpecKey] = tsKey
        pod.Labels[batch.JobNamespaceKey] = job.Namespace
        pod.Labels[batch.QueueNameKey] = job.Spec.Queue
        if len(job.Labels) > 0 {
                if value, found := job.Labels[schedulingv2.PodPreemptable]; found {
                        pod.Labels[schedulingv2.PodPreemptable] = value
                }
        }

        if jobForwarding {
                pod.Annotations[batch.JobForwardingKey] = "true"
                pod.Labels[batch.JobForwardingKey] = "true"
        }

        return pod
}

func applyPolicies(job *batch.Job, req *apis.Request) v1alpha1.Action {
        if len(req.Action) != 0 {
                return req.Action
        }

        if req.Event == v1alpha1.OutOfSyncEvent {
                return v1alpha1.SyncJobAction
        }

        // For all the requests triggered from discarded job resources will perform sync action instead
        if req.JobVersion < job.Status.Version {
                klog.Infof("Request %s is outdated, will perform sync instead.", req)
                return v1alpha1.SyncJobAction
        }

        // Overwrite Job level policies
        if len(req.TaskName) != 0 {
                // Parse task level policies
                for _, task := range job.Spec.Tasks {
                        if task.Name == req.TaskName {
                                for _, policy := range task.Policies {
                                        policyEvents := getEventlist(policy)

                                        if len(policyEvents) > 0 && len(req.Event) > 0 {
                                                if checkEventExist(policyEvents, req.Event) || checkEventExist(policyEvents, v1alpha1.AnyEvent) {
                                                        return policy.Action
                                                }
                                        }

                                        // 0 is not an error code, is prevented in validation admission controller
                                        if policy.ExitCode != nil && *policy.ExitCode == req.ExitCode {
                                                return policy.Action
                                        }
                                }
                                break
                        }
                }
        }

        // Parse Job level policies
        for _, policy := range job.Spec.Policies {
                policyEvents := getEventlist(policy)

                if len(policyEvents) > 0 && len(req.Event) > 0 {
                        if checkEventExist(policyEvents, req.Event) || checkEventExist(policyEvents, v1alpha1.AnyEvent) {
                                return policy.Action
                        }
                }

                // 0 is not an error code, is prevented in validation admission controller
                if policy.ExitCode != nil && *policy.ExitCode == req.ExitCode {
                        return policy.Action
                }
        }

        return v1alpha1.SyncJobAction
}

func getEventlist(policy batch.LifecyclePolicy) []v1alpha1.Event {
        policyEventsList := policy.Events
        if len(policy.Event) > 0 {
                policyEventsList = append(policyEventsList, policy.Event)
        }
        return policyEventsList
}

func checkEventExist(policyEvents []v1alpha1.Event, reqEvent v1alpha1.Event) bool {
        for _, event := range policyEvents {
                if event == reqEvent {
                        return true
                }
        }
        return false
}

func addResourceList(list, req, limit v1.ResourceList) {
        for name, quantity := range req {
                if value, ok := list[name]; !ok {
                        list[name] = quantity.DeepCopy()
                } else {
                        value.Add(quantity)
                        list[name] = value
                }
        }

        if req != nil {
                return
        }

        // If Requests is omitted for a container,
        // it defaults to Limits if that is explicitly specified.
        for name, quantity := range limit {
                if value, ok := list[name]; !ok {
                        list[name] = quantity.DeepCopy()
                } else {
                        value.Add(quantity)
                        list[name] = value
                }
        }
}

// TaskPriority structure.
type TaskPriority struct {
        priority int32

        batch.TaskSpec
}

// TasksPriority is a slice of TaskPriority.
type TasksPriority []TaskPriority

func (p TasksPriority) Len() int { return len(p) }

func (p TasksPriority) Less(i, j int) bool {
        return p[i].priority > p[j].priority
}

func (p TasksPriority) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

func isControlledBy(obj metav1.Object, gvk schema.GroupVersionKind) bool {
        controllerRef := metav1.GetControllerOf(obj)
        if controllerRef == nil {
                return false
        }
        if controllerRef.APIVersion == gvk.GroupVersion().String() && controllerRef.Kind == gvk.Kind {
                return true
        }
        return false
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package ssh

import (
        "crypto/rand"
        "crypto/rsa"
        "crypto/x509"
        "encoding/pem"
        "flag"
        "fmt"

        "golang.org/x/crypto/ssh"
        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/apis/helpers"
        jobhelpers "volcano.sh/volcano/pkg/controllers/job/helpers"
        pluginsinterface "volcano.sh/volcano/pkg/controllers/job/plugins/interface"
)

type sshPlugin struct {
        // Arguments given for the plugin
        pluginArguments []string

        client pluginsinterface.PluginClientset

        // flag parse args
        sshKeyFilePath string

        // private key string
        sshPrivateKey string

        // public key string
        sshPublicKey string
}

// New creates ssh plugin
func New(client pluginsinterface.PluginClientset, arguments []string) pluginsinterface.PluginInterface {
        p := sshPlugin{
                pluginArguments: arguments,
                client:          client,
                sshKeyFilePath:  SSHAbsolutePath,
        }

        p.addFlags()

        return &p
}

func (sp *sshPlugin) Name() string {
        return "ssh"
}

func (sp *sshPlugin) OnPodCreate(pod *v1.Pod, job *batch.Job) error {
        sp.mountRsaKey(pod, job)

        return nil
}

func (sp *sshPlugin) OnJobAdd(job *batch.Job) error {
        if job.Status.ControlledResources["plugin-"+sp.Name()] == sp.Name() {
                return nil
        }

        var data map[string][]byte
        var err error
        if len(sp.sshPrivateKey) > 0 {
                data, err = withUserProvidedRsaKey(job, sp.sshPrivateKey, sp.sshPublicKey)
        } else {
                data, err = generateRsaKey(job)
        }
        if err != nil {
                return err
        }

        if err := helpers.CreateOrUpdateSecret(job, sp.client.KubeClients, data, sp.secretName(job)); err != nil {
                return fmt.Errorf("create secret for job <%s/%s> with ssh plugin failed for %v",
                        job.Namespace, job.Name, err)
        }

        job.Status.ControlledResources["plugin-"+sp.Name()] = sp.Name()

        return nil
}

func (sp *sshPlugin) OnJobDelete(job *batch.Job) error {
        if job.Status.ControlledResources["plugin-"+sp.Name()] != sp.Name() {
                return nil
        }
        if err := helpers.DeleteSecret(job, sp.client.KubeClients, sp.secretName(job)); err != nil {
                return err
        }
        delete(job.Status.ControlledResources, "plugin-"+sp.Name())

        return nil
}

// TODO: currently a container using a Secret as a subPath volume mount will not receive Secret updates.
// we may not update the job secret due to the above reason now.
// related issue: https://github.com/volcano-sh/volcano/issues/1420
func (sp *sshPlugin) OnJobUpdate(job *batch.Job) error {
        //data, err := generateRsaKey(job)
        //if err != nil {
        //        return err
        //}
        //
        //if err := helpers.CreateOrUpdateSecret(job, sp.client.KubeClients, data, sp.secretName(job)); err != nil {
        //        return fmt.Errorf("update secret for job <%s/%s> with ssh plugin failed for %v",
        //                job.Namespace, job.Name, err)
        //}

        return nil
}

func (sp *sshPlugin) mountRsaKey(pod *v1.Pod, job *batch.Job) {
        secretName := sp.secretName(job)

        sshVolume := v1.Volume{
                Name: secretName,
        }

        var mode int32 = 0600
        sshVolume.Secret = &v1.SecretVolumeSource{
                SecretName: secretName,
                Items: []v1.KeyToPath{
                        {
                                Key:  SSHPrivateKey,
                                Path: SSHRelativePath + "/" + SSHPrivateKey,
                        },
                        {
                                Key:  SSHPublicKey,
                                Path: SSHRelativePath + "/" + SSHPublicKey,
                        },
                        {
                                Key:  SSHAuthorizedKeys,
                                Path: SSHRelativePath + "/" + SSHAuthorizedKeys,
                        },
                        {
                                Key:  SSHConfig,
                                Path: SSHRelativePath + "/" + SSHConfig,
                        },
                },
                DefaultMode: &mode,
        }

        if sp.sshKeyFilePath != SSHAbsolutePath {
                var noRootMode int32 = 0600
                sshVolume.Secret.DefaultMode = &noRootMode
        }

        pod.Spec.Volumes = append(pod.Spec.Volumes, sshVolume)

        for i, c := range pod.Spec.Containers {
                vm := v1.VolumeMount{
                        MountPath: sp.sshKeyFilePath,
                        SubPath:   SSHRelativePath,
                        Name:      secretName,
                }

                pod.Spec.Containers[i].VolumeMounts = append(c.VolumeMounts, vm)
        }
        for i, c := range pod.Spec.InitContainers {
                vm := v1.VolumeMount{
                        MountPath: sp.sshKeyFilePath,
                        SubPath:   SSHRelativePath,
                        Name:      secretName,
                }

                pod.Spec.InitContainers[i].VolumeMounts = append(c.VolumeMounts, vm)
        }
}

func generateRsaKey(job *batch.Job) (map[string][]byte, error) {
        bitSize := 1024

        privateKey, err := rsa.GenerateKey(rand.Reader, bitSize)
        if err != nil {
                klog.Errorf("rsa generateKey err: %v", err)
                return nil, err
        }

        // id_rsa
        privBlock := pem.Block{
                Type:  "RSA PRIVATE KEY",
                Bytes: x509.MarshalPKCS1PrivateKey(privateKey),
        }
        privateKeyBytes := pem.EncodeToMemory(&privBlock)

        // id_rsa.pub
        publicRsaKey, err := ssh.NewPublicKey(&privateKey.PublicKey)
        if err != nil {
                klog.Errorf("ssh newPublicKey err: %v", err)
                return nil, err
        }
        publicKeyBytes := ssh.MarshalAuthorizedKey(publicRsaKey)

        data := make(map[string][]byte)
        data[SSHPrivateKey] = privateKeyBytes
        data[SSHPublicKey] = publicKeyBytes
        data[SSHAuthorizedKeys] = publicKeyBytes
        data[SSHConfig] = []byte(generateSSHConfig(job))

        return data, nil
}

func withUserProvidedRsaKey(job *batch.Job, sshPrivateKey string, sshPublicKey string) (map[string][]byte, error) {
        data := make(map[string][]byte)
        data[SSHPrivateKey] = []byte(sshPrivateKey)
        data[SSHPublicKey] = []byte(sshPublicKey)
        data[SSHAuthorizedKeys] = []byte(sshPublicKey)
        data[SSHConfig] = []byte(generateSSHConfig(job))

        return data, nil
}

func (sp *sshPlugin) secretName(job *batch.Job) string {
        return fmt.Sprintf("%s-%s", job.Name, sp.Name())
}

func (sp *sshPlugin) addFlags() {
        flagSet := flag.NewFlagSet(sp.Name(), flag.ContinueOnError)
        flagSet.StringVar(&sp.sshKeyFilePath, "ssh-key-file-path", sp.sshKeyFilePath, "The path used to store "+
                "ssh private and public keys, it is `/root/.ssh` by default.")
        flagSet.StringVar(&sp.sshPrivateKey, "ssh-private-key", sp.sshPrivateKey, "The input string of the private key")
        flagSet.StringVar(&sp.sshPublicKey, "ssh-public-key", sp.sshPublicKey, "The input string of the public key")

        if err := flagSet.Parse(sp.pluginArguments); err != nil {
                klog.Errorf("plugin %s flagset parse failed, err: %v", sp.Name(), err)
        }
}

func generateSSHConfig(job *batch.Job) string {
        config := "StrictHostKeyChecking no\nUserKnownHostsFile /dev/null\n"

        for _, ts := range job.Spec.Tasks {
                for i := 0; i < int(ts.Replicas); i++ {
                        hostName := ts.Template.Spec.Hostname
                        subdomain := ts.Template.Spec.Subdomain
                        if len(hostName) == 0 {
                                hostName = jobhelpers.MakePodName(job.Name, ts.Name, i)
                        }
                        if len(subdomain) == 0 {
                                subdomain = job.Name
                        }

                        config += "Host " + hostName + "\n"
                        config += "  HostName " + hostName + "." + subdomain + "\n"
                        if len(ts.Template.Spec.Hostname) != 0 {
                                break
                        }
                }
        }

        return config
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package podgroup

import (
        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/util/wait"
        coreinformers "k8s.io/client-go/informers/core/v1"
        "k8s.io/client-go/kubernetes"
        corelisters "k8s.io/client-go/listers/core/v1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        vcclientset "volcano.sh/apis/pkg/client/clientset/versioned"
        informerfactory "volcano.sh/apis/pkg/client/informers/externalversions"
        schedulinginformer "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1"
        schedulinglister "volcano.sh/apis/pkg/client/listers/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/framework"
)

func init() {
        framework.RegisterController(&pgcontroller{})
}

// pgcontroller the Podgroup pgcontroller type.
type pgcontroller struct {
        kubeClient kubernetes.Interface
        vcClient   vcclientset.Interface

        podInformer coreinformers.PodInformer
        pgInformer  schedulinginformer.PodGroupInformer

        // A store of pods
        podLister corelisters.PodLister
        podSynced func() bool

        // A store of podgroups
        pgLister schedulinglister.PodGroupLister
        pgSynced func() bool

        queue workqueue.RateLimitingInterface
}

func (pg *pgcontroller) Name() string {
        return "pg-controller"
}

// Initialize create new Podgroup Controller.
func (pg *pgcontroller) Initialize(opt *framework.ControllerOption) error {
        pg.kubeClient = opt.KubeClient
        pg.vcClient = opt.VolcanoClient

        pg.queue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())

        pg.podInformer = opt.SharedInformerFactory.Core().V1().Pods()
        pg.podLister = pg.podInformer.Lister()
        pg.podSynced = pg.podInformer.Informer().HasSynced
        pg.podInformer.Informer().AddEventHandler(
                cache.FilteringResourceEventHandler{
                        FilterFunc: func(obj interface{}) bool {
                                switch v := obj.(type) {
                                case *v1.Pod:
                                        if v.Spec.SchedulerName == opt.SchedulerName &&
                                                (v.Annotations == nil || v.Annotations[scheduling.KubeGroupNameAnnotationKey] == "") {
                                                return true
                                        }
                                        return false
                                default:
                                        return false
                                }
                        },
                        Handler: cache.ResourceEventHandlerFuncs{
                                AddFunc: pg.addPod,
                        },
                })

        pg.pgInformer = informerfactory.NewSharedInformerFactory(pg.vcClient, 0).Scheduling().V1beta1().PodGroups()
        pg.pgLister = pg.pgInformer.Lister()
        pg.pgSynced = pg.pgInformer.Informer().HasSynced

        return nil
}

// Run start NewPodgroupController.
func (pg *pgcontroller) Run(stopCh <-chan struct{}) {
        go pg.podInformer.Informer().Run(stopCh)
        go pg.pgInformer.Informer().Run(stopCh)

        cache.WaitForCacheSync(stopCh, pg.podSynced, pg.pgSynced)

        go wait.Until(pg.worker, 0, stopCh)

        klog.Infof("PodgroupController is running ...... ")
}

func (pg *pgcontroller) worker() {
        for pg.processNextReq() {
        }
}

func (pg *pgcontroller) processNextReq() bool {
        obj, shutdown := pg.queue.Get()
        if shutdown {
                klog.Errorf("Fail to pop item from queue")
                return false
        }

        req := obj.(podRequest)
        defer pg.queue.Done(req)

        pod, err := pg.podLister.Pods(req.podNamespace).Get(req.podName)
        if err != nil {
                klog.Errorf("Failed to get pod by <%v> from cache: %v", req, err)
                return true
        }

        // normal pod use volcano
        if err := pg.createNormalPodPGIfNotExist(pod); err != nil {
                klog.Errorf("Failed to handle Pod <%s/%s>: %v", pod.Namespace, pod.Name, err)
                pg.queue.AddRateLimited(req)
                return true
        }

        // If no error, forget it.
        pg.queue.Forget(req)

        return true
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package podgroup

import (
        "context"

        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/runtime/schema"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/helpers"
        scheduling "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

type podRequest struct {
        podName      string
        podNamespace string
}

func (pg *pgcontroller) addPod(obj interface{}) {
        pod, ok := obj.(*v1.Pod)
        if !ok {
                klog.Errorf("Failed to convert %v to v1.Pod", obj)
                return
        }

        req := podRequest{
                podName:      pod.Name,
                podNamespace: pod.Namespace,
        }

        pg.queue.Add(req)
}

func (pg *pgcontroller) updatePodAnnotations(pod *v1.Pod, pgName string) error {
        if pod.Annotations == nil {
                pod.Annotations = make(map[string]string)
        }
        if pod.Annotations[scheduling.KubeGroupNameAnnotationKey] == "" {
                pod.Annotations[scheduling.KubeGroupNameAnnotationKey] = pgName
        } else {
                if pod.Annotations[scheduling.KubeGroupNameAnnotationKey] != pgName {
                        klog.Errorf("normal pod %s/%s annotations %s value is not %s, but %s", pod.Namespace, pod.Name,
                                scheduling.KubeGroupNameAnnotationKey, pgName, pod.Annotations[scheduling.KubeGroupNameAnnotationKey])
                }
                return nil
        }

        if _, err := pg.kubeClient.CoreV1().Pods(pod.Namespace).Update(context.TODO(), pod, metav1.UpdateOptions{}); err != nil {
                klog.Errorf("Failed to update pod <%s/%s>: %v", pod.Namespace, pod.Name, err)
                return err
        }

        return nil
}

func (pg *pgcontroller) createNormalPodPGIfNotExist(pod *v1.Pod) error {
        pgName := helpers.GeneratePodgroupName(pod)

        if _, err := pg.pgLister.PodGroups(pod.Namespace).Get(pgName); err != nil {
                if !apierrors.IsNotFound(err) {
                        klog.Errorf("Failed to get normal PodGroup for Pod <%s/%s>: %v",
                                pod.Namespace, pod.Name, err)
                        return err
                }

                obj := &scheduling.PodGroup{
                        ObjectMeta: metav1.ObjectMeta{
                                Namespace:       pod.Namespace,
                                Name:            pgName,
                                OwnerReferences: newPGOwnerReferences(pod),
                                Annotations:     map[string]string{},
                                Labels:          map[string]string{},
                        },
                        Spec: scheduling.PodGroupSpec{
                                MinMember:         1,
                                PriorityClassName: pod.Spec.PriorityClassName,
                                MinResources:      calcPGMinResources(pod),
                        },
                }
                if queueName, ok := pod.Annotations[scheduling.QueueNameAnnotationKey]; ok {
                        obj.Spec.Queue = queueName
                }

                if value, ok := pod.Annotations[scheduling.PodPreemptable]; ok {
                        obj.Annotations[scheduling.PodPreemptable] = value
                }
                if value, ok := pod.Annotations[scheduling.RevocableZone]; ok {
                        obj.Annotations[scheduling.RevocableZone] = value
                }
                if value, ok := pod.Labels[scheduling.PodPreemptable]; ok {
                        obj.Labels[scheduling.PodPreemptable] = value
                }

                if value, found := pod.Annotations[scheduling.JDBMinAvailable]; found {
                        obj.Annotations[scheduling.JDBMinAvailable] = value
                } else if value, found := pod.Annotations[scheduling.JDBMaxUnavailable]; found {
                        obj.Annotations[scheduling.JDBMaxUnavailable] = value
                }

                if _, err := pg.vcClient.SchedulingV1beta1().PodGroups(pod.Namespace).Create(context.TODO(), obj, metav1.CreateOptions{}); err != nil {
                        klog.Errorf("Failed to create normal PodGroup for Pod <%s/%s>: %v",
                                pod.Namespace, pod.Name, err)
                        return err
                }
        }

        return pg.updatePodAnnotations(pod, pgName)
}

func newPGOwnerReferences(pod *v1.Pod) []metav1.OwnerReference {
        if len(pod.OwnerReferences) != 0 {
                for _, ownerReference := range pod.OwnerReferences {
                        if ownerReference.Controller != nil && *ownerReference.Controller {
                                return pod.OwnerReferences
                        }
                }
        }

        gvk := schema.GroupVersionKind{
                Group:   v1.SchemeGroupVersion.Group,
                Version: v1.SchemeGroupVersion.Version,
                Kind:    "Pod",
        }
        ref := metav1.NewControllerRef(pod, gvk)
        return []metav1.OwnerReference{*ref}
}

// addResourceList add list resource quantity
func addResourceList(list, req, limit v1.ResourceList) {
        for name, quantity := range req {
                if value, ok := list[name]; !ok {
                        list[name] = quantity.DeepCopy()
                } else {
                        value.Add(quantity)
                        list[name] = value
                }
        }

        if req != nil {
                return
        }

        // If Requests is omitted for a container,
        // it defaults to Limits if that is explicitly specified.
        for name, quantity := range limit {
                if value, ok := list[name]; !ok {
                        list[name] = quantity.DeepCopy()
                } else {
                        value.Add(quantity)
                        list[name] = value
                }
        }
}

// calcPGMinResources calculate podgroup minimum resource
func calcPGMinResources(pod *v1.Pod) *v1.ResourceList {
        pgMinRes := v1.ResourceList{}

        for _, c := range pod.Spec.Containers {
                addResourceList(pgMinRes, c.Resources.Requests, c.Resources.Limits)
        }

        return &pgMinRes
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"
        "sync"
        "time"

        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        utilruntime "k8s.io/apimachinery/pkg/util/runtime"
        "k8s.io/apimachinery/pkg/util/wait"
        "k8s.io/client-go/kubernetes"
        corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/client-go/tools/record"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        vcclientset "volcano.sh/apis/pkg/client/clientset/versioned"
        versionedscheme "volcano.sh/apis/pkg/client/clientset/versioned/scheme"
        informerfactory "volcano.sh/apis/pkg/client/informers/externalversions"
        busv1alpha1informer "volcano.sh/apis/pkg/client/informers/externalversions/bus/v1alpha1"
        schedulinginformer "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1"
        busv1alpha1lister "volcano.sh/apis/pkg/client/listers/bus/v1alpha1"
        schedulinglister "volcano.sh/apis/pkg/client/listers/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
        "volcano.sh/volcano/pkg/controllers/framework"
        queuestate "volcano.sh/volcano/pkg/controllers/queue/state"
)

func init() {
        framework.RegisterController(&queuecontroller{})
}

// queuecontroller manages queue status.
type queuecontroller struct {
        kubeClient kubernetes.Interface
        vcClient   vcclientset.Interface

        // informer
        queueInformer schedulinginformer.QueueInformer
        pgInformer    schedulinginformer.PodGroupInformer

        // queueLister
        queueLister schedulinglister.QueueLister
        queueSynced cache.InformerSynced

        // podGroup lister
        pgLister schedulinglister.PodGroupLister
        pgSynced cache.InformerSynced

        cmdInformer busv1alpha1informer.CommandInformer
        cmdLister   busv1alpha1lister.CommandLister
        cmdSynced   cache.InformerSynced

        // queues that need to be updated.
        queue        workqueue.RateLimitingInterface
        commandQueue workqueue.RateLimitingInterface

        pgMutex sync.RWMutex
        // queue name -> podgroup namespace/name
        podGroups map[string]map[string]struct{}

        syncHandler        func(req *apis.Request) error
        syncCommandHandler func(cmd *busv1alpha1.Command) error

        enqueueQueue func(req *apis.Request)

        recorder      record.EventRecorder
        maxRequeueNum int
}

func (c *queuecontroller) Name() string {
        return "queue-controller"
}

// NewQueueController creates a QueueController.
func (c *queuecontroller) Initialize(opt *framework.ControllerOption) error {
        c.vcClient = opt.VolcanoClient
        c.kubeClient = opt.KubeClient

        factory := informerfactory.NewSharedInformerFactory(c.vcClient, 0)
        queueInformer := factory.Scheduling().V1beta1().Queues()
        pgInformer := factory.Scheduling().V1beta1().PodGroups()

        eventBroadcaster := record.NewBroadcaster()
        eventBroadcaster.StartLogging(klog.Infof)
        eventBroadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: c.kubeClient.CoreV1().Events("")})

        c.queueInformer = queueInformer
        c.pgInformer = pgInformer
        c.queueLister = queueInformer.Lister()
        c.queueSynced = queueInformer.Informer().HasSynced
        c.pgLister = pgInformer.Lister()
        c.pgSynced = pgInformer.Informer().HasSynced
        c.queue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
        c.commandQueue = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
        c.podGroups = make(map[string]map[string]struct{})
        c.recorder = eventBroadcaster.NewRecorder(versionedscheme.Scheme, v1.EventSource{Component: "vc-controller-manager"})
        c.maxRequeueNum = opt.MaxRequeueNum
        if c.maxRequeueNum < 0 {
                c.maxRequeueNum = -1
        }

        queueInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    c.addQueue,
                UpdateFunc: c.updateQueue,
                DeleteFunc: c.deleteQueue,
        })

        pgInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    c.addPodGroup,
                UpdateFunc: c.updatePodGroup,
                DeleteFunc: c.deletePodGroup,
        })

        c.cmdInformer = informerfactory.NewSharedInformerFactory(c.vcClient, 0).Bus().V1alpha1().Commands()
        c.cmdInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
                FilterFunc: func(obj interface{}) bool {
                        switch v := obj.(type) {
                        case *busv1alpha1.Command:
                                return IsQueueReference(v.TargetObject)
                        default:
                                return false
                        }
                },
                Handler: cache.ResourceEventHandlerFuncs{
                        AddFunc: c.addCommand,
                },
        })
        c.cmdLister = c.cmdInformer.Lister()
        c.cmdSynced = c.cmdInformer.Informer().HasSynced

        queuestate.SyncQueue = c.syncQueue
        queuestate.OpenQueue = c.openQueue
        queuestate.CloseQueue = c.closeQueue

        c.syncHandler = c.handleQueue
        c.syncCommandHandler = c.handleCommand

        c.enqueueQueue = c.enqueue

        return nil
}

// Run starts QueueController.
func (c *queuecontroller) Run(stopCh <-chan struct{}) {
        defer utilruntime.HandleCrash()
        defer c.queue.ShutDown()
        defer c.commandQueue.ShutDown()

        klog.Infof("Starting queue controller.")
        defer klog.Infof("Shutting down queue controller.")

        go c.queueInformer.Informer().Run(stopCh)
        go c.pgInformer.Informer().Run(stopCh)
        go c.cmdInformer.Informer().Run(stopCh)

        if !cache.WaitForCacheSync(stopCh, c.queueSynced, c.pgSynced, c.cmdSynced) {
                klog.Errorf("unable to sync caches for queue controller.")
                return
        }

        go wait.Until(c.worker, 0, stopCh)
        go wait.Until(c.commandWorker, 0, stopCh)

        <-stopCh
}

// worker runs a worker thread that just dequeues items, processes them, and
// marks them done. You may run as many of these in parallel as you wish; the
// workqueue guarantees that they will not end up processing the same `queue`
// at the same time.
func (c *queuecontroller) worker() {
        for c.processNextWorkItem() {
        }
}

func (c *queuecontroller) processNextWorkItem() bool {
        obj, shutdown := c.queue.Get()
        if shutdown {
                return false
        }
        defer c.queue.Done(obj)

        req, ok := obj.(*apis.Request)
        if !ok {
                klog.Errorf("%v is not a valid queue request struct.", obj)
                return true
        }

        err := c.syncHandler(req)
        c.handleQueueErr(err, obj)

        return true
}

func (c *queuecontroller) handleQueue(req *apis.Request) error {
        startTime := time.Now()
        defer func() {
                klog.V(4).Infof("Finished syncing queue %s (%v).", req.QueueName, time.Since(startTime))
        }()

        queue, err := c.queueLister.Get(req.QueueName)
        if err != nil {
                if apierrors.IsNotFound(err) {
                        klog.V(4).Infof("Queue %s has been deleted.", req.QueueName)
                        return nil
                }

                return fmt.Errorf("get queue %s failed for %v", req.QueueName, err)
        }

        queueState := queuestate.NewState(queue)
        if queueState == nil {
                return fmt.Errorf("queue %s state %s is invalid", queue.Name, queue.Status.State)
        }

        klog.V(4).Infof("Begin execute %s action for queue %s, current status %s", req.Action, req.QueueName, queue.Status.State)
        if err := queueState.Execute(req.Action); err != nil {
                return fmt.Errorf("sync queue %s failed for %v, event is %v, action is %s",
                        req.QueueName, err, req.Event, req.Action)
        }

        return nil
}

func (c *queuecontroller) handleQueueErr(err error, obj interface{}) {
        if err == nil {
                c.queue.Forget(obj)
                return
        }

        if c.maxRequeueNum == -1 || c.queue.NumRequeues(obj) < c.maxRequeueNum {
                klog.V(4).Infof("Error syncing queue request %v for %v.", obj, err)
                c.queue.AddRateLimited(obj)
                return
        }

        req, _ := obj.(*apis.Request)
        c.recordEventsForQueue(req.QueueName, v1.EventTypeWarning, string(req.Action),
                fmt.Sprintf("%v queue failed for %v", req.Action, err))
        klog.V(2).Infof("Dropping queue request %v out of the queue for %v.", obj, err)
        c.queue.Forget(obj)
}

func (c *queuecontroller) commandWorker() {
        for c.processNextCommand() {
        }
}

func (c *queuecontroller) processNextCommand() bool {
        obj, shutdown := c.commandQueue.Get()
        if shutdown {
                return false
        }
        defer c.commandQueue.Done(obj)

        cmd, ok := obj.(*busv1alpha1.Command)
        if !ok {
                klog.Errorf("%v is not a valid Command struct.", obj)
                return true
        }

        err := c.syncCommandHandler(cmd)
        c.handleCommandErr(err, obj)

        return true
}

func (c *queuecontroller) handleCommand(cmd *busv1alpha1.Command) error {
        startTime := time.Now()
        defer func() {
                klog.V(4).Infof("Finished syncing command %s/%s (%v).", cmd.Namespace, cmd.Name, time.Since(startTime))
        }()

        err := c.vcClient.BusV1alpha1().Commands(cmd.Namespace).Delete(context.TODO(), cmd.Name, metav1.DeleteOptions{})
        if err != nil {
                if apierrors.IsNotFound(err) {
                        return nil
                }

                return fmt.Errorf("failed to delete command <%s/%s> for %v", cmd.Namespace, cmd.Name, err)
        }

        req := &apis.Request{
                QueueName: cmd.TargetObject.Name,
                Event:     busv1alpha1.CommandIssuedEvent,
                Action:    busv1alpha1.Action(cmd.Action),
        }

        c.enqueueQueue(req)

        return nil
}

func (c *queuecontroller) handleCommandErr(err error, obj interface{}) {
        if err == nil {
                c.commandQueue.Forget(obj)
                return
        }

        if c.maxRequeueNum == -1 || c.commandQueue.NumRequeues(obj) < c.maxRequeueNum {
                klog.V(4).Infof("Error syncing command %v for %v.", obj, err)
                c.commandQueue.AddRateLimited(obj)
                return
        }

        klog.V(2).Infof("Dropping command %v out of the queue for %v.", obj, err)
        c.commandQueue.Forget(obj)
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "context"
        "fmt"
        "reflect"

        "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/queue/state"

        v1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/tools/cache"

        "k8s.io/klog"
)

func (c *queuecontroller) syncQueue(queue *schedulingv1beta1.Queue, updateStateFn state.UpdateQueueStatusFn) error {
        klog.V(4).Infof("Begin to sync queue %s.", queue.Name)
        defer klog.V(4).Infof("End sync queue %s.", queue.Name)

        podGroups := c.getPodGroups(queue.Name)
        queueStatus := schedulingv1beta1.QueueStatus{}

        for _, pgKey := range podGroups {
                // Ignore error here, tt can not occur.
                ns, name, _ := cache.SplitMetaNamespaceKey(pgKey)

                // TODO: check NotFound error and sync local cache.
                pg, err := c.pgLister.PodGroups(ns).Get(name)
                if err != nil {
                        return err
                }

                switch pg.Status.Phase {
                case schedulingv1beta1.PodGroupPending:
                        queueStatus.Pending++
                case schedulingv1beta1.PodGroupRunning:
                        queueStatus.Running++
                case schedulingv1beta1.PodGroupUnknown:
                        queueStatus.Unknown++
                case schedulingv1beta1.PodGroupInqueue:
                        queueStatus.Inqueue++
                }
        }

        if updateStateFn != nil {
                updateStateFn(&queueStatus, podGroups)
        } else {
                queueStatus.State = queue.Status.State
        }

        // ignore update when status does not change
        if reflect.DeepEqual(queueStatus, queue.Status) {
                return nil
        }

        newQueue := queue.DeepCopy()
        newQueue.Status = queueStatus
        if _, err := c.vcClient.SchedulingV1beta1().Queues().UpdateStatus(context.TODO(), newQueue, metav1.UpdateOptions{}); err != nil {
                klog.Errorf("Failed to update status of Queue %s: %v.", newQueue.Name, err)
                return err
        }

        return nil
}

func (c *queuecontroller) openQueue(queue *schedulingv1beta1.Queue, updateStateFn state.UpdateQueueStatusFn) error {
        klog.V(4).Infof("Begin to open queue %s.", queue.Name)

        newQueue := queue.DeepCopy()
        newQueue.Status.State = schedulingv1beta1.QueueStateOpen

        if queue.Status.State != newQueue.Status.State {
                if _, err := c.vcClient.SchedulingV1beta1().Queues().Update(context.TODO(), newQueue, metav1.UpdateOptions{}); err != nil {
                        c.recorder.Event(newQueue, v1.EventTypeWarning, string(v1alpha1.OpenQueueAction),
                                fmt.Sprintf("Open queue failed for %v", err))
                        return err
                }

                c.recorder.Event(newQueue, v1.EventTypeNormal, string(v1alpha1.OpenQueueAction), "Open queue succeed")
        } else {
                return nil
        }

        q, err := c.vcClient.SchedulingV1beta1().Queues().Get(context.TODO(), newQueue.Name, metav1.GetOptions{})
        if err != nil {
                return err
        }

        newQueue = q.DeepCopy()
        if updateStateFn != nil {
                updateStateFn(&newQueue.Status, nil)
        } else {
                return fmt.Errorf("internal error, update state function should be provided")
        }

        if queue.Status.State != newQueue.Status.State {
                if _, err := c.vcClient.SchedulingV1beta1().Queues().UpdateStatus(context.TODO(), newQueue, metav1.UpdateOptions{}); err != nil {
                        c.recorder.Event(newQueue, v1.EventTypeWarning, string(v1alpha1.OpenQueueAction),
                                fmt.Sprintf("Update queue status from %s to %s failed for %v",
                                        queue.Status.State, newQueue.Status.State, err))
                        return err
                }
        }

        return nil
}

func (c *queuecontroller) closeQueue(queue *schedulingv1beta1.Queue, updateStateFn state.UpdateQueueStatusFn) error {
        klog.V(4).Infof("Begin to close queue %s.", queue.Name)

        newQueue := queue.DeepCopy()
        newQueue.Status.State = schedulingv1beta1.QueueStateClosed

        if queue.Status.State != newQueue.Status.State {
                if _, err := c.vcClient.SchedulingV1beta1().Queues().Update(context.TODO(), newQueue, metav1.UpdateOptions{}); err != nil {
                        c.recorder.Event(newQueue, v1.EventTypeWarning, string(v1alpha1.CloseQueueAction),
                                fmt.Sprintf("Close queue failed for %v", err))
                        return err
                }

                c.recorder.Event(newQueue, v1.EventTypeNormal, string(v1alpha1.CloseQueueAction), "Close queue succeed")
        } else {
                return nil
        }

        q, err := c.vcClient.SchedulingV1beta1().Queues().Get(context.TODO(), newQueue.Name, metav1.GetOptions{})
        if err != nil {
                return err
        }

        newQueue = q.DeepCopy()
        podGroups := c.getPodGroups(newQueue.Name)
        if updateStateFn != nil {
                updateStateFn(&newQueue.Status, podGroups)
        } else {
                return fmt.Errorf("internal error, update state function should be provided")
        }

        if queue.Status.State != newQueue.Status.State {
                if _, err := c.vcClient.SchedulingV1beta1().Queues().UpdateStatus(context.TODO(), newQueue, metav1.UpdateOptions{}); err != nil {
                        c.recorder.Event(newQueue, v1.EventTypeWarning, string(v1alpha1.CloseQueueAction),
                                fmt.Sprintf("Update queue status from %s to %s failed for %v",
                                        queue.Status.State, newQueue.Status.State, err))
                        return err
                }
        }

        return nil
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        "k8s.io/client-go/tools/cache"
        "k8s.io/klog"

        busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1"
        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/controllers/apis"
)

func (c *queuecontroller) enqueue(req *apis.Request) {
        c.queue.Add(req)
}

func (c *queuecontroller) addQueue(obj interface{}) {
        queue := obj.(*schedulingv1beta1.Queue)

        req := &apis.Request{
                QueueName: queue.Name,

                Event:  busv1alpha1.OutOfSyncEvent,
                Action: busv1alpha1.SyncQueueAction,
        }

        c.enqueue(req)
}

func (c *queuecontroller) deleteQueue(obj interface{}) {
        queue, ok := obj.(*schedulingv1beta1.Queue)
        if !ok {
                tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
                if !ok {
                        klog.Errorf("Couldn't get object from tombstone %#v.", obj)
                        return
                }
                queue, ok = tombstone.Obj.(*schedulingv1beta1.Queue)
                if !ok {
                        klog.Errorf("Tombstone contained object that is not a Queue: %#v.", obj)
                        return
                }
        }

        c.pgMutex.Lock()
        defer c.pgMutex.Unlock()
        delete(c.podGroups, queue.Name)
}

func (c *queuecontroller) updateQueue(_, _ interface{}) {
        // currently do not care about queue update
}

func (c *queuecontroller) addPodGroup(obj interface{}) {
        pg := obj.(*schedulingv1beta1.PodGroup)
        key, _ := cache.MetaNamespaceKeyFunc(obj)

        c.pgMutex.Lock()
        defer c.pgMutex.Unlock()

        if c.podGroups[pg.Spec.Queue] == nil {
                c.podGroups[pg.Spec.Queue] = make(map[string]struct{})
        }
        c.podGroups[pg.Spec.Queue][key] = struct{}{}

        req := &apis.Request{
                QueueName: pg.Spec.Queue,

                Event:  busv1alpha1.OutOfSyncEvent,
                Action: busv1alpha1.SyncQueueAction,
        }

        c.enqueue(req)
}

func (c *queuecontroller) updatePodGroup(old, new interface{}) {
        oldPG := old.(*schedulingv1beta1.PodGroup)
        newPG := new.(*schedulingv1beta1.PodGroup)

        // Note: we have no use case update PodGroup.Spec.Queue
        // So do not consider it here.
        if oldPG.Status.Phase != newPG.Status.Phase {
                c.addPodGroup(newPG)
        }
}

func (c *queuecontroller) deletePodGroup(obj interface{}) {
        pg, ok := obj.(*schedulingv1beta1.PodGroup)
        if !ok {
                tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
                if !ok {
                        klog.Errorf("Couldn't get object from tombstone %#v.", obj)
                        return
                }
                pg, ok = tombstone.Obj.(*schedulingv1beta1.PodGroup)
                if !ok {
                        klog.Errorf("Tombstone contained object that is not a PodGroup: %#v.", obj)
                        return
                }
        }

        key, _ := cache.MetaNamespaceKeyFunc(obj)

        c.pgMutex.Lock()
        defer c.pgMutex.Unlock()

        delete(c.podGroups[pg.Spec.Queue], key)

        req := &apis.Request{
                QueueName: pg.Spec.Queue,

                Event:  busv1alpha1.OutOfSyncEvent,
                Action: busv1alpha1.SyncQueueAction,
        }

        c.enqueue(req)
}

func (c *queuecontroller) addCommand(obj interface{}) {
        cmd, ok := obj.(*busv1alpha1.Command)
        if !ok {
                klog.Errorf("Obj %v is not command.", obj)
                return
        }

        c.commandQueue.Add(cmd)
}

func (c *queuecontroller) getPodGroups(key string) []string {
        c.pgMutex.RLock()
        defer c.pgMutex.RUnlock()

        if c.podGroups[key] == nil {
                return nil
        }
        podGroups := make([]string, 0, len(c.podGroups[key]))
        for pgKey := range c.podGroups[key] {
                podGroups = append(podGroups, pgKey)
        }

        return podGroups
}

func (c *queuecontroller) recordEventsForQueue(name, eventType, reason, message string) {
        queue, err := c.queueLister.Get(name)
        if err != nil {
                klog.Errorf("Get queue %s failed for %v.", name, err)
                return
        }

        c.recorder.Event(queue, eventType, reason, message)
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package queue

import (
        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// IsQueueReference return if ownerReference is Queue Kind.
func IsQueueReference(ref *metav1.OwnerReference) bool {
        if ref == nil {
                return false
        }

        if ref.APIVersion != schedulingv1beta1.SchemeGroupVersion.String() {
                return false
        }

        if ref.Kind != "Queue" {
                return false
        }

        return true
}

/*
 Copyright 2021 The Volcano Authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package allocate

import (
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/metrics"
        "volcano.sh/volcano/pkg/scheduler/util"
)

var targetJob = util.Reservation.TargetJob

type Action struct{}

func New() *Action {
        return &Action{}
}

func (alloc *Action) Name() string {
        return "allocate"
}

func (alloc *Action) Initialize() {}

func (alloc *Action) Execute(ssn *framework.Session) {
        klog.V(3).Infof("Enter Allocate ...")
        defer klog.V(3).Infof("Leaving Allocate ...")

        // the allocation for pod may have many stages
        // 1. pick a namespace named N (using ssn.NamespaceOrderFn)
        // 2. pick a queue named Q from N (using ssn.QueueOrderFn)
        // 3. pick a job named J from Q (using ssn.JobOrderFn)
        // 4. pick a task T from J (using ssn.TaskOrderFn)
        // 5. use predicateFn to filter out node that T can not be allocated on.
        // 6. use ssn.NodeOrderFn to judge the best node and assign it to T

        namespaces := util.NewPriorityQueue(ssn.NamespaceOrderFn)

        // jobsMap is map[api.NamespaceName]map[api.QueueID]PriorityQueue(*api.JobInfo)
        // used to find job with highest priority in given queue and namespace
        jobsMap := map[api.NamespaceName]map[api.QueueID]*util.PriorityQueue{}

        for _, job := range ssn.Jobs {
                if job.IsPending() {
                        klog.V(4).Infof("Job <%s/%s> Queue <%s> skip allocate, reason: job status is pending.",
                                job.Namespace, job.Name, job.Queue)
                        continue
                }
                if vr := ssn.JobValid(job); vr != nil && !vr.Pass {
                        klog.V(4).Infof("Job <%s/%s> Queue <%s> skip allocate, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message)
                        continue
                }

                if _, found := ssn.Queues[job.Queue]; !found {
                        klog.Warningf("Skip adding Job <%s/%s> because its queue %s is not found",
                                job.Namespace, job.Name, job.Queue)
                        continue
                }

                namespace := api.NamespaceName(job.Namespace)
                queueMap, found := jobsMap[namespace]
                if !found {
                        namespaces.Push(namespace)

                        queueMap = make(map[api.QueueID]*util.PriorityQueue)
                        jobsMap[namespace] = queueMap
                }

                jobs, found := queueMap[job.Queue]
                if !found {
                        jobs = util.NewPriorityQueue(ssn.JobOrderFn)
                        queueMap[job.Queue] = jobs
                }

                klog.V(4).Infof("Added Job <%s/%s> into Queue <%s>", job.Namespace, job.Name, job.Queue)
                jobs.Push(job)
        }

        klog.V(3).Infof("Try to allocate resource to %d Namespaces", len(jobsMap))

        pendingTasks := map[api.JobID]*util.PriorityQueue{}

        allNodes := ssn.NodeList
        unlockedNodes := allNodes
        if targetJob != nil && len(util.Reservation.LockedNodes) != 0 {
                unlockedNodes = unlockedNodes[0:0]
                for _, node := range allNodes {
                        if _, exist := util.Reservation.LockedNodes[node.Name]; !exist {
                                unlockedNodes = append(unlockedNodes, node)
                        }
                }
        }
        for _, unlockedNode := range unlockedNodes {
                klog.V(4).Infof("unlockedNode ID: %s, Name: %s", unlockedNode.Node.UID, unlockedNode.Node.Name)
        }
        predicateFn := func(task *api.TaskInfo, node *api.NodeInfo) error {
                // Check for Resource Predicate
                if !task.InitResreq.LessEqual(node.FutureIdle(), api.Zero) {
                        return api.NewFitError(task, node, api.NodeResourceFitFailed)
                }

                return ssn.PredicateFn(task, node)
        }

        // To pick <namespace, queue> tuple for job, we choose to pick namespace firstly.
        // Because we believe that number of queues would less than namespaces in most case.
        // And, this action would make the resource usage among namespace balanced.
        for {
                if namespaces.Empty() {
                        break
                }

                // pick namespace from namespaces PriorityQueue
                namespace := namespaces.Pop().(api.NamespaceName)

                queueInNamespace := jobsMap[namespace]

                // pick queue for given namespace
                //
                // This block use a algorithm with time complex O(n).
                // But at least PriorityQueue could not be used here,
                // because the allocation of job would change the priority of queue among all namespaces,
                // and the PriorityQueue have no ability to update priority for a special queue.
                var queue *api.QueueInfo
                for queueID := range queueInNamespace {
                        currentQueue := ssn.Queues[queueID]
                        if ssn.Overused(currentQueue) {
                                klog.V(3).Infof("Namespace <%s> Queue <%s> is overused, ignore it.", namespace, currentQueue.Name)
                                delete(queueInNamespace, queueID)
                                continue
                        }
                        if jobs, found := queueInNamespace[currentQueue.UID]; found && jobs.Empty() {
                                continue
                        }

                        if queue == nil || ssn.QueueOrderFn(currentQueue, queue) {
                                queue = currentQueue
                        }
                }

                if queue == nil {
                        klog.V(3).Infof("Namespace <%s> have no queue, skip it", namespace)
                        continue
                }

                klog.V(3).Infof("Try to allocate resource to Jobs in Namespace <%s> Queue <%v>", namespace, queue.Name)

                jobs, found := queueInNamespace[queue.UID]
                if !found || jobs.Empty() {
                        delete(queueInNamespace, queue.UID)
                        namespaces.Push(namespace)
                        klog.V(4).Infof("Can not find jobs for queue %s.", queue.Name)
                        continue
                }

                job := jobs.Pop().(*api.JobInfo)
                var nodes []*api.NodeInfo
                if targetJob != nil && job.UID == targetJob.UID {
                        klog.V(4).Infof("Try to allocate resource to target job: %s", job.Name)
                        nodes = allNodes
                } else {
                        nodes = unlockedNodes
                }
                if _, found = pendingTasks[job.UID]; !found {
                        tasks := util.NewPriorityQueue(ssn.TaskOrderFn)
                        for _, task := range job.TaskStatusIndex[api.Pending] {
                                // Skip BestEffort task in 'allocate' action.
                                if task.Resreq.IsEmpty() {
                                        klog.V(4).Infof("Task <%v/%v> is BestEffort task, skip it.",
                                                task.Namespace, task.Name)
                                        continue
                                }

                                tasks.Push(task)
                        }
                        pendingTasks[job.UID] = tasks
                }
                tasks := pendingTasks[job.UID]

                klog.V(3).Infof("Try to allocate resource to %d tasks of Job <%v/%v>",
                        tasks.Len(), job.Namespace, job.Name)

                stmt := framework.NewStatement(ssn)

                for !tasks.Empty() {
                        task := tasks.Pop().(*api.TaskInfo)

                        // Check whether the queue is overused on dimension that the task requested
                        taskRequest := task.Resreq.ResourceNames()
                        if underusedResources := ssn.UnderusedResources(queue); underusedResources != nil && !underusedResources.Contains(taskRequest) {
                                klog.V(3).Infof("Queue <%s> is overused when considering task <%s>, ignore it.", queue.Name, task.Name)
                                continue
                        }

                        klog.V(3).Infof("There are <%d> nodes for Job <%v/%v>", len(nodes), job.Namespace, job.Name)

                        predicateNodes, fitErrors := util.PredicateNodes(task, nodes, predicateFn)
                        if len(predicateNodes) == 0 {
                                job.NodesFitErrors[task.UID] = fitErrors
                                break
                        }

                        var candidateNodes []*api.NodeInfo
                        for _, n := range predicateNodes {
                                if task.InitResreq.LessEqual(n.Idle, api.Zero) || task.InitResreq.LessEqual(n.FutureIdle(), api.Zero) {
                                        candidateNodes = append(candidateNodes, n)
                                }
                        }

                        // If not candidate nodes for this task, skip it.
                        if len(candidateNodes) == 0 {
                                continue
                        }

                        nodeScores := util.PrioritizeNodes(task, candidateNodes, ssn.BatchNodeOrderFn, ssn.NodeOrderMapFn, ssn.NodeOrderReduceFn)

                        node := ssn.BestNodeFn(task, nodeScores)
                        if node == nil {
                                node = util.SelectBestNode(nodeScores)
                        }

                        // Allocate idle resource to the task.
                        if task.InitResreq.LessEqual(node.Idle, api.Zero) {
                                klog.V(3).Infof("Binding Task <%v/%v> to node <%v>",
                                        task.Namespace, task.Name, node.Name)
                                if err := stmt.Allocate(task, node); err != nil {
                                        klog.Errorf("Failed to bind Task %v on %v in Session %v, err: %v",
                                                task.UID, node.Name, ssn.UID, err)
                                } else {
                                        metrics.UpdateE2eSchedulingDurationByJob(job.Name, string(job.Queue), job.Namespace, metrics.Duration(job.CreationTimestamp.Time))
                                }
                        } else {
                                klog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s> with limited resources",
                                        task.Namespace, task.Name, node.Name)

                                // Allocate releasing resource to the task if any.
                                if task.InitResreq.LessEqual(node.FutureIdle(), api.Zero) {
                                        klog.V(3).Infof("Pipelining Task <%v/%v> to node <%v> for <%v> on <%v>",
                                                task.Namespace, task.Name, node.Name, task.InitResreq, node.Releasing)
                                        if err := stmt.Pipeline(task, node.Name); err != nil {
                                                klog.Errorf("Failed to pipeline Task %v on %v in Session %v for %v.",
                                                        task.UID, node.Name, ssn.UID, err)
                                        } else {
                                                metrics.UpdateE2eSchedulingDurationByJob(job.Name, string(job.Queue), job.Namespace, metrics.Duration(job.CreationTimestamp.Time))
                                        }
                                }
                        }

                        if ssn.JobReady(job) && !tasks.Empty() {
                                jobs.Push(job)
                                break
                        }
                }

                if ssn.JobReady(job) {
                        stmt.Commit()
                } else {
                        if !ssn.JobPipelined(job) {
                                stmt.Discard()
                        }
                }

                // Added Namespace back until no job in Namespace.
                namespaces.Push(namespace)
        }
}

func (alloc *Action) UnInitialize() {}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package preempt

import (
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/metrics"
        "volcano.sh/volcano/pkg/scheduler/util"
)

type Action struct{}

func New() *Action {
        return &Action{}
}

func (alloc *Action) Name() string {
        return "preempt"
}

func (alloc *Action) Initialize() {}

func (alloc *Action) Execute(ssn *framework.Session) {
        klog.V(3).Infof("Enter Preempt ...")
        defer klog.V(3).Infof("Leaving Preempt ...")

        preemptorsMap := map[api.QueueID]*util.PriorityQueue{}
        preemptorTasks := map[api.JobID]*util.PriorityQueue{}

        var underRequest []*api.JobInfo
        queues := map[api.QueueID]*api.QueueInfo{}

        for _, job := range ssn.Jobs {
                if job.IsPending() {
                        continue
                }

                if vr := ssn.JobValid(job); vr != nil && !vr.Pass {
                        klog.V(4).Infof("Job <%s/%s> Queue <%s> skip preemption, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message)
                        continue
                }

                if queue, found := ssn.Queues[job.Queue]; !found {
                        continue
                } else if _, existed := queues[queue.UID]; !existed {
                        klog.V(3).Infof("Added Queue <%s> for Job <%s/%s>",
                                queue.Name, job.Namespace, job.Name)
                        queues[queue.UID] = queue
                }

                // check job if starting for more resources.
                if ssn.JobStarving(job) {
                        if _, found := preemptorsMap[job.Queue]; !found {
                                preemptorsMap[job.Queue] = util.NewPriorityQueue(ssn.JobOrderFn)
                        }
                        preemptorsMap[job.Queue].Push(job)
                        underRequest = append(underRequest, job)
                        preemptorTasks[job.UID] = util.NewPriorityQueue(ssn.TaskOrderFn)
                        for _, task := range job.TaskStatusIndex[api.Pending] {
                                preemptorTasks[job.UID].Push(task)
                        }
                }
        }

        // Preemption between Jobs within Queue.
        for _, queue := range queues {
                for {
                        preemptors := preemptorsMap[queue.UID]

                        // If no preemptors, no preemption.
                        if preemptors == nil || preemptors.Empty() {
                                klog.V(4).Infof("No preemptors in Queue <%s>, break.", queue.Name)
                                break
                        }

                        preemptorJob := preemptors.Pop().(*api.JobInfo)

                        stmt := framework.NewStatement(ssn)
                        assigned := false
                        for {
                                // If job is not request more resource, then stop preempting.
                                if !ssn.JobStarving(preemptorJob) {
                                        break
                                }

                                // If not preemptor tasks, next job.
                                if preemptorTasks[preemptorJob.UID].Empty() {
                                        klog.V(3).Infof("No preemptor task in job <%s/%s>.",
                                                preemptorJob.Namespace, preemptorJob.Name)
                                        break
                                }

                                preemptor := preemptorTasks[preemptorJob.UID].Pop().(*api.TaskInfo)

                                if preempted, _ := preempt(ssn, stmt, preemptor, func(task *api.TaskInfo) bool {
                                        // Ignore non running task.
                                        if task.Status != api.Running {
                                                return false
                                        }
                                        // Ignore task with empty resource request.
                                        if task.Resreq.IsEmpty() {
                                                return false
                                        }
                                        job, found := ssn.Jobs[task.Job]
                                        if !found {
                                                return false
                                        }
                                        // Preempt other jobs within queue
                                        return job.Queue == preemptorJob.Queue && preemptor.Job != task.Job
                                }); preempted {
                                        assigned = true
                                }
                        }

                        // Commit changes only if job is pipelined, otherwise try next job.
                        if ssn.JobPipelined(preemptorJob) {
                                stmt.Commit()
                        } else {
                                stmt.Discard()
                                continue
                        }

                        if assigned {
                                preemptors.Push(preemptorJob)
                        }
                }

                // Preemption between Task within Job.
                for _, job := range underRequest {
                        // Fix: preemptor numbers lose when in same job
                        preemptorTasks[job.UID] = util.NewPriorityQueue(ssn.TaskOrderFn)
                        for _, task := range job.TaskStatusIndex[api.Pending] {
                                preemptorTasks[job.UID].Push(task)
                        }
                        for {
                                if _, found := preemptorTasks[job.UID]; !found {
                                        break
                                }

                                if preemptorTasks[job.UID].Empty() {
                                        break
                                }

                                preemptor := preemptorTasks[job.UID].Pop().(*api.TaskInfo)

                                stmt := framework.NewStatement(ssn)
                                assigned, _ := preempt(ssn, stmt, preemptor, func(task *api.TaskInfo) bool {
                                        // Ignore non running task.
                                        if task.Status != api.Running {
                                                return false
                                        }
                                        // Ignore task with empty resource request.
                                        if task.Resreq.IsEmpty() {
                                                return false
                                        }
                                        // Preempt tasks within job.
                                        return preemptor.Job == task.Job
                                })
                                stmt.Commit()

                                // If no preemption, next job.
                                if !assigned {
                                        break
                                }
                        }
                }
        }

        // call victimTasksFn to evict tasks
        victimTasks(ssn)
}

func (alloc *Action) UnInitialize() {}

func preempt(
        ssn *framework.Session,
        stmt *framework.Statement,
        preemptor *api.TaskInfo,
        filter func(*api.TaskInfo) bool,
) (bool, error) {
        assigned := false

        allNodes := ssn.NodeList

        predicateNodes, _ := util.PredicateNodes(preemptor, allNodes, ssn.PredicateFn)

        nodeScores := util.PrioritizeNodes(preemptor, predicateNodes, ssn.BatchNodeOrderFn, ssn.NodeOrderMapFn, ssn.NodeOrderReduceFn)

        selectedNodes := util.SortNodes(nodeScores)
        for _, node := range selectedNodes {
                klog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.",
                        preemptor.Namespace, preemptor.Name, node.Name)

                var preemptees []*api.TaskInfo
                for _, task := range node.Tasks {
                        if filter == nil {
                                preemptees = append(preemptees, task.Clone())
                        } else if filter(task) {
                                preemptees = append(preemptees, task.Clone())
                        }
                }
                victims := ssn.Preemptable(preemptor, preemptees)
                metrics.UpdatePreemptionVictimsCount(len(victims))

                if err := util.ValidateVictims(preemptor, node, victims); err != nil {
                        klog.V(3).Infof("No validated victims on Node <%s>: %v", node.Name, err)
                        continue
                }

                victimsQueue := util.NewPriorityQueue(func(l, r interface{}) bool {
                        return !ssn.TaskOrderFn(l, r)
                })
                for _, victim := range victims {
                        victimsQueue.Push(victim)
                }
                // Preempt victims for tasks, pick lowest priority task first.
                preempted := api.EmptyResource()

                for !victimsQueue.Empty() {
                        // If reclaimed enough resources, break loop to avoid Sub panic.
                        if preemptor.InitResreq.LessEqual(node.FutureIdle(), api.Zero) {
                                break
                        }
                        preemptee := victimsQueue.Pop().(*api.TaskInfo)
                        klog.V(3).Infof("Try to preempt Task <%s/%s> for Tasks <%s/%s>",
                                preemptee.Namespace, preemptee.Name, preemptor.Namespace, preemptor.Name)
                        if err := stmt.Evict(preemptee, "preempt"); err != nil {
                                klog.Errorf("Failed to preempt Task <%s/%s> for Tasks <%s/%s>: %v",
                                        preemptee.Namespace, preemptee.Name, preemptor.Namespace, preemptor.Name, err)
                                continue
                        }
                        preempted.Add(preemptee.Resreq)
                }

                metrics.RegisterPreemptionAttempts()
                klog.V(3).Infof("Preempted <%v> for Task <%s/%s> requested <%v>.",
                        preempted, preemptor.Namespace, preemptor.Name, preemptor.InitResreq)

                if preemptor.InitResreq.LessEqual(node.FutureIdle(), api.Zero) {
                        if err := stmt.Pipeline(preemptor, node.Name); err != nil {
                                klog.Errorf("Failed to pipeline Task <%s/%s> on Node <%s>",
                                        preemptor.Namespace, preemptor.Name, node.Name)
                        }

                        // Ignore pipeline error, will be corrected in next scheduling loop.
                        assigned = true

                        break
                }
        }

        return assigned, nil
}

func victimTasks(ssn *framework.Session) {
        stmt := framework.NewStatement(ssn)
        victimTasks := ssn.VictimTasks()
        for _, victim := range victimTasks {
                if err := stmt.Evict(victim.Clone(), "evict"); err != nil {
                        klog.Errorf("Failed to evict Task <%s/%s>: %v",
                                victim.Namespace, victim.Name, err)
                        continue
                }
        }
        stmt.Commit()
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reclaim

import (
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/util"
)

type Action struct{}

func New() *Action {
        return &Action{}
}

func (ra *Action) Name() string {
        return "reclaim"
}

func (ra *Action) Initialize() {}

func (ra *Action) Execute(ssn *framework.Session) {
        klog.V(3).Infof("Enter Reclaim ...")
        defer klog.V(3).Infof("Leaving Reclaim ...")

        queues := util.NewPriorityQueue(ssn.QueueOrderFn)
        queueMap := map[api.QueueID]*api.QueueInfo{}

        preemptorsMap := map[api.QueueID]*util.PriorityQueue{}
        preemptorTasks := map[api.JobID]*util.PriorityQueue{}

        klog.V(3).Infof("There are <%d> Jobs and <%d> Queues in total for scheduling.",
                len(ssn.Jobs), len(ssn.Queues))

        for _, job := range ssn.Jobs {
                if job.IsPending() {
                        continue
                }

                if vr := ssn.JobValid(job); vr != nil && !vr.Pass {
                        klog.V(4).Infof("Job <%s/%s> Queue <%s> skip reclaim, reason: %v, message %v", job.Namespace, job.Name, job.Queue, vr.Reason, vr.Message)
                        continue
                }

                if queue, found := ssn.Queues[job.Queue]; !found {
                        klog.Errorf("Failed to find Queue <%s> for Job <%s/%s>",
                                job.Queue, job.Namespace, job.Name)
                        continue
                } else if _, existed := queueMap[queue.UID]; !existed {
                        klog.V(4).Infof("Added Queue <%s> for Job <%s/%s>", queue.Name, job.Namespace, job.Name)
                        queueMap[queue.UID] = queue
                        queues.Push(queue)
                }

                if len(job.TaskStatusIndex[api.Pending]) != 0 {
                        if _, found := preemptorsMap[job.Queue]; !found {
                                preemptorsMap[job.Queue] = util.NewPriorityQueue(ssn.JobOrderFn)
                        }
                        preemptorsMap[job.Queue].Push(job)
                        preemptorTasks[job.UID] = util.NewPriorityQueue(ssn.TaskOrderFn)
                        for _, task := range job.TaskStatusIndex[api.Pending] {
                                preemptorTasks[job.UID].Push(task)
                        }
                }
        }

        for {
                // If no queues, break
                if queues.Empty() {
                        break
                }

                var job *api.JobInfo
                var task *api.TaskInfo

                queue := queues.Pop().(*api.QueueInfo)
                if ssn.Overused(queue) {
                        klog.V(3).Infof("Queue <%s> is overused, ignore it.", queue.Name)
                        continue
                }

                // Found "high" priority job
                jobs, found := preemptorsMap[queue.UID]
                if !found || jobs.Empty() {
                        continue
                } else {
                        job = jobs.Pop().(*api.JobInfo)
                }

                // Found "high" priority task to reclaim others
                if tasks, found := preemptorTasks[job.UID]; !found || tasks.Empty() {
                        continue
                } else {
                        task = tasks.Pop().(*api.TaskInfo)
                }

                // Check whether the queue is overused on dimension that the task requested
                taskRequest := task.Resreq.ResourceNames()
                if underusedResources := ssn.UnderusedResources(queue); underusedResources != nil && !underusedResources.Contains(taskRequest) {
                        klog.V(3).Infof("Queue <%s> is overused when considering task <%s>, ignore it.", queue.Name, task.Name)
                        continue
                }

                assigned := false
                for _, n := range ssn.Nodes {
                        // If predicates failed, next node.
                        if err := ssn.PredicateFn(task, n); err != nil {
                                continue
                        }

                        klog.V(3).Infof("Considering Task <%s/%s> on Node <%s>.",
                                task.Namespace, task.Name, n.Name)

                        var reclaimees []*api.TaskInfo
                        for _, task := range n.Tasks {
                                // Ignore non running task.
                                if task.Status != api.Running {
                                        continue
                                }

                                if j, found := ssn.Jobs[task.Job]; !found {
                                        continue
                                } else if j.Queue != job.Queue {
                                        q := ssn.Queues[j.Queue]
                                        if !q.Reclaimable() {
                                                continue
                                        }
                                        // Clone task to avoid modify Task's status on node.
                                        reclaimees = append(reclaimees, task.Clone())
                                }
                        }
                        victims := ssn.Reclaimable(task, reclaimees)

                        if err := util.ValidateVictims(task, n, victims); err != nil {
                                klog.V(3).Infof("No validated victims on Node <%s>: %v", n.Name, err)
                                continue
                        }

                        resreq := task.InitResreq.Clone()
                        reclaimed := api.EmptyResource()

                        // Reclaim victims for tasks.
                        for _, reclaimee := range victims {
                                klog.Errorf("Try to reclaim Task <%s/%s> for Tasks <%s/%s>",
                                        reclaimee.Namespace, reclaimee.Name, task.Namespace, task.Name)
                                if err := ssn.Evict(reclaimee, "reclaim"); err != nil {
                                        klog.Errorf("Failed to reclaim Task <%s/%s> for Tasks <%s/%s>: %v",
                                                reclaimee.Namespace, reclaimee.Name, task.Namespace, task.Name, err)
                                        continue
                                }
                                reclaimed.Add(reclaimee.Resreq)
                                // If reclaimed enough resources, break loop to avoid Sub panic.
                                if resreq.LessEqual(reclaimed, api.Zero) {
                                        break
                                }
                        }

                        klog.V(3).Infof("Reclaimed <%v> for task <%s/%s> requested <%v>.",
                                reclaimed, task.Namespace, task.Name, task.InitResreq)

                        if task.InitResreq.LessEqual(reclaimed, api.Zero) {
                                if err := ssn.Pipeline(task, n.Name); err != nil {
                                        klog.Errorf("Failed to pipeline Task <%s/%s> on Node <%s>",
                                                task.Namespace, task.Name, n.Name)
                                }

                                // Ignore error of pipeline, will be corrected in next scheduling loop.
                                assigned = true

                                break
                        }
                }

                if assigned {
                        jobs.Push(job)
                }
                queues.Push(queue)
        }
}

func (ra *Action) UnInitialize() {
}

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"
)

// ClusterInfo is a snapshot of cluster by cache.
type ClusterInfo struct {
        Jobs           map[JobID]*JobInfo
        Nodes          map[string]*NodeInfo
        Queues         map[QueueID]*QueueInfo
        NamespaceInfo  map[NamespaceName]*NamespaceInfo
        RevocableNodes map[string]*NodeInfo
        NodeList       []string
}

func (ci ClusterInfo) String() string {
        str := "Cache:\n"

        if len(ci.Nodes) != 0 {
                str += "Nodes:\n"
                for _, n := range ci.Nodes {
                        str += fmt.Sprintf("\t %s: idle(%v) used(%v) allocatable(%v) pods(%d)\n",
                                n.Name, n.Idle, n.Used, n.Allocatable, len(n.Tasks))

                        i := 0
                        for _, p := range n.Tasks {
                                str += fmt.Sprintf("\t\t %d: %v\n", i, p)
                                i++
                        }
                }
        }

        if len(ci.Jobs) != 0 {
                str += "Jobs:\n"
                for _, job := range ci.Jobs {
                        str += fmt.Sprintf("\t Job(%s) name(%s) minAvailable(%v)\n",
                                job.UID, job.Name, job.MinAvailable)

                        i := 0
                        for _, task := range job.Tasks {
                                str += fmt.Sprintf("\t\t %d: %v\n", i, task)
                                i++
                        }
                }
        }

        if len(ci.NamespaceInfo) != 0 {
                str += "Namespaces:\n"
                for _, ns := range ci.NamespaceInfo {
                        str += fmt.Sprintf("\t Namespace(%s) Weight(%v)\n",
                                ns.Name, ns.Weight)
                }
        }

        if len(ci.NodeList) != 0 {
                str += fmt.Sprintf("NodeList: %v\n", ci.NodeList)
        }

        return str
}

/*
Copyright 2020 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        v1 "k8s.io/api/core/v1"
)

// GPUDevice include gpu id, memory and the pods that are sharing it.
type GPUDevice struct {
        // GPU ID
        ID int
        // The pods that are sharing this GPU
        PodMap map[string]*v1.Pod
        // memory per card
        Memory uint
}

// NewGPUDevice creates a device
func NewGPUDevice(id int, mem uint) *GPUDevice {
        return &GPUDevice{
                ID:     id,
                Memory: mem,
                PodMap: map[string]*v1.Pod{},
        }
}

// getUsedGPUMemory calculates the used memory of the device.
func (g *GPUDevice) getUsedGPUMemory() uint {
        res := uint(0)
        for _, pod := range g.PodMap {
                if pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed {
                        continue
                } else {
                        gpuRequest := GetGPUResourceOfPod(pod)
                        res += gpuRequest
                }
        }
        return res
}

// GetGPUResourceOfPod returns the GPU resource required by the pod.
func GetGPUResourceOfPod(pod *v1.Pod) uint {
        var mem uint
        for _, container := range pod.Spec.Containers {
                mem += getGPUResourceOfContainer(&container)
        }
        return mem
}

// getGPUResourceOfPod returns the GPU resource required by the container.
func getGPUResourceOfContainer(container *v1.Container) uint {
        var mem uint
        if val, ok := container.Resources.Limits[VolcanoGPUResource]; ok {
                mem = uint(val.Value())
        }
        return mem
}

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        clientcache "k8s.io/client-go/tools/cache"
)

// PodKey returns the string key of a pod.
func PodKey(pod *v1.Pod) TaskID {
        key, err := clientcache.MetaNamespaceKeyFunc(pod)
        if err != nil {
                return TaskID(fmt.Sprintf("%v/%v", pod.Namespace, pod.Name))
        }
        return TaskID(key)
}

func getTaskStatus(pod *v1.Pod) TaskStatus {
        switch pod.Status.Phase {
        case v1.PodRunning:
                if pod.DeletionTimestamp != nil {
                        return Releasing
                }

                return Running
        case v1.PodPending:
                if pod.DeletionTimestamp != nil {
                        return Releasing
                }

                if len(pod.Spec.NodeName) == 0 {
                        return Pending
                }
                return Bound
        case v1.PodUnknown:
                return Unknown
        case v1.PodSucceeded:
                return Succeeded
        case v1.PodFailed:
                return Failed
        }

        return Unknown
}

// AllocatedStatus checks whether the tasks has AllocatedStatus
func AllocatedStatus(status TaskStatus) bool {
        switch status {
        case Bound, Binding, Running, Allocated:
                return true
        default:
                return false
        }
}

// MergeErrors is used to merge multiple errors into single error
func MergeErrors(errs ...error) error {
        msg := "errors: "

        foundErr := false
        i := 1

        for _, e := range errs {
                if e != nil {
                        if foundErr {
                                msg = fmt.Sprintf("%s, %d: ", msg, i)
                        } else {
                                msg = fmt.Sprintf("%s %d: ", msg, i)
                        }

                        msg = fmt.Sprintf("%s%v", msg, e)
                        foundErr = true
                        i++
                }
        }

        if foundErr {
                return fmt.Errorf("%s", msg)
        }

        return nil
}

// JobTerminated checks whether job was terminated.
func JobTerminated(job *JobInfo) bool {
        return job.PodGroup == nil && len(job.Tasks) == 0
}

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "errors"
        "fmt"
        "sort"
        "strconv"
        "strings"
        "time"

        v1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/klog"
        volumescheduling "k8s.io/kubernetes/pkg/controller/volume/scheduling"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

// DisruptionBudget define job min pod available and max pod unvailable value
type DisruptionBudget struct {
        MinAvailable  string
        MaxUnavilable string
}

// NewDisruptionBudget create disruption budget for job
func NewDisruptionBudget(minAvailable, maxUnavilable string) *DisruptionBudget {
        disruptionBudget := &DisruptionBudget{
                MinAvailable:  minAvailable,
                MaxUnavilable: maxUnavilable,
        }
        return disruptionBudget
}

// Clone return a clone of DisruptionBudget
func (db *DisruptionBudget) Clone() *DisruptionBudget {
        return &DisruptionBudget{
                MinAvailable:  db.MinAvailable,
                MaxUnavilable: db.MaxUnavilable,
        }
}

// JobWaitingTime is maximum waiting time that a job could stay Pending in service level agreement
// when job waits longer than waiting time, it should be inqueue at once, and cluster should reserve resources for it
const JobWaitingTime = "sla-waiting-time"

// TaskID is UID type for Task
type TaskID types.UID

// TransactionContext holds all the fields that needed by scheduling transaction
type TransactionContext struct {
        NodeName string
        Status   TaskStatus
}

// Clone return a clone of TransactionContext
func (ctx *TransactionContext) Clone() *TransactionContext {
        if ctx == nil {
                return nil
        }
        clone := *ctx
        return &clone
}

// TaskInfo will have all infos about the task
type TaskInfo struct {
        UID TaskID
        Job JobID

        Name      string
        Namespace string

        // Resreq is the resource that used when task running.
        Resreq *Resource
        // InitResreq is the resource that used to launch a task.
        InitResreq *Resource

        TransactionContext
        // LastTransaction holds the context of last scheduling transaction
        LastTransaction *TransactionContext

        Priority    int32
        VolumeReady bool
        Preemptable bool
        BestEffort  bool

        // RevocableZone support set volcano.sh/revocable-zone annotaion or label for pod/podgroup
        // we only support empty value or * value for this version and we will support specify revocable zone name for futrue release
        // empty value means workload can not use revocable node
        // * value means workload can use all the revocable node for during node active revocable time.
        RevocableZone string

        TopologyPolicy string
        PodVolumes     *volumescheduling.PodVolumes
        Pod            *v1.Pod
}

func getJobID(pod *v1.Pod) JobID {
        if gn, found := pod.Annotations[v1beta1.KubeGroupNameAnnotationKey]; found && len(gn) != 0 {
                // Make sure Pod and PodGroup belong to the same namespace.
                jobID := fmt.Sprintf("%s/%s", pod.Namespace, gn)
                return JobID(jobID)
        }

        return ""
}

func getTaskID(pod *v1.Pod) TaskID {
        if ts, found := pod.Annotations[batch.TaskSpecKey]; found && len(ts) != 0 {
                return TaskID(ts)
        }

        return ""
}

// NewTaskInfo creates new taskInfo object for a Pod
func NewTaskInfo(pod *v1.Pod) *TaskInfo {
        initResReq := GetPodResourceRequest(pod)
        resReq := initResReq
        bestEffort := initResReq.IsEmpty()
        preemptable := GetPodPreemptable(pod)
        revocableZone := GetPodRevocableZone(pod)
        topologyPolicy := GetPodTopologyPolicy(pod)

        jobID := getJobID(pod)

        ti := &TaskInfo{
                UID:            TaskID(pod.UID),
                Job:            jobID,
                Name:           pod.Name,
                Namespace:      pod.Namespace,
                Priority:       1,
                Pod:            pod,
                Resreq:         resReq,
                InitResreq:     initResReq,
                Preemptable:    preemptable,
                BestEffort:     bestEffort,
                RevocableZone:  revocableZone,
                TopologyPolicy: topologyPolicy,

                TransactionContext: TransactionContext{
                        NodeName: pod.Spec.NodeName,
                        Status:   getTaskStatus(pod),
                },
        }

        if pod.Spec.Priority != nil {
                ti.Priority = *pod.Spec.Priority
        }

        return ti
}

// GetTransactionContext get transaction context of a task
func (ti *TaskInfo) GetTransactionContext() TransactionContext {
        return ti.TransactionContext
}

// GenerateLastTxContext generate and set context of last transaction for a task
func (ti *TaskInfo) GenerateLastTxContext() {
        ctx := ti.GetTransactionContext()
        ti.LastTransaction = &ctx
}

// ClearLastTxContext clear context of last transaction for a task
func (ti *TaskInfo) ClearLastTxContext() {
        ti.LastTransaction = nil
}

// Clone is used for cloning a task
func (ti *TaskInfo) Clone() *TaskInfo {
        return &TaskInfo{
                UID:            ti.UID,
                Job:            ti.Job,
                Name:           ti.Name,
                Namespace:      ti.Namespace,
                Priority:       ti.Priority,
                PodVolumes:     ti.PodVolumes,
                Pod:            ti.Pod,
                Resreq:         ti.Resreq.Clone(),
                InitResreq:     ti.InitResreq.Clone(),
                VolumeReady:    ti.VolumeReady,
                Preemptable:    ti.Preemptable,
                BestEffort:     ti.BestEffort,
                RevocableZone:  ti.RevocableZone,
                TopologyPolicy: ti.TopologyPolicy,

                TransactionContext: TransactionContext{
                        NodeName: ti.NodeName,
                        Status:   ti.Status,
                },
                LastTransaction: ti.LastTransaction.Clone(),
        }
}

// String returns the taskInfo details in a string
func (ti TaskInfo) String() string {
        return fmt.Sprintf("Task (%v:%v/%v): job %v, status %v, pri %v"+
                "resreq %v, preemptable %v, revocableZone %v, TopologyPolicy %v",
                ti.UID, ti.Namespace, ti.Name, ti.Job, ti.Status, ti.Priority,
                ti.Resreq, ti.Preemptable, ti.RevocableZone, ti.TopologyPolicy)
}

// JobID is the type of JobInfo's ID.
type JobID types.UID

type tasksMap map[TaskID]*TaskInfo

// NodeResourceMap stores resource in a node
type NodeResourceMap map[string]*Resource

// JobInfo will have all info of a Job
type JobInfo struct {
        UID JobID

        Name      string
        Namespace string

        Queue QueueID

        Priority int32

        MinAvailable int32

        WaitingTime *time.Duration

        JobFitErrors   string
        NodesFitErrors map[TaskID]*FitErrors

        // All tasks of the Job.
        TaskStatusIndex       map[TaskStatus]tasksMap
        Tasks                 tasksMap
        TaskMinAvailable      map[TaskID]int32
        TaskMinAvailableTotal int32

        Allocated    *Resource
        TotalRequest *Resource

        CreationTimestamp metav1.Time
        PodGroup          *PodGroup

        ScheduleStartTimestamp metav1.Time

        Preemptable bool

        // RevocableZone support set volcano.sh/revocable-zone annotaion or label for pod/podgroup
        // we only support empty value or * value for this version and we will support specify revocable zone name for futrue release
        // empty value means workload can not use revocable node
        // * value means workload can use all the revocable node for during node active revocable time.
        RevocableZone string
        Budget        *DisruptionBudget
}

// NewJobInfo creates a new jobInfo for set of tasks
func NewJobInfo(uid JobID, tasks ...*TaskInfo) *JobInfo {
        job := &JobInfo{
                UID:              uid,
                MinAvailable:     0,
                NodesFitErrors:   make(map[TaskID]*FitErrors),
                Allocated:        EmptyResource(),
                TotalRequest:     EmptyResource(),
                TaskStatusIndex:  map[TaskStatus]tasksMap{},
                Tasks:            tasksMap{},
                TaskMinAvailable: map[TaskID]int32{},
        }

        for _, task := range tasks {
                job.AddTaskInfo(task)
        }

        return job
}

// UnsetPodGroup removes podGroup details from a job
func (ji *JobInfo) UnsetPodGroup() {
        ji.PodGroup = nil
}

// SetPodGroup sets podGroup details to a job
func (ji *JobInfo) SetPodGroup(pg *PodGroup) {
        ji.Name = pg.Name
        ji.Namespace = pg.Namespace
        ji.MinAvailable = pg.Spec.MinMember
        ji.Queue = QueueID(pg.Spec.Queue)
        ji.CreationTimestamp = pg.GetCreationTimestamp()

        var err error
        ji.WaitingTime, err = ji.extractWaitingTime(pg)
        if err != nil {
                klog.Warningf("Error occurs in parsing waiting time for job <%s/%s>, err: %s.",
                        pg.Namespace, pg.Name, err.Error())
                ji.WaitingTime = nil
        }

        ji.Preemptable = ji.extractPreemptable(pg)
        ji.RevocableZone = ji.extractRevocableZone(pg)
        ji.Budget = ji.extractBudget(pg)

        taskMinAvailableTotal := int32(0)
        for task, member := range pg.Spec.MinTaskMember {
                ji.TaskMinAvailable[TaskID(task)] = member
                taskMinAvailableTotal += member
        }
        ji.TaskMinAvailableTotal = taskMinAvailableTotal

        ji.PodGroup = pg
}

// extractWaitingTime reads sla waiting time for job from podgroup annotations
// TODO: should also read from given field in volcano job spec
func (ji *JobInfo) extractWaitingTime(pg *PodGroup) (*time.Duration, error) {
        if _, exist := pg.Annotations[JobWaitingTime]; !exist {
                return nil, nil
        }

        jobWaitingTime, err := time.ParseDuration(pg.Annotations[JobWaitingTime])
        if err != nil {
                return nil, err
        }

        if jobWaitingTime <= 0 {
                return nil, errors.New("invalid sla waiting time")
        }

        return &jobWaitingTime, nil
}

// extractPreemptable return volcano.sh/preemptable value for job
func (ji *JobInfo) extractPreemptable(pg *PodGroup) bool {
        // check annotaion first
        if len(pg.Annotations) > 0 {
                if value, found := pg.Annotations[v1beta1.PodPreemptable]; found {
                        b, err := strconv.ParseBool(value)
                        if err != nil {
                                klog.Warningf("invalid %s=%s", v1beta1.PodPreemptable, value)
                                return false
                        }
                        return b
                }
        }

        // it annotation does not exit, check label
        if len(pg.Labels) > 0 {
                if value, found := pg.Labels[v1beta1.PodPreemptable]; found {
                        b, err := strconv.ParseBool(value)
                        if err != nil {
                                klog.Warningf("invalid %s=%s", v1beta1.PodPreemptable, value)
                                return false
                        }
                        return b
                }
        }

        return false
}

// extractRevocableZone return volcano.sh/revocable-zone value for pod/podgroup
func (ji *JobInfo) extractRevocableZone(pg *PodGroup) string {
        // check annotaion first
        if len(pg.Annotations) > 0 {
                if value, found := pg.Annotations[v1beta1.RevocableZone]; found {
                        if value != "*" {
                                return ""
                        }
                        return value
                }

                if value, found := pg.Annotations[v1beta1.PodPreemptable]; found {
                        if b, err := strconv.ParseBool(value); err == nil && b {
                                return "*"
                        }
                }
        }

        return ""
}

// extractBudget return budget value for job
func (ji *JobInfo) extractBudget(pg *PodGroup) *DisruptionBudget {
        if len(pg.Annotations) > 0 {
                if value, found := pg.Annotations[v1beta1.JDBMinAvailable]; found {
                        return NewDisruptionBudget(value, "")
                } else if value, found := pg.Annotations[v1beta1.JDBMaxUnavailable]; found {
                        return NewDisruptionBudget("", value)
                }
        }

        return NewDisruptionBudget("", "")
}

// GetMinResources return the min resources of podgroup.
func (ji *JobInfo) GetMinResources() *Resource {
        if ji.PodGroup.Spec.MinResources == nil {
                return EmptyResource()
        }

        return NewResource(*ji.PodGroup.Spec.MinResources)
}

func (ji *JobInfo) addTaskIndex(ti *TaskInfo) {
        if _, found := ji.TaskStatusIndex[ti.Status]; !found {
                ji.TaskStatusIndex[ti.Status] = tasksMap{}
        }
        ji.TaskStatusIndex[ti.Status][ti.UID] = ti
}

// AddTaskInfo is used to add a task to a job
func (ji *JobInfo) AddTaskInfo(ti *TaskInfo) {
        ji.Tasks[ti.UID] = ti
        ji.addTaskIndex(ti)
        ji.TotalRequest.Add(ti.Resreq)
        if AllocatedStatus(ti.Status) {
                ji.Allocated.Add(ti.Resreq)
        }
}

// UpdateTaskStatus is used to update task's status in a job.
// If error occurs both task and job are guaranteed to be in the original state.
func (ji *JobInfo) UpdateTaskStatus(task *TaskInfo, status TaskStatus) error {
        if err := validateStatusUpdate(task.Status, status); err != nil {
                return err
        }

        // First remove the task (if exist) from the task list.
        if _, found := ji.Tasks[task.UID]; found {
                if err := ji.DeleteTaskInfo(task); err != nil {
                        return err
                }
        }

        // Update task's status to the target status once task addition is guaranteed to succeed.
        task.Status = status
        ji.AddTaskInfo(task)

        return nil
}

func (ji *JobInfo) deleteTaskIndex(ti *TaskInfo) {
        if tasks, found := ji.TaskStatusIndex[ti.Status]; found {
                delete(tasks, ti.UID)

                if len(tasks) == 0 {
                        delete(ji.TaskStatusIndex, ti.Status)
                }
        }
}

// DeleteTaskInfo is used to delete a task from a job
func (ji *JobInfo) DeleteTaskInfo(ti *TaskInfo) error {
        if task, found := ji.Tasks[ti.UID]; found {
                ji.TotalRequest.Sub(task.Resreq)
                if AllocatedStatus(task.Status) {
                        ji.Allocated.Sub(task.Resreq)
                }
                delete(ji.Tasks, task.UID)
                ji.deleteTaskIndex(task)
                return nil
        }

        return fmt.Errorf("failed to find task <%v/%v> in job <%v/%v>",
                ti.Namespace, ti.Name, ji.Namespace, ji.Name)
}

// Clone is used to clone a jobInfo object
func (ji *JobInfo) Clone() *JobInfo {
        info := &JobInfo{
                UID:       ji.UID,
                Name:      ji.Name,
                Namespace: ji.Namespace,
                Queue:     ji.Queue,
                Priority:  ji.Priority,

                MinAvailable:   ji.MinAvailable,
                WaitingTime:    ji.WaitingTime,
                JobFitErrors:   ji.JobFitErrors,
                NodesFitErrors: make(map[TaskID]*FitErrors),
                Allocated:      EmptyResource(),
                TotalRequest:   EmptyResource(),

                PodGroup: ji.PodGroup,

                TaskStatusIndex:       map[TaskStatus]tasksMap{},
                TaskMinAvailable:      ji.TaskMinAvailable,
                TaskMinAvailableTotal: ji.TaskMinAvailableTotal,
                Tasks:                 tasksMap{},
                Preemptable:           ji.Preemptable,
                RevocableZone:         ji.RevocableZone,
                Budget:                ji.Budget.Clone(),
        }

        ji.CreationTimestamp.DeepCopyInto(&info.CreationTimestamp)

        for _, task := range ji.Tasks {
                info.AddTaskInfo(task.Clone())
        }

        return info
}

// String returns a jobInfo object in string format
func (ji JobInfo) String() string {
        res := ""

        i := 0
        for _, task := range ji.Tasks {
                res += fmt.Sprintf("\n\t %d: %v", i, task)
                i++
        }

        return fmt.Sprintf("Job (%v): namespace %v (%v), name %v, minAvailable %d, podGroup %+v, preemptable %+v, revocableZone %+v, minAvailable %+v, maxAvailable %+v",
                ji.UID, ji.Namespace, ji.Queue, ji.Name, ji.MinAvailable, ji.PodGroup, ji.Preemptable, ji.RevocableZone, ji.Budget.MinAvailable, ji.Budget.MaxUnavilable) + res
}

// FitError returns detailed information on why a job's task failed to fit on
// each available node
func (ji *JobInfo) FitError() string {
        sortReasonsHistogram := func(reasons map[string]int) []string {
                reasonStrings := []string{}
                for k, v := range reasons {
                        reasonStrings = append(reasonStrings, fmt.Sprintf("%v %v", v, k))
                }
                sort.Strings(reasonStrings)
                return reasonStrings
        }

        // Stat histogram for all tasks of the job
        reasons := make(map[string]int)
        for status, taskMap := range ji.TaskStatusIndex {
                reasons[status.String()] += len(taskMap)
        }
        reasons["minAvailable"] = int(ji.MinAvailable)
        reasonMsg := fmt.Sprintf("%v, %v", scheduling.PodGroupNotReady, strings.Join(sortReasonsHistogram(reasons), ", "))

        // Stat histogram for pending tasks only
        reasons = make(map[string]int)
        for uid := range ji.TaskStatusIndex[Pending] {
                reason, _ := ji.TaskSchedulingReason(uid)
                reasons[reason]++
        }
        if len(reasons) > 0 {
                reasonMsg += "; " + fmt.Sprintf("%s: %s", Pending.String(), strings.Join(sortReasonsHistogram(reasons), ", "))
        }
        return reasonMsg
}

// TaskSchedulingReason get detailed reason and message of the given task
// It returns detailed reason and message for tasks based on last scheduling transaction.
func (ji *JobInfo) TaskSchedulingReason(tid TaskID) (reason string, msg string) {
        taskInfo, exists := ji.Tasks[tid]
        if !exists {
                return "", ""
        }

        // Get detailed scheduling reason based on LastTransaction
        ctx := taskInfo.GetTransactionContext()
        if taskInfo.LastTransaction != nil {
                ctx = *taskInfo.LastTransaction
        }

        msg = ji.JobFitErrors
        switch status := ctx.Status; status {
        case Allocated, Pipelined:
                // Pod is schedulable
                msg = fmt.Sprintf("Pod %s/%s can possibly be assigned to %s", taskInfo.Namespace, taskInfo.Name, ctx.NodeName)
                if status == Pipelined {
                        msg += " once resource is released"
                }
                return PodReasonSchedulable, msg
        case Pending:
                if fe := ji.NodesFitErrors[tid]; fe != nil {
                        // Pod is not schedulable
                        return PodReasonUnschedulable, fe.Error()
                }
                // Pod is not scheduled yet
                return PodReasonUndetermined, msg
        default:
                return status.String(), msg
        }
}

// ReadyTaskNum returns the number of tasks that are ready or that is best-effort.
func (ji *JobInfo) ReadyTaskNum() int32 {
        occupied := 0
        occupied += len(ji.TaskStatusIndex[Bound])
        occupied += len(ji.TaskStatusIndex[Binding])
        occupied += len(ji.TaskStatusIndex[Running])
        occupied += len(ji.TaskStatusIndex[Allocated])
        occupied += len(ji.TaskStatusIndex[Succeeded])

        if tasks, found := ji.TaskStatusIndex[Pending]; found {
                for _, task := range tasks {
                        if task.BestEffort {
                                occupied++
                        }
                }
        }

        return int32(occupied)
}

// WaitingTaskNum returns the number of tasks that are pipelined.
func (ji *JobInfo) WaitingTaskNum() int32 {
        return int32(len(ji.TaskStatusIndex[Pipelined]))
}

// CheckTaskMinAvailable returns whether each task of job is valid.
func (ji *JobInfo) CheckTaskMinAvailable() bool {
        // if job minAvailable is less than sumof(task minAvailable), skip this check
        if ji.MinAvailable < ji.TaskMinAvailableTotal {
                return true
        }

        actual := map[TaskID]int32{}
        for status, tasks := range ji.TaskStatusIndex {
                if AllocatedStatus(status) ||
                        status == Succeeded ||
                        status == Pipelined ||
                        status == Pending {
                        for _, task := range tasks {
                                actual[getTaskID(task.Pod)]++
                        }
                }
        }

        klog.V(4).Infof("job %s/%s actual: %+v, ji.TaskMinAvailable: %+v", ji.Name, ji.Namespace, actual, ji.TaskMinAvailable)
        for task, minAvailable := range ji.TaskMinAvailable {
                if act, ok := actual[task]; !ok || act < minAvailable {
                        return false
                }
        }

        return true
}

// ValidTaskNum returns the number of tasks that are valid.
func (ji *JobInfo) ValidTaskNum() int32 {
        occupied := 0
        for status, tasks := range ji.TaskStatusIndex {
                if AllocatedStatus(status) ||
                        status == Succeeded ||
                        status == Pipelined ||
                        status == Pending {
                        occupied += len(tasks)
                }
        }

        return int32(occupied)
}

// Ready returns whether job is ready for run
func (ji *JobInfo) Ready() bool {
        occupied := ji.ReadyTaskNum()

        return occupied >= ji.MinAvailable
}

// IsPending returns whether job is in pending status
func (ji *JobInfo) IsPending() bool {
        if ji.PodGroup == nil || ji.PodGroup.Status.Phase == scheduling.PodGroupPending || ji.PodGroup.Status.Phase == "" {
                return true
        }

        return false
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/klog"
)

// NamespaceName is name of namespace
type NamespaceName string

const (
        // NamespaceWeightKey is the key in ResourceQuota.spec.hard indicating the weight of this namespace
        NamespaceWeightKey = "volcano.sh/namespace.weight"
        // DefaultNamespaceWeight is the default weight of namespace
        DefaultNamespaceWeight = 1
)

// NamespaceInfo records information of namespace
type NamespaceInfo struct {
        // Name is the name of this namespace
        Name NamespaceName
        // Weight is the highest weight among many ResourceQuota.
        Weight int64
}

// GetWeight returns weight of a namespace, any invalid case would get default value
func (n *NamespaceInfo) GetWeight() int64 {
        if n == nil || n.Weight == 0 {
                return DefaultNamespaceWeight
        }
        return n.Weight
}

type quotaItem struct {
        name   string
        weight int64
}

func quotaItemKeyFunc(obj interface{}) (string, error) {
        item, ok := obj.(*quotaItem)
        if !ok {
                return "", fmt.Errorf("obj with type %T could not parse", obj)
        }
        return item.name, nil
}

// for big root heap
func quotaItemLessFunc(a interface{}, b interface{}) bool {
        A := a.(*quotaItem)
        B := b.(*quotaItem)
        return A.weight > B.weight
}

// NamespaceCollection will record all details about namespace
type NamespaceCollection struct {
        Name string

        quotaWeight *cache.Heap
}

// NewNamespaceCollection creates new NamespaceCollection object to record all information about a namespace
func NewNamespaceCollection(name string) *NamespaceCollection {
        n := &NamespaceCollection{
                Name:        name,
                quotaWeight: cache.NewHeap(quotaItemKeyFunc, quotaItemLessFunc),
        }
        // add at least one item into quotaWeight.
        // Because cache.Heap.Pop would be blocked until queue is not empty
        n.updateWeight(&quotaItem{
                name:   NamespaceWeightKey,
                weight: DefaultNamespaceWeight,
        })
        return n
}

func (n *NamespaceCollection) deleteWeight(q *quotaItem) {
        n.quotaWeight.Delete(q)
}

func (n *NamespaceCollection) updateWeight(q *quotaItem) {
        n.quotaWeight.Update(q)
}

func itemFromQuota(quota *v1.ResourceQuota) *quotaItem {
        var weight int64 = DefaultNamespaceWeight

        quotaWeight, ok := quota.Spec.Hard[NamespaceWeightKey]
        if ok {
                weight = quotaWeight.Value()
        }

        item := &quotaItem{
                name:   quota.Name,
                weight: weight,
        }
        return item
}

// Update modify the registered information according quota object
func (n *NamespaceCollection) Update(quota *v1.ResourceQuota) {
        n.updateWeight(itemFromQuota(quota))
}

// Delete remove the registered information according quota object
func (n *NamespaceCollection) Delete(quota *v1.ResourceQuota) {
        n.deleteWeight(itemFromQuota(quota))
}

// Snapshot will clone a NamespaceInfo without Heap according NamespaceCollection
func (n *NamespaceCollection) Snapshot() *NamespaceInfo {
        var weight int64 = DefaultNamespaceWeight

        obj, err := n.quotaWeight.Pop()
        if err != nil {
                klog.Warningf("namespace %s, quota weight meets error %v when pop", n.Name, err)
        } else {
                item := obj.(*quotaItem)
                weight = item.weight
                n.quotaWeight.Add(item)
        }

        return &NamespaceInfo{
                Name:   NamespaceName(n.Name),
                Weight: weight,
        }
}

/*
 Copyright 2021 The Volcano Authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package api

import (
        "fmt"
        "strconv"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

// NodeInfo is node level aggregated information.
type NodeInfo struct {
        Name string
        Node *v1.Node

        // The state of node
        State NodeState

        // The releasing resource on that node
        Releasing *Resource
        // The pipelined resource on that node
        Pipelined *Resource
        // The idle resource on that node
        Idle *Resource
        // The used resource on that node, including running and terminating
        // pods
        Used *Resource

        Allocatable *Resource
        Capability  *Resource

        Tasks             map[TaskID]*TaskInfo
        NumaInfo          *NumatopoInfo
        NumaChgFlag       NumaChgFlag
        NumaSchedulerInfo *NumatopoInfo
        RevocableZone     string

        // Used to store custom information
        Others     map[string]interface{}
        GPUDevices map[int]*GPUDevice

        // enable node resource oversubscription
        OversubscriptionNode bool
        // OfflineJobEvicting true means node resource usage too high then dispatched pod can not use oversubscription resource
        OfflineJobEvicting bool

        // Resource Oversubscription feature: the Oversubscription Resource reported in annotation
        OversubscriptionResource *Resource
}

// FutureIdle returns resources that will be idle in the future:
//
// That is current idle resources plus released resources minus pipelined resources.
func (ni *NodeInfo) FutureIdle() *Resource {
        return ni.Idle.Clone().Add(ni.Releasing).Sub(ni.Pipelined)
}

// GetNodeAllocatable return node Allocatable without OversubscriptionResource resource
func (ni *NodeInfo) GetNodeAllocatable() *Resource {
        return NewResource(ni.Node.Status.Allocatable)
}

// NodeState defines the current state of node.
type NodeState struct {
        Phase  NodePhase
        Reason string
}

// NewNodeInfo is used to create new nodeInfo object
func NewNodeInfo(node *v1.Node) *NodeInfo {
        nodeInfo := &NodeInfo{
                Releasing: EmptyResource(),
                Pipelined: EmptyResource(),
                Idle:      EmptyResource(),
                Used:      EmptyResource(),

                Allocatable: EmptyResource(),
                Capability:  EmptyResource(),

                OversubscriptionResource: EmptyResource(),
                Tasks:                    make(map[TaskID]*TaskInfo),

                GPUDevices: make(map[int]*GPUDevice),
        }

        nodeInfo.setOversubscription(node)

        if node != nil {
                nodeInfo.Name = node.Name
                nodeInfo.Node = node
                nodeInfo.Idle = NewResource(node.Status.Allocatable).Add(nodeInfo.OversubscriptionResource)
                nodeInfo.Allocatable = NewResource(node.Status.Allocatable).Add(nodeInfo.OversubscriptionResource)
                nodeInfo.Capability = NewResource(node.Status.Capacity).Add(nodeInfo.OversubscriptionResource)
        }
        nodeInfo.setNodeGPUInfo(node)
        nodeInfo.setNodeState(node)
        nodeInfo.setRevocableZone(node)

        return nodeInfo
}

// RefreshNumaSchedulerInfoByCrd used to update scheduler numa information based the CRD numatopo
func (ni *NodeInfo) RefreshNumaSchedulerInfoByCrd() {
        if ni.NumaInfo == nil {
                ni.NumaSchedulerInfo = nil
                return
        }

        tmp := ni.NumaInfo.DeepCopy()
        if ni.NumaChgFlag == NumaInfoMoreFlag {
                ni.NumaSchedulerInfo = tmp
        } else if ni.NumaChgFlag == NumaInfoLessFlag {
                numaResMap := ni.NumaSchedulerInfo.NumaResMap
                for resName, resInfo := range tmp.NumaResMap {
                        klog.V(5).Infof("resource %s Allocatable : current %v new %v on node %s",
                                resName, numaResMap[resName], resInfo, ni.Name)
                        if numaResMap[resName].Allocatable.Size() >= resInfo.Allocatable.Size() {
                                numaResMap[resName].Allocatable = resInfo.Allocatable.Clone()
                                numaResMap[resName].Capacity = resInfo.Capacity
                        }
                }
        }

        ni.NumaChgFlag = NumaInfoResetFlag
}

// Clone used to clone nodeInfo Object
func (ni *NodeInfo) Clone() *NodeInfo {
        res := NewNodeInfo(ni.Node)

        for _, p := range ni.Tasks {
                res.AddTask(p)
        }

        if ni.NumaSchedulerInfo != nil {
                res.NumaSchedulerInfo = ni.NumaSchedulerInfo.DeepCopy()
                klog.V(5).Infof("node[%s]", ni.Name)
                for resName, resInfo := range res.NumaSchedulerInfo.NumaResMap {
                        klog.V(5).Infof("current resource %s : %v", resName, resInfo)
                }

                klog.V(5).Infof("current Policies : %v", res.NumaSchedulerInfo.Policies)
        }

        res.Others = ni.Others
        return res
}

// Ready returns whether node is ready for scheduling
func (ni *NodeInfo) Ready() bool {
        return ni.State.Phase == Ready
}

func (ni *NodeInfo) setRevocableZone(node *v1.Node) {
        if node == nil {
                klog.Warningf("the argument node is null.")
                return
        }

        revocableZone := ""
        if len(node.Labels) > 0 {
                if value, found := node.Labels[v1beta1.RevocableZone]; found {
                        revocableZone = value
                }
        }
        ni.RevocableZone = revocableZone
}

// Check node if enable Oversubscription and set Oversubscription resources
// Only support oversubscription cpu and memory resource for this version
func (ni *NodeInfo) setOversubscription(node *v1.Node) {
        if node == nil {
                return
        }

        ni.OversubscriptionNode = false
        ni.OfflineJobEvicting = false
        if len(node.Labels) > 0 {
                if value, found := node.Labels[OversubscriptionNode]; found {
                        b, err := strconv.ParseBool(value)
                        if err == nil {
                                ni.OversubscriptionNode = b
                        } else {
                                ni.OversubscriptionNode = false
                        }
                        klog.V(5).Infof("Set node %s Oversubscription to %v", node.Name, ni.OversubscriptionNode)
                }
        }

        if len(node.Annotations) > 0 {
                if value, found := node.Annotations[OfflineJobEvicting]; found {
                        b, err := strconv.ParseBool(value)
                        if err == nil {
                                ni.OfflineJobEvicting = b
                        } else {
                                ni.OfflineJobEvicting = false
                        }
                        klog.V(5).Infof("Set node %s OfflineJobEvicting to %v", node.Name, ni.OfflineJobEvicting)
                }
                if value, found := node.Annotations[OversubscriptionCPU]; found {
                        ni.OversubscriptionResource.MilliCPU, _ = strconv.ParseFloat(value, 64)
                        klog.V(5).Infof("Set node %s Oversubscription CPU to %v", node.Name, ni.OversubscriptionResource.MilliCPU)
                }
                if value, found := node.Annotations[OversubscriptionMemory]; found {
                        ni.OversubscriptionResource.Memory, _ = strconv.ParseFloat(value, 64)
                        klog.V(5).Infof("Set node %s Oversubscription Memory to %v", node.Name, ni.OversubscriptionResource.Memory)
                }
        }
}

func (ni *NodeInfo) setNodeState(node *v1.Node) {
        // If node is nil, the node is un-initialized in cache
        if node == nil {
                ni.State = NodeState{
                        Phase:  NotReady,
                        Reason: "UnInitialized",
                }
                klog.Warningf("set the node %s status to %s for the reason UnInitialized.", node.Name, NotReady.String())
                return
        }

        // set NodeState according to resources
        if !ni.Used.LessEqual(ni.Allocatable, Zero) {
                ni.State = NodeState{
                        Phase:  NotReady,
                        Reason: "OutOfSync",
                }
                return
        }

        // If node not ready, e.g. power off
        for _, cond := range node.Status.Conditions {
                if cond.Type == v1.NodeReady && cond.Status != v1.ConditionTrue {
                        ni.State = NodeState{
                                Phase:  NotReady,
                                Reason: "NotReady",
                        }
                        klog.Warningf("set the node %s status to %s.", node.Name, NotReady.String())
                        return
                }
        }

        // Node is ready (ignore node conditions because of taint/toleration)
        ni.State = NodeState{
                Phase:  Ready,
                Reason: "",
        }

        klog.V(4).Infof("set the node %s status to %s.", node.Name, Ready.String())
}

func (ni *NodeInfo) setNodeGPUInfo(node *v1.Node) {
        if node == nil {
                return
        }
        memory, ok := node.Status.Capacity[VolcanoGPUResource]
        if !ok {
                return
        }
        totalMemory := memory.Value()

        res, ok := node.Status.Capacity[VolcanoGPUNumber]
        if !ok {
                return
        }
        gpuNumber := res.Value()
        if gpuNumber == 0 {
                klog.Warningf("invalid %s=%s", VolcanoGPUNumber, res.String())
                return
        }

        memoryPerCard := uint(totalMemory / gpuNumber)
        for i := 0; i < int(gpuNumber); i++ {
                ni.GPUDevices[i] = NewGPUDevice(i, memoryPerCard)
        }
}

// SetNode sets kubernetes node object to nodeInfo object
func (ni *NodeInfo) SetNode(node *v1.Node) {
        ni.setOversubscription(node)
        ni.setNodeState(node)
        ni.setNodeGPUInfo(node)
        ni.setRevocableZone(node)

        if !ni.Ready() {
                klog.Warningf("Failed to set node info, phase: %s, reason: %s",
                        ni.State.Phase, ni.State.Reason)
                return
        }

        ni.Name = node.Name
        ni.Node = node

        ni.Allocatable = NewResource(node.Status.Allocatable).Add(ni.OversubscriptionResource)
        ni.Capability = NewResource(node.Status.Capacity).Add(ni.OversubscriptionResource)
        ni.Releasing = EmptyResource()
        ni.Pipelined = EmptyResource()
        ni.Idle = NewResource(node.Status.Allocatable).Add(ni.OversubscriptionResource)
        ni.Used = EmptyResource()

        for _, ti := range ni.Tasks {
                switch ti.Status {
                case Releasing:
                        ni.Idle.Sub(ti.Resreq)
                        ni.Releasing.Add(ti.Resreq)
                        ni.Used.Add(ti.Resreq)
                        ni.AddGPUResource(ti.Pod)
                case Pipelined:
                        ni.Pipelined.Add(ti.Resreq)
                default:
                        ni.Idle.Sub(ti.Resreq)
                        ni.Used.Add(ti.Resreq)
                        ni.AddGPUResource(ti.Pod)
                }
        }
}

func (ni *NodeInfo) allocateIdleResource(ti *TaskInfo) error {
        if ti.Resreq.LessEqual(ni.Idle, Zero) {
                ni.Idle.Sub(ti.Resreq)
                return nil
        }

        return fmt.Errorf("selected node NotReady")
}

// AddTask is used to add a task in nodeInfo object
//
// If error occurs both task and node are guaranteed to be in the original state.
func (ni *NodeInfo) AddTask(task *TaskInfo) error {
        if len(task.NodeName) > 0 && len(ni.Name) > 0 && task.NodeName != ni.Name {
                return fmt.Errorf("task <%v/%v> already on different node <%v>",
                        task.Namespace, task.Name, task.NodeName)
        }

        key := PodKey(task.Pod)
        if _, found := ni.Tasks[key]; found {
                return fmt.Errorf("task <%v/%v> already on node <%v>",
                        task.Namespace, task.Name, ni.Name)
        }

        // Node will hold a copy of task to make sure the status
        // change will not impact resource in node.
        ti := task.Clone()

        if ni.Node != nil {
                switch ti.Status {
                case Releasing:
                        if err := ni.allocateIdleResource(ti); err != nil {
                                return err
                        }
                        ni.Releasing.Add(ti.Resreq)
                        ni.Used.Add(ti.Resreq)
                        ni.AddGPUResource(ti.Pod)
                case Pipelined:
                        ni.Pipelined.Add(ti.Resreq)
                default:
                        if err := ni.allocateIdleResource(ti); err != nil {
                                return err
                        }
                        ni.Used.Add(ti.Resreq)
                        ni.AddGPUResource(ti.Pod)
                }
        }

        // Update task node name upon successful task addition.
        task.NodeName = ni.Name
        ti.NodeName = ni.Name
        ni.Tasks[key] = ti

        return nil
}

// RemoveTask used to remove a task from nodeInfo object.
//
// If error occurs both task and node are guaranteed to be in the original state.
func (ni *NodeInfo) RemoveTask(ti *TaskInfo) error {
        key := PodKey(ti.Pod)

        task, found := ni.Tasks[key]
        if !found {
                klog.Warningf("failed to find task <%v/%v> on host <%v>",
                        ti.Namespace, ti.Name, ni.Name)
                return nil
        }

        if ni.Node != nil {
                switch task.Status {
                case Releasing:
                        ni.Releasing.Sub(task.Resreq)
                        ni.Idle.Add(task.Resreq)
                        ni.Used.Sub(task.Resreq)
                        ni.SubGPUResource(ti.Pod)
                case Pipelined:
                        ni.Pipelined.Sub(task.Resreq)
                default:
                        ni.Idle.Add(task.Resreq)
                        ni.Used.Sub(task.Resreq)
                        ni.SubGPUResource(ti.Pod)
                }
        }

        delete(ni.Tasks, key)

        return nil
}

// UpdateTask is used to update a task in nodeInfo object.
//
// If error occurs both task and node are guaranteed to be in the original state.
func (ni *NodeInfo) UpdateTask(ti *TaskInfo) error {
        if err := ni.RemoveTask(ti); err != nil {
                return err
        }

        if err := ni.AddTask(ti); err != nil {
                // This should never happen if task removal was successful,
                // because only possible error during task addition is when task is still on a node.
                klog.Fatalf("Failed to add Task <%s,%s> to Node <%s> during task update",
                        ti.Namespace, ti.Name, ni.Name)
        }
        return nil
}

// String returns nodeInfo details in string format
func (ni NodeInfo) String() string {
        tasks := ""

        i := 0
        for _, task := range ni.Tasks {
                tasks += fmt.Sprintf("\n\t %d: %v", i, task)
                i++
        }

        return fmt.Sprintf("Node (%s): allocatable<%v> idle <%v>, used <%v>, releasing <%v>, oversubscribution <%v>, "+
                "state <phase %s, reaseon %s>, oversubscributionNode <%v>, offlineJobEvicting <%v>,taints <%v>%s",
                ni.Name, ni.Allocatable, ni.Idle, ni.Used, ni.Releasing, ni.OversubscriptionResource, ni.State.Phase, ni.State.Reason, ni.OversubscriptionNode, ni.OfflineJobEvicting, ni.Node.Spec.Taints, tasks)
}

// Pods returns all pods running in that node
func (ni *NodeInfo) Pods() (pods []*v1.Pod) {
        for _, t := range ni.Tasks {
                pods = append(pods, t.Pod)
        }

        return
}

// GetDevicesIdleGPUMemory returns all the idle GPU memory by gpu card.
func (ni *NodeInfo) GetDevicesIdleGPUMemory() map[int]uint {
        devicesAllGPUMemory := ni.getDevicesAllGPUMemory()
        devicesUsedGPUMemory := ni.getDevicesUsedGPUMemory()
        res := map[int]uint{}
        for id, allMemory := range devicesAllGPUMemory {
                if usedMemory, found := devicesUsedGPUMemory[id]; found {
                        res[id] = allMemory - usedMemory
                } else {
                        res[id] = allMemory
                }
        }
        return res
}

func (ni *NodeInfo) getDevicesUsedGPUMemory() map[int]uint {
        res := map[int]uint{}
        for _, device := range ni.GPUDevices {
                res[device.ID] = device.getUsedGPUMemory()
        }
        return res
}

func (ni *NodeInfo) getDevicesAllGPUMemory() map[int]uint {
        res := map[int]uint{}
        for _, device := range ni.GPUDevices {
                res[device.ID] = device.Memory
        }
        return res
}

// AddGPUResource adds the pod to GPU pool if it is assigned
func (ni *NodeInfo) AddGPUResource(pod *v1.Pod) {
        gpuRes := GetGPUResourceOfPod(pod)
        if gpuRes > 0 {
                id := GetGPUIndex(pod)
                if dev := ni.GPUDevices[id]; dev != nil {
                        dev.PodMap[string(pod.UID)] = pod
                }
        }
}

// SubGPUResource frees the gpu hold by the pod
func (ni *NodeInfo) SubGPUResource(pod *v1.Pod) {
        gpuRes := GetGPUResourceOfPod(pod)
        if gpuRes > 0 {
                id := GetGPUIndex(pod)
                if dev := ni.GPUDevices[id]; dev != nil {
                        delete(dev.PodMap, string(pod.UID))
                }
        }
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        v1 "k8s.io/api/core/v1"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"

        nodeinfov1alpha1 "volcano.sh/apis/pkg/apis/nodeinfo/v1alpha1"
)

// NumaChgFlag indicate node numainfo changed status
type NumaChgFlag int

const (
        // NumaInfoResetFlag indicate reset operate
        NumaInfoResetFlag NumaChgFlag = 0b00
        // NumaInfoMoreFlag indicate the received allocatable resource is getting more
        NumaInfoMoreFlag NumaChgFlag = 0b11
        // NumaInfoLessFlag indicate the received allocatable resource is getting less
        NumaInfoLessFlag NumaChgFlag = 0b10
)

// ResourceInfo is the allocatable information for the resource
type ResourceInfo struct {
        Allocatable cpuset.CPUSet
        Capacity    int
}

// NumatopoInfo is the information about topology manager on the node
type NumatopoInfo struct {
        Namespace   string
        Name        string
        Policies    map[nodeinfov1alpha1.PolicyName]string
        NumaResMap  map[string]*ResourceInfo
        CPUDetail   topology.CPUDetails
        ResReserved v1.ResourceList
}

// DeepCopy used to copy NumatopoInfo
func (info *NumatopoInfo) DeepCopy() *NumatopoInfo {
        numaInfo := &NumatopoInfo{
                Namespace:   info.Namespace,
                Name:        info.Name,
                Policies:    make(map[nodeinfov1alpha1.PolicyName]string),
                NumaResMap:  make(map[string]*ResourceInfo),
                CPUDetail:   topology.CPUDetails{},
                ResReserved: make(v1.ResourceList),
        }

        policies := info.Policies
        for name, policy := range policies {
                numaInfo.Policies[name] = policy
        }

        for resName, resInfo := range info.NumaResMap {
                var tmpInfo ResourceInfo
                tmpInfo.Capacity = resInfo.Capacity
                tmpInfo.Allocatable = resInfo.Allocatable.Clone()
                numaInfo.NumaResMap[resName] = &tmpInfo
        }

        cpuDetail := info.CPUDetail
        for cpuID, detail := range cpuDetail {
                numaInfo.CPUDetail[cpuID] = detail
        }

        resReserved := info.ResReserved
        for resName, res := range resReserved {
                numaInfo.ResReserved[resName] = res
        }

        return numaInfo
}

// Compare is the function to show the change of the resource on kubelet
// return val:
// - true : the resource on kubelet is getting more or no change
// - false :  the resource on kubelet is getting less
func (info *NumatopoInfo) Compare(newInfo *NumatopoInfo) bool {
        for resName := range info.NumaResMap {
                oldSize := info.NumaResMap[resName].Allocatable.Size()
                newSize := newInfo.NumaResMap[resName].Allocatable.Size()
                if oldSize <= newSize {
                        return true
                }
        }

        return false
}

// Allocate is the function to remove the allocated resource
func (info *NumatopoInfo) Allocate(resSets ResNumaSets) {
        for resName := range resSets {
                info.NumaResMap[resName].Allocatable = info.NumaResMap[resName].Allocatable.Difference(resSets[resName])
        }
}

// Release is the function to reclaim the allocated resource
func (info *NumatopoInfo) Release(resSets ResNumaSets) {
        for resName := range resSets {
                info.NumaResMap[resName].Allocatable = info.NumaResMap[resName].Allocatable.Union(resSets[resName])
        }
}

// GenerateNodeResNumaSets return the idle resource sets of all node
func GenerateNodeResNumaSets(nodes map[string]*NodeInfo) map[string]ResNumaSets {
        nodeSlice := make(map[string]ResNumaSets)
        for _, node := range nodes {
                if node.NumaSchedulerInfo == nil {
                        continue
                }

                resMaps := make(ResNumaSets)
                for resName, resMap := range node.NumaSchedulerInfo.NumaResMap {
                        resMaps[resName] = resMap.Allocatable.Clone()
                }

                nodeSlice[node.Name] = resMaps
        }

        return nodeSlice
}

// GenerateNumaNodes return the numa IDs of all node
func GenerateNumaNodes(nodes map[string]*NodeInfo) map[string][]int {
        nodeNumaMap := make(map[string][]int)

        for _, node := range nodes {
                if node.NumaSchedulerInfo == nil {
                        continue
                }

                nodeNumaMap[node.Name] = node.NumaSchedulerInfo.CPUDetail.NUMANodes().ToSlice()
        }

        return nodeNumaMap
}

// ResNumaSets is the set map of the resource
type ResNumaSets map[string]cpuset.CPUSet

// Allocate is to remove the allocated resource which is assigned to task
func (resSets ResNumaSets) Allocate(taskSets ResNumaSets) {
        for resName := range taskSets {
                if _, ok := resSets[resName]; !ok {
                        continue
                }
                resSets[resName] = resSets[resName].Difference(taskSets[resName])
        }
}

// Release is to reclaim the allocated resource which is assigned to task
func (resSets ResNumaSets) Release(taskSets ResNumaSets) {
        for resName := range taskSets {
                if _, ok := resSets[resName]; !ok {
                        continue
                }
                resSets[resName] = resSets[resName].Union(taskSets[resName])
        }
}

// Clone is the copy action
func (resSets ResNumaSets) Clone() ResNumaSets {
        newSets := make(ResNumaSets)
        for resName := range resSets {
                newSets[resName] = resSets[resName].Clone()
        }

        return newSets
}

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"
        "strconv"
        "strings"
        "time"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

// Refer k8s.io/kubernetes/pkg/scheduler/algorithm/predicates/predicates.go#GetResourceRequest.
//
// GetResourceRequest returns a *Resource that covers the largest width in each resource dimension.
// Because init-containers run sequentially, we collect the max in each dimension iteratively.
// In contrast, we sum the resource vectors for regular containers since they run simultaneously.
//
// To be consistent with kubernetes default scheduler, it is only used for predicates of actions(e.g.
// allocate, backfill, preempt, reclaim), please use GetPodResourceWithoutInitContainers for other cases.
//
// Example:
//
// Pod:
//   InitContainers
//     IC1:
//       CPU: 2
//       Memory: 1G
//     IC2:
//       CPU: 2
//       Memory: 3G
//   Containers
//     C1:
//       CPU: 2
//       Memory: 1G
//     C2:
//       CPU: 1
//       Memory: 1G
//
// Result: CPU: 3, Memory: 3G

// GetPodResourceRequest returns all the resource required for that pod
func GetPodResourceRequest(pod *v1.Pod) *Resource {
        result := GetPodResourceWithoutInitContainers(pod)

        // take max_resource(sum_pod, any_init_container)
        for _, container := range pod.Spec.InitContainers {
                result.SetMaxResource(NewResource(container.Resources.Requests))
        }

        return result
}

// GetPodPreemptable return volcano.sh/preemptable value for pod
func GetPodPreemptable(pod *v1.Pod) bool {
        // check annotaion first
        if len(pod.Annotations) > 0 {
                if value, found := pod.Annotations[v1beta1.PodPreemptable]; found {
                        b, err := strconv.ParseBool(value)
                        if err != nil {
                                klog.Warningf("invalid %s=%s", v1beta1.PodPreemptable, value)
                                return false
                        }
                        return b
                }
        }

        // it annotation does not exit, check label
        if len(pod.Labels) > 0 {
                if value, found := pod.Labels[v1beta1.PodPreemptable]; found {
                        b, err := strconv.ParseBool(value)
                        if err != nil {
                                klog.Warningf("invalid %s=%s", v1beta1.PodPreemptable, value)
                                return false
                        }
                        return b
                }
        }

        return false
}

// GetPodRevocableZone return volcano.sh/revocable-zone value for pod/podgroup
func GetPodRevocableZone(pod *v1.Pod) string {
        if len(pod.Annotations) > 0 {
                if value, found := pod.Annotations[v1beta1.RevocableZone]; found {
                        if value != "*" {
                                return ""
                        }
                        return value
                }

                if value, found := pod.Annotations[v1beta1.PodPreemptable]; found {
                        if b, err := strconv.ParseBool(value); err == nil && b {
                                return "*"
                        }
                }
        }
        return ""
}

// GetPodTopologyPolicy return volcano.sh/numa-topology-policy value for pod
func GetPodTopologyPolicy(pod *v1.Pod) string {
        if len(pod.Annotations) > 0 {
                if value, found := pod.Annotations[v1beta1.NumaPolicyKey]; found {
                        return value
                }
        }
        return ""
}

// GetPodResourceWithoutInitContainers returns Pod's resource request, it does not contain
// init containers' resource request.
func GetPodResourceWithoutInitContainers(pod *v1.Pod) *Resource {
        result := EmptyResource()
        for _, container := range pod.Spec.Containers {
                result.Add(NewResource(container.Resources.Requests))
        }

        return result
}

// GetGPUIndex returns the ID of the GPU
func GetGPUIndex(pod *v1.Pod) int {
        if len(pod.Annotations) > 0 {
                value, found := pod.Annotations[GPUIndex]
                if found {
                        id, err := strconv.Atoi(value)
                        if err != nil {
                                klog.Errorf("invalid %s=%s", GPUIndex, value)
                                return -1
                        }
                        return id
                }
        }

        return -1
}

func escapeJSONPointer(p string) string {
        // Escaping reference name using https://tools.ietf.org/html/rfc6901
        p = strings.Replace(p, "~", "~0", -1)
        p = strings.Replace(p, "/", "~1", -1)
        return p
}

// AddGPUIndexPatch returns the patch adding GPU index
func AddGPUIndexPatch(id int) string {
        return fmt.Sprintf(`[{"op": "add", "path": "/metadata/annotations/%s", "value":"%d"},`+
                `{"op": "add", "path": "/metadata/annotations/%s", "value": "%d"}]`,
                escapeJSONPointer(PredicateTime), time.Now().UnixNano(),
                escapeJSONPointer(GPUIndex), id)
}

// RemoveGPUIndexPatch returns the patch removing GPU index
func RemoveGPUIndexPatch() string {
        return fmt.Sprintf(`[{"op": "remove", "path": "/metadata/annotations/%s"},`+
                `{"op": "remove", "path": "/metadata/annotations/%s"]`, escapeJSONPointer(PredicateTime), escapeJSONPointer(GPUIndex))
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "k8s.io/apimachinery/pkg/types"

        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

// QueueID is UID type, serves as unique ID for each queue
type QueueID types.UID

// QueueInfo will have all details about queue
type QueueInfo struct {
        UID  QueueID
        Name string

        Weight int32

        // Weights is a list of slash sperated float numbers.
        // Each of them is a weight corresponding the
        // hierarchy level.
        Weights string
        // Hierarchy is a list of node name along the
        // path from the root to the node itself.
        Hierarchy string

        Queue *scheduling.Queue
}

// NewQueueInfo creates new queueInfo object
func NewQueueInfo(queue *scheduling.Queue) *QueueInfo {
        return &QueueInfo{
                UID:  QueueID(queue.Name),
                Name: queue.Name,

                Weight:    queue.Spec.Weight,
                Hierarchy: queue.Annotations[v1beta1.KubeHierarchyAnnotationKey],
                Weights:   queue.Annotations[v1beta1.KubeHierarchyWeightAnnotationKey],

                Queue: queue,
        }
}

// Clone is used to clone queueInfo object
func (q *QueueInfo) Clone() *QueueInfo {
        return &QueueInfo{
                UID:       q.UID,
                Name:      q.Name,
                Weight:    q.Weight,
                Hierarchy: q.Hierarchy,
                Weights:   q.Weights,
                Queue:     q.Queue,
        }
}

// Reclaimable return whether queue is reclaimable
func (q *QueueInfo) Reclaimable() bool {
        if q == nil {
                return false
        }

        if q.Queue == nil {
                return false
        }

        if q.Queue.Spec.Reclaimable == nil {
                return true
        }

        return *q.Queue.Spec.Reclaimable
}

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"
        "math"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/resource"
        v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"

        "volcano.sh/volcano/pkg/scheduler/util/assert"
)

const (
        // GPUResourceName need to follow https://github.com/NVIDIA/k8s-device-plugin/blob/66a35b71ac4b5cbfb04714678b548bd77e5ba719/server.go#L20
        GPUResourceName = "nvidia.com/gpu"
)

const (
        minResource float64 = 0.1
)

// DimensionDefaultValue means default value for black resource dimension
type DimensionDefaultValue string

const (
        // Zero means resource dimension not defined will be treated as zero
        Zero DimensionDefaultValue = "Zero"
        // Infinity means resource dimension not defined will be treated as infinity
        Infinity DimensionDefaultValue = "Infinity"
)

// Resource struct defines all the resource type
type Resource struct {
        MilliCPU float64
        Memory   float64

        // ScalarResources
        ScalarResources map[v1.ResourceName]float64

        // MaxTaskNum is only used by predicates; it should NOT
        // be accounted in other operators, e.g. Add.
        MaxTaskNum int
}

// EmptyResource creates a empty resource object and returns
func EmptyResource() *Resource {
        return &Resource{}
}

// NewResource creates a new resource object from resource list
func NewResource(rl v1.ResourceList) *Resource {
        r := EmptyResource()
        for rName, rQuant := range rl {
                switch rName {
                case v1.ResourceCPU:
                        r.MilliCPU += float64(rQuant.MilliValue())
                case v1.ResourceMemory:
                        r.Memory += float64(rQuant.Value())
                case v1.ResourcePods:
                        r.MaxTaskNum += int(rQuant.Value())
                default:
                        //NOTE: When converting this back to k8s resource, we need record the format as well as / 1000
                        if v1helper.IsScalarResourceName(rName) {
                                r.AddScalar(rName, float64(rQuant.MilliValue()))
                        }
                }
        }
        return r
}

// Clone is used to clone a resource type, which is a deep copy function.
func (r *Resource) Clone() *Resource {
        clone := &Resource{
                MilliCPU:   r.MilliCPU,
                Memory:     r.Memory,
                MaxTaskNum: r.MaxTaskNum,
        }

        if r.ScalarResources != nil {
                clone.ScalarResources = make(map[v1.ResourceName]float64)
                for k, v := range r.ScalarResources {
                        clone.ScalarResources[k] = v
                }
        }

        return clone
}

// String returns resource details in string format
func (r *Resource) String() string {
        str := fmt.Sprintf("cpu %0.2f, memory %0.2f", r.MilliCPU, r.Memory)
        for rName, rQuant := range r.ScalarResources {
                str = fmt.Sprintf("%s, %s %0.2f", str, rName, rQuant)
        }
        return str
}

// ResourceNames returns all resource types
func (r *Resource) ResourceNames() ResourceNameList {
        resNames := ResourceNameList{}

        if r.MilliCPU >= minResource {
                resNames = append(resNames, v1.ResourceCPU)
        }

        if r.Memory >= minResource {
                resNames = append(resNames, v1.ResourceMemory)
        }

        for rName, rMount := range r.ScalarResources {
                if rMount >= minResource {
                        resNames = append(resNames, rName)
                }
        }

        return resNames
}

// Get returns the resource value for that particular resource type
func (r *Resource) Get(rn v1.ResourceName) float64 {
        switch rn {
        case v1.ResourceCPU:
                return r.MilliCPU
        case v1.ResourceMemory:
                return r.Memory
        default:
                if r.ScalarResources == nil {
                        return 0
                }
                return r.ScalarResources[rn]
        }
}

// IsEmpty returns false if any kind of resource is not less than min value, otherwise returns true
func (r *Resource) IsEmpty() bool {
        if !(r.MilliCPU < minResource && r.Memory < minResource) {
                return false
        }

        for _, rQuant := range r.ScalarResources {
                if rQuant >= minResource {
                        return false
                }
        }

        return true
}

// IsZero returns false if the given kind of resource is not less than min value
func (r *Resource) IsZero(rn v1.ResourceName) bool {
        switch rn {
        case v1.ResourceCPU:
                return r.MilliCPU < minResource
        case v1.ResourceMemory:
                return r.Memory < minResource
        default:
                if r.ScalarResources == nil {
                        return true
                }

                _, found := r.ScalarResources[rn]
                assert.Assertf(found, "unknown resource %s", rn)

                return r.ScalarResources[rn] < minResource
        }
}

// Add is used to add two given resources
func (r *Resource) Add(rr *Resource) *Resource {
        r.MilliCPU += rr.MilliCPU
        r.Memory += rr.Memory

        for rName, rQuant := range rr.ScalarResources {
                if r.ScalarResources == nil {
                        r.ScalarResources = map[v1.ResourceName]float64{}
                }
                r.ScalarResources[rName] += rQuant
        }

        return r
}

//Sub subtracts two Resource objects.
func (r *Resource) Sub(rr *Resource) *Resource {
        assert.Assertf(rr.LessEqual(r, Zero), "resource is not sufficient to do operation: <%v> sub <%v>", r, rr)

        r.MilliCPU -= rr.MilliCPU
        r.Memory -= rr.Memory

        if r.ScalarResources == nil {
                return r
        }
        for rrName, rrQuant := range rr.ScalarResources {
                r.ScalarResources[rrName] -= rrQuant
        }

        return r
}

// Multi multiples the resource with ratio provided
func (r *Resource) Multi(ratio float64) *Resource {
        r.MilliCPU *= ratio
        r.Memory *= ratio
        for rName, rQuant := range r.ScalarResources {
                r.ScalarResources[rName] = rQuant * ratio
        }
        return r
}

// SetMaxResource compares with ResourceList and takes max value for each Resource.
func (r *Resource) SetMaxResource(rr *Resource) {
        if r == nil || rr == nil {
                return
        }

        if rr.MilliCPU > r.MilliCPU {
                r.MilliCPU = rr.MilliCPU
        }
        if rr.Memory > r.Memory {
                r.Memory = rr.Memory
        }

        for rrName, rrQuant := range rr.ScalarResources {
                if r.ScalarResources == nil {
                        r.ScalarResources = make(map[v1.ResourceName]float64)
                        for k, v := range rr.ScalarResources {
                                r.ScalarResources[k] = v
                        }
                        return
                }
                _, ok := r.ScalarResources[rrName]
                if !ok || rrQuant > r.ScalarResources[rrName] {
                        r.ScalarResources[rrName] = rrQuant
                }
        }
}

//FitDelta Computes the delta between a resource object representing available
//resources an operand representing resources being requested.  Any
//field that is less than 0 after the operation represents an
//insufficient resource.
func (r *Resource) FitDelta(rr *Resource) *Resource {
        if rr.MilliCPU > 0 {
                r.MilliCPU -= rr.MilliCPU + minResource
        }

        if rr.Memory > 0 {
                r.Memory -= rr.Memory + minResource
        }

        if r.ScalarResources == nil {
                r.ScalarResources = make(map[v1.ResourceName]float64)
        }

        for rrName, rrQuant := range rr.ScalarResources {
                if rrQuant > 0 {
                        _, ok := r.ScalarResources[rrName]
                        if !ok {
                                r.ScalarResources[rrName] = 0
                        }
                        r.ScalarResources[rrName] -= rrQuant + minResource
                }
        }

        return r
}

// Less returns true only on condition that all dimensions of resources in r are less than that of rr,
// Otherwise returns false.
// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
func (r *Resource) Less(rr *Resource, defaultValue DimensionDefaultValue) bool {
        lessFunc := func(l, r float64) bool {
                return l < r
        }

        leftResource := r.Clone()
        rightResource := rr.Clone()

        if !lessFunc(leftResource.MilliCPU, rightResource.MilliCPU) {
                return false
        }
        if !lessFunc(leftResource.Memory, rightResource.Memory) {
                return false
        }

        r.setDefaultValue(leftResource, rightResource, defaultValue)

        for resourceName, leftValue := range leftResource.ScalarResources {
                rightValue := rightResource.ScalarResources[resourceName]
                if rightValue == -1 {
                        continue
                }
                if leftValue == -1 || !lessFunc(leftValue, rightValue) {
                        return false
                }
        }
        return true
}

// LessEqual returns true only on condition that all dimensions of resources in r are less than or equal with that of rr,
// Otherwise returns false.
// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
func (r *Resource) LessEqual(rr *Resource, defaultValue DimensionDefaultValue) bool {
        lessEqualFunc := func(l, r, diff float64) bool {
                if l < r || math.Abs(l-r) < diff {
                        return true
                }
                return false
        }

        leftResource := r.Clone()
        rightResource := rr.Clone()

        if !lessEqualFunc(leftResource.MilliCPU, rightResource.MilliCPU, minResource) {
                return false
        }
        if !lessEqualFunc(leftResource.Memory, rightResource.Memory, minResource) {
                return false
        }

        r.setDefaultValue(leftResource, rightResource, defaultValue)

        for resourceName, leftValue := range leftResource.ScalarResources {
                rightValue := rightResource.ScalarResources[resourceName]
                if rightValue == -1 {
                        continue
                }
                if leftValue == -1 || !lessEqualFunc(leftValue, rightValue, minResource) {
                        return false
                }
        }
        return true
}

// LessPartly returns true if there exists any dimension whose resource amount in r is less than that in rr.
// Otherwise returns false.
// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
func (r *Resource) LessPartly(rr *Resource, defaultValue DimensionDefaultValue) bool {
        lessFunc := func(l, r float64) bool {
                return l < r
        }

        leftResource := r.Clone()
        rightResource := rr.Clone()

        if lessFunc(leftResource.MilliCPU, rightResource.MilliCPU) || lessFunc(leftResource.Memory, rightResource.Memory) {
                return true
        }

        r.setDefaultValue(leftResource, rightResource, defaultValue)

        for resourceName, leftValue := range leftResource.ScalarResources {
                rightValue := rightResource.ScalarResources[resourceName]
                if leftValue == -1 {
                        continue
                }
                if rightValue == -1 || lessFunc(leftValue, rightValue) {
                        return true
                }
        }
        return false
}

// LessEqualPartly returns true if there exists any dimension whose resource amount in r is less than or equal with that in rr.
// Otherwise returns false.
// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
func (r *Resource) LessEqualPartly(rr *Resource, defaultValue DimensionDefaultValue) bool {
        lessEqualFunc := func(l, r, diff float64) bool {
                if l < r || math.Abs(l-r) < diff {
                        return true
                }
                return false
        }

        leftResource := r.Clone()
        rightResource := rr.Clone()

        if lessEqualFunc(leftResource.MilliCPU, rightResource.MilliCPU, minResource) || lessEqualFunc(leftResource.Memory, rightResource.Memory, minResource) {
                return true
        }

        r.setDefaultValue(leftResource, rightResource, defaultValue)

        for resourceName, leftValue := range leftResource.ScalarResources {
                rightValue := rightResource.ScalarResources[resourceName]
                if leftValue == -1 {
                        continue
                }
                if rightValue == -1 || lessEqualFunc(leftValue, rightValue, minResource) {
                        return true
                }
        }
        return false
}

// Equal returns true only on condition that values in all dimension are equal with each other for r and rr
// Otherwise returns false.
// @param defaultValue "default value for resource dimension not defined in ScalarResources. Its value can only be one of 'Zero' and 'Infinity'"
func (r *Resource) Equal(rr *Resource, defaultValue DimensionDefaultValue) bool {
        equalFunc := func(l, r, diff float64) bool {
                return l == r || math.Abs(l-r) < diff
        }

        leftResource := r.Clone()
        rightResource := rr.Clone()

        if !equalFunc(leftResource.MilliCPU, rightResource.MilliCPU, minResource) || !equalFunc(leftResource.Memory, rightResource.Memory, minResource) {
                return false
        }

        r.setDefaultValue(leftResource, rightResource, defaultValue)

        for resourceName, leftValue := range leftResource.ScalarResources {
                rightValue := rightResource.ScalarResources[resourceName]
                if !equalFunc(leftValue, rightValue, minResource) {
                        return false
                }
        }
        return true
}

// Diff calculate the difference between two resource object
// Note: if `defaultValue` equals `Infinity`, the difference between two values will be `Infinity`, marked as -1
func (r *Resource) Diff(rr *Resource, defaultValue DimensionDefaultValue) (*Resource, *Resource) {
        leftRes := r.Clone()
        rightRes := rr.Clone()
        increasedVal := EmptyResource()
        decreasedVal := EmptyResource()
        r.setDefaultValue(leftRes, rightRes, defaultValue)

        if leftRes.MilliCPU > rightRes.MilliCPU {
                increasedVal.MilliCPU = leftRes.MilliCPU - rightRes.MilliCPU
        } else {
                decreasedVal.MilliCPU = rightRes.MilliCPU - leftRes.MilliCPU
        }

        if leftRes.Memory > rightRes.Memory {
                increasedVal.Memory = leftRes.Memory - rightRes.Memory
        } else {
                decreasedVal.Memory = rightRes.Memory - leftRes.Memory
        }

        increasedVal.ScalarResources = make(map[v1.ResourceName]float64, 0)
        decreasedVal.ScalarResources = make(map[v1.ResourceName]float64, 0)
        for lName, lQuant := range leftRes.ScalarResources {
                rQuant, _ := rightRes.ScalarResources[lName]
                if lQuant == -1 {
                        increasedVal.ScalarResources[lName] = -1
                        continue
                }
                if rQuant == -1 {
                        decreasedVal.ScalarResources[lName] = -1
                        continue
                }
                if lQuant > rQuant {
                        increasedVal.ScalarResources[lName] = lQuant - rQuant
                } else {
                        decreasedVal.ScalarResources[lName] = rQuant - lQuant
                }
        }

        return increasedVal, decreasedVal
}

// AddScalar adds a resource by a scalar value of this resource.
func (r *Resource) AddScalar(name v1.ResourceName, quantity float64) {
        r.SetScalar(name, r.ScalarResources[name]+quantity)
}

// SetScalar sets a resource by a scalar value of this resource.
func (r *Resource) SetScalar(name v1.ResourceName, quantity float64) {
        // Lazily allocate scalar resource map.
        if r.ScalarResources == nil {
                r.ScalarResources = map[v1.ResourceName]float64{}
        }
        r.ScalarResources[name] = quantity
}

// MinDimensionResource is used to reset the r resource dimension which is less than rr
// e.g r resource is <cpu 2000.00, memory 4047845376.00, hugepages-2Mi 0.00, hugepages-1Gi 0.00>
// rr resource is <cpu 3000.00, memory 1000.00>
// return r resource is <cpu 2000.00, memory 1000.00, hugepages-2Mi 0.00, hugepages-1Gi 0.00>
func (r *Resource) MinDimensionResource(rr *Resource) *Resource {
        if rr.MilliCPU < r.MilliCPU {
                r.MilliCPU = rr.MilliCPU
        }
        if rr.Memory < r.Memory {
                r.Memory = rr.Memory
        }

        if rr.ScalarResources == nil {
                if r.ScalarResources != nil {
                        for name := range r.ScalarResources {
                                r.ScalarResources[name] = 0
                        }
                }
        } else {
                if r.ScalarResources != nil {
                        for name, quant := range rr.ScalarResources {
                                if quant < r.ScalarResources[name] {
                                        r.ScalarResources[name] = quant
                                }
                        }
                }
        }
        return r
}

// setDefaultValue sets default value for resource dimension not defined of ScalarResource in leftResource and rightResource
// @param defaultValue "default value for resource dimension not defined in ScalarResources. It can only be one of 'Zero' or 'Infinity'"
func (r *Resource) setDefaultValue(leftResource, rightResource *Resource, defaultValue DimensionDefaultValue) {
        if leftResource.ScalarResources == nil {
                leftResource.ScalarResources = map[v1.ResourceName]float64{}
        }
        if rightResource.ScalarResources == nil {
                rightResource.ScalarResources = map[v1.ResourceName]float64{}
        }
        for resourceName := range leftResource.ScalarResources {
                _, ok := rightResource.ScalarResources[resourceName]
                if !ok {
                        if defaultValue == Zero {
                                rightResource.ScalarResources[resourceName] = 0
                        } else if defaultValue == Infinity {
                                rightResource.ScalarResources[resourceName] = -1
                        }
                }
        }

        for resourceName := range rightResource.ScalarResources {
                _, ok := leftResource.ScalarResources[resourceName]
                if !ok {
                        if defaultValue == Zero {
                                leftResource.ScalarResources[resourceName] = 0
                        } else if defaultValue == Infinity {
                                leftResource.ScalarResources[resourceName] = -1
                        }
                }
        }
}

// ParseResourceList parses the given configuration map into an API
// ResourceList or returns an error.
func ParseResourceList(m map[string]string) (v1.ResourceList, error) {
        if len(m) == 0 {
                return nil, nil
        }
        rl := make(v1.ResourceList)
        for k, v := range m {
                switch v1.ResourceName(k) {
                // CPU, memory, local storage, and PID resources are supported.
                case v1.ResourceCPU, v1.ResourceMemory, v1.ResourceEphemeralStorage:
                        q, err := resource.ParseQuantity(v)
                        if err != nil {
                                return nil, err
                        }
                        if q.Sign() == -1 {
                                return nil, fmt.Errorf("resource quantity for %q cannot be negative: %v", k, v)
                        }
                        rl[v1.ResourceName(k)] = q
                default:
                        return nil, fmt.Errorf("cannot reserve %q resource", k)
                }
        }
        return rl, nil
}

func GetMinResource() float64 {
        return minResource
}

// ResourceNameList struct defines resource name collection
type ResourceNameList []v1.ResourceName

// Contains judges whether rr is subset of r
func (r ResourceNameList) Contains(rr ResourceNameList) bool {
        for _, rrName := range ([]v1.ResourceName)(rr) {
                isResourceExist := false
                for _, rName := range ([]v1.ResourceName)(r) {
                        if rName == rrName {
                                isResourceExist = true
                                break
                        }
                }
                if !isResourceExist {
                        return false
                }
        }
        return true
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "k8s.io/apimachinery/pkg/types"

        "volcano.sh/apis/pkg/apis/scheduling"
)

// ClusterID is UID type, serves as unique ID for each queue
type ClusterID types.UID

// SiloClusterInfo will have all details about queue
type SiloClusterInfo struct {
        UID     ClusterID
        Cluster *scheduling.Cluster
}

// NewSiloClusterInfo creates new queueInfo object
func NewSiloClusterInfo(cluster *scheduling.Cluster) *SiloClusterInfo {
        return &SiloClusterInfo{
                UID:     ClusterID(cluster.Name),
                Cluster: cluster,
        }
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/resource"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
)

func buildNode(name string, alloc v1.ResourceList) *v1.Node {
        return &v1.Node{
                ObjectMeta: metav1.ObjectMeta{
                        Name: name,
                },
                Status: v1.NodeStatus{
                        Capacity:    alloc,
                        Allocatable: alloc,
                },
        }
}

func buildPod(ns, n, nn string, p v1.PodPhase, req v1.ResourceList, owner []metav1.OwnerReference, labels map[string]string) *v1.Pod {
        return &v1.Pod{
                ObjectMeta: metav1.ObjectMeta{
                        UID:             types.UID(fmt.Sprintf("%v-%v", ns, n)),
                        Name:            n,
                        Namespace:       ns,
                        OwnerReferences: owner,
                        Labels:          labels,
                },
                Status: v1.PodStatus{
                        Phase: p,
                },
                Spec: v1.PodSpec{
                        NodeName: nn,
                        Containers: []v1.Container{
                                {
                                        Resources: v1.ResourceRequirements{
                                                Requests: req,
                                        },
                                },
                        },
                },
        }
}

func buildResourceList(cpu string, memory string) v1.ResourceList {
        return v1.ResourceList{
                v1.ResourceCPU:    resource.MustParse(cpu),
                v1.ResourceMemory: resource.MustParse(memory),
        }
}

func buildResource(cpu string, memory string) *Resource {
        return NewResource(v1.ResourceList{
                v1.ResourceCPU:    resource.MustParse(cpu),
                v1.ResourceMemory: resource.MustParse(memory),
        })
}

func buildOwnerReference(owner string) metav1.OwnerReference {
        controller := true
        return metav1.OwnerReference{
                Controller: &controller,
                UID:        types.UID(owner),
        }
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
        k8sframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
)

// TaskStatus defines the status of a task/pod.
type TaskStatus int

const (
        // Pending means the task is pending in the apiserver.
        Pending TaskStatus = 1 << iota

        // Allocated means the scheduler assigns a host to it.
        Allocated

        // Pipelined means the scheduler assigns a host to wait for releasing resource.
        Pipelined

        // Binding means the scheduler send Bind request to apiserver.
        Binding

        // Bound means the task/Pod bounds to a host.
        Bound

        // Running means a task is running on the host.
        Running

        // Releasing means a task/pod is deleted.
        Releasing

        // Succeeded means that all containers in the pod have voluntarily terminated
        // with a container exit code of 0, and the system is not going to restart any of these containers.
        Succeeded

        // Failed means that all containers in the pod have terminated, and at least one container has
        // terminated in a failure (exited with a non-zero exit code or was stopped by the system).
        Failed

        // Unknown means the status of task/pod is unknown to the scheduler.
        Unknown
)

func (ts TaskStatus) String() string {
        switch ts {
        case Pending:
                return "Pending"
        case Allocated:
                return "Allocated"
        case Pipelined:
                return "Pipelined"
        case Binding:
                return "Binding"
        case Bound:
                return "Bound"
        case Running:
                return "Running"
        case Releasing:
                return "Releasing"
        case Succeeded:
                return "Succeeded"
        case Failed:
                return "Failed"
        default:
                return "Unknown"
        }
}

// NodePhase defines the phase of node
type NodePhase int

const (
        // Ready means the node is ready for scheduling
        Ready NodePhase = 1 << iota
        // NotReady means the node is not ready for scheduling
        NotReady
)

func (np NodePhase) String() string {
        switch np {
        case Ready:
                return "Ready"
        case NotReady:
                return "NotReady"
        }

        return "Unknown"
}

// validateStatusUpdate validates whether the status transfer is valid.
func validateStatusUpdate(oldStatus, newStatus TaskStatus) error {
        return nil
}

// LessFn is the func declaration used by sort or priority queue.
type LessFn func(interface{}, interface{}) bool

// CompareFn is the func declaration used by sort or priority queue.
type CompareFn func(interface{}, interface{}) int

// ValidateFn is the func declaration used to check object's status.
type ValidateFn func(interface{}) bool

// ValidateResult is struct to which can used to determine the result
type ValidateResult struct {
        Pass    bool
        Reason  string
        Message string
}

// ValidateExFn is the func declaration used to validate the result.
type ValidateExFn func(interface{}) *ValidateResult

// VoteFn is the func declaration used to check object's complicated status.
type VoteFn func(interface{}) int

// JobEnqueuedFn is the func declaration used to call after job enqueued.
type JobEnqueuedFn func(interface{})

// PredicateFn is the func declaration used to predicate node for task.
type PredicateFn func(*TaskInfo, *NodeInfo) error

// BestNodeFn is the func declaration used to return the nodeScores to plugins.
type BestNodeFn func(*TaskInfo, map[float64][]*NodeInfo) *NodeInfo

// EvictableFn is the func declaration used to evict tasks.
type EvictableFn func(*TaskInfo, []*TaskInfo) ([]*TaskInfo, int)

// NodeOrderFn is the func declaration used to get priority score for a node for a particular task.
type NodeOrderFn func(*TaskInfo, *NodeInfo) (float64, error)

// BatchNodeOrderFn is the func declaration used to get priority score for ALL nodes for a particular task.
type BatchNodeOrderFn func(*TaskInfo, []*NodeInfo) (map[string]float64, error)

// NodeMapFn is the func declaration used to get priority score for a node for a particular task.
type NodeMapFn func(*TaskInfo, *NodeInfo) (float64, error)

// NodeReduceFn is the func declaration used to reduce priority score for a node for a particular task.
type NodeReduceFn func(*TaskInfo, k8sframework.NodeScoreList) error

// NodeOrderMapFn is the func declaration used to get priority score of all plugins for a node for a particular task.
type NodeOrderMapFn func(*TaskInfo, *NodeInfo) (map[string]float64, float64, error)

// NodeOrderReduceFn is the func declaration used to reduce priority score of all nodes for a plugin for a particular task.
type NodeOrderReduceFn func(*TaskInfo, map[string]k8sframework.NodeScoreList) (map[string]float64, error)

// TargetJobFn is the func declaration used to select the target job satisfies some conditions
type TargetJobFn func([]*JobInfo) *JobInfo

// ReservedNodesFn is the func declaration used to select the reserved nodes
type ReservedNodesFn func()

// VictimTasksFn is the func declaration used to select victim tasks
type VictimTasksFn func() []*TaskInfo

// UnderUsedResourceFn is the func declaration used to get under used resource list for queue
type UnderUsedResourceFn func(*QueueInfo) ResourceNameList

package api

import (
        "fmt"
        "sort"
        "strings"
)

const (
        // NodePodNumberExceeded means pods in node exceed the allocatable pod number
        NodePodNumberExceeded = "node(s) pod number exceeded"
        // NodeResourceFitFailed means node could not fit the request of pod
        NodeResourceFitFailed = "node(s) resource fit failed"

        // AllNodeUnavailableMsg is the default error message
        AllNodeUnavailableMsg = "all nodes are unavailable"
)

// These are reasons for a pod's transition to a condition.
const (
        // PodReasonUnschedulable reason in PodScheduled PodCondition means that the scheduler
        // can't schedule the pod right now, for example due to insufficient resources in the cluster.
        PodReasonUnschedulable = "Unschedulable"
        // PodReasonSchedulable reason in PodScheduled PodCondition means that the scheduler
        // can schedule the pod right now, but not bind yet
        PodReasonSchedulable = "Schedulable"
        // PodReasonUndetermined reason in PodScheduled PodCondition means that the scheduler
        // skips scheduling the pod which left the pod `Undetermined`, for example due to unschedulable pod already occurred.
        PodReasonUndetermined = "Undetermined"
)

// FitErrors is set of FitError on many nodes
type FitErrors struct {
        nodes map[string]*FitError
        err   string
}

// NewFitErrors returns an FitErrors
func NewFitErrors() *FitErrors {
        f := new(FitErrors)
        f.nodes = make(map[string]*FitError)
        return f
}

// SetError set the common error message in FitErrors
func (f *FitErrors) SetError(err string) {
        f.err = err
}

// SetNodeError set the node error in FitErrors
func (f *FitErrors) SetNodeError(nodeName string, err error) {
        var fe *FitError
        switch obj := err.(type) {
        case *FitError:
                obj.NodeName = nodeName
                fe = obj
        default:
                fe = &FitError{
                        NodeName: nodeName,
                        Reasons:  []string{obj.Error()},
                }
        }

        f.nodes[nodeName] = fe
}

// Error returns the final error message
func (f *FitErrors) Error() string {
        reasons := make(map[string]int)

        for _, node := range f.nodes {
                for _, reason := range node.Reasons {
                        reasons[reason]++
                }
        }

        sortReasonsHistogram := func() []string {
                reasonStrings := []string{}
                for k, v := range reasons {
                        reasonStrings = append(reasonStrings, fmt.Sprintf("%v %v", v, k))
                }
                sort.Strings(reasonStrings)
                return reasonStrings
        }
        if f.err == "" {
                f.err = AllNodeUnavailableMsg
        }
        reasonMsg := fmt.Sprintf(f.err+": %v.", strings.Join(sortReasonsHistogram(), ", "))
        return reasonMsg
}

// FitError describe the reason why task could not fit that node
type FitError struct {
        taskNamespace string
        taskName      string
        NodeName      string
        Reasons       []string
}

// NewFitError return FitError by message
func NewFitError(task *TaskInfo, node *NodeInfo, message ...string) *FitError {
        fe := &FitError{
                taskName:      task.Name,
                taskNamespace: task.Namespace,
                NodeName:      node.Name,
                Reasons:       message,
        }
        return fe
}

// Error returns the final error message
func (f *FitError) Error() string {
        return fmt.Sprintf("task %s/%s on node %s fit failed: %s", f.taskNamespace, f.taskName, f.NodeName, strings.Join(f.Reasons, ", "))
}

/*
 Copyright 2021 The Volcano Authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package cache

import (
        "context"
        "fmt"
        "sync"
        "time"

        v1 "k8s.io/api/core/v1"
        "k8s.io/api/scheduling/v1beta1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/runtime"
        utilruntime "k8s.io/apimachinery/pkg/util/runtime"
        "k8s.io/apimachinery/pkg/util/wait"
        "k8s.io/client-go/informers"
        infov1 "k8s.io/client-go/informers/core/v1"
        schedv1 "k8s.io/client-go/informers/scheduling/v1beta1"
        storagev1 "k8s.io/client-go/informers/storage/v1"
        storagev1alpha1 "k8s.io/client-go/informers/storage/v1alpha1"
        "k8s.io/client-go/kubernetes"
        corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
        "k8s.io/client-go/rest"
        "k8s.io/client-go/tools/cache"
        "k8s.io/client-go/tools/record"
        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"
        podutil "k8s.io/kubernetes/pkg/api/v1/pod"
        volumescheduling "k8s.io/kubernetes/pkg/controller/volume/scheduling"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/apis/pkg/apis/scheduling"
        schedulingscheme "volcano.sh/apis/pkg/apis/scheduling/scheme"
        vcv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        vcclient "volcano.sh/apis/pkg/client/clientset/versioned"
        "volcano.sh/apis/pkg/client/clientset/versioned/scheme"
        vcinformer "volcano.sh/apis/pkg/client/informers/externalversions"
        cpuinformerv1 "volcano.sh/apis/pkg/client/informers/externalversions/nodeinfo/v1alpha1"
        vcinformerv1 "volcano.sh/apis/pkg/client/informers/externalversions/scheduling/v1beta1"

        "volcano.sh/volcano/cmd/scheduler/app/options"
        schedulingapi "volcano.sh/volcano/pkg/scheduler/api"
)

func init() {
        schemeBuilder := runtime.SchemeBuilder{
                v1.AddToScheme,
        }

        utilruntime.Must(schemeBuilder.AddToScheme(scheme.Scheme))
}

// New returns a Cache implementation.
func New(config *rest.Config, schedulerName string, defaultQueue string) Cache {
        return newSchedulerCache(config, schedulerName, defaultQueue)
}

// SchedulerCache cache for the kube batch
type SchedulerCache struct {
        sync.Mutex

        kubeClient   *kubernetes.Clientset
        vcClient     *vcclient.Clientset
        defaultQueue string
        // schedulerName is the name for volcano scheduler
        schedulerName string

        podInformer                infov1.PodInformer
        nodeInformer               infov1.NodeInformer
        podGroupInformerV1beta1    vcinformerv1.PodGroupInformer
        queueInformerV1beta1       vcinformerv1.QueueInformer
        pvInformer                 infov1.PersistentVolumeInformer
        pvcInformer                infov1.PersistentVolumeClaimInformer
        scInformer                 storagev1.StorageClassInformer
        pcInformer                 schedv1.PriorityClassInformer
        quotaInformer              infov1.ResourceQuotaInformer
        csiNodeInformer            storagev1.CSINodeInformer
        csiDriverInformer          storagev1.CSIDriverInformer
        csiStorageCapacityInformer storagev1alpha1.CSIStorageCapacityInformer
        cpuInformer                cpuinformerv1.NumatopologyInformer

        Binder         Binder
        Evictor        Evictor
        StatusUpdater  StatusUpdater
        PodGroupBinder BatchBinder
        VolumeBinder   VolumeBinder

        Recorder record.EventRecorder

        Jobs                 map[schedulingapi.JobID]*schedulingapi.JobInfo
        Nodes                map[string]*schedulingapi.NodeInfo
        Queues               map[schedulingapi.QueueID]*schedulingapi.QueueInfo
        PriorityClasses      map[string]*v1beta1.PriorityClass
        NodeList             []string
        defaultPriorityClass *v1beta1.PriorityClass
        defaultPriority      int32

        NamespaceCollection map[string]*schedulingapi.NamespaceCollection

        errTasks    workqueue.RateLimitingInterface
        deletedJobs workqueue.RateLimitingInterface

        informerFactory informers.SharedInformerFactory
}

type defaultBinder struct {
        kubeclient *kubernetes.Clientset
}

//Bind will send bind request to api server
func (db *defaultBinder) Bind(p *v1.Pod, hostname string) error {
        if err := db.kubeclient.CoreV1().Pods(p.Namespace).Bind(context.TODO(),
                &v1.Binding{
                        ObjectMeta: metav1.ObjectMeta{Namespace: p.Namespace, Name: p.Name, UID: p.UID, Annotations: p.Annotations},
                        Target: v1.ObjectReference{
                                Kind: "Node",
                                Name: hostname,
                        },
                },
                metav1.CreateOptions{}); err != nil {
                klog.Errorf("Failed to bind pod <%v/%v>: %#v", p.Namespace, p.Name, err)
                return err
        }
        return nil
}

type defaultEvictor struct {
        kubeclient *kubernetes.Clientset
        recorder   record.EventRecorder
}

// Evict will send delete pod request to api server
func (de *defaultEvictor) Evict(p *v1.Pod, reason string) error {
        klog.V(3).Infof("Evicting pod %v/%v, because of %v", p.Namespace, p.Name, reason)

        evictMsg := fmt.Sprintf("Pod is evicted, because of %v", reason)
        annotations := map[string]string{}
        // record that we are evicting the pod
        de.recorder.AnnotatedEventf(p, annotations, v1.EventTypeWarning, "Evict", evictMsg)

        pod := p.DeepCopy()
        condition := &v1.PodCondition{
                Type:    v1.PodReady,
                Status:  v1.ConditionFalse,
                Reason:  "Evict",
                Message: evictMsg,
        }
        if !podutil.UpdatePodCondition(&pod.Status, condition) {
                klog.V(1).Infof("UpdatePodCondition: existed condition, not update")
                klog.V(1).Infof("%+v", pod.Status.Conditions)
                return nil
        }
        if _, err := de.kubeclient.CoreV1().Pods(p.Namespace).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{}); err != nil {
                klog.Errorf("Failed to update pod <%v/%v> status: %v", pod.Namespace, pod.Name, err)
                return err
        }
        if err := de.kubeclient.CoreV1().Pods(p.Namespace).Delete(context.TODO(), p.Name, metav1.DeleteOptions{}); err != nil {
                klog.Errorf("Failed to evict pod <%v/%v>: %#v", p.Namespace, p.Name, err)
                return err
        }

        return nil
}

// defaultStatusUpdater is the default implementation of the StatusUpdater interface
type defaultStatusUpdater struct {
        kubeclient *kubernetes.Clientset
        vcclient   *vcclient.Clientset
}

// following the same logic as podutil.UpdatePodCondition
func podConditionHaveUpdate(status *v1.PodStatus, condition *v1.PodCondition) bool {
        lastTransitionTime := metav1.Now()
        // Try to find this pod condition.
        _, oldCondition := podutil.GetPodCondition(status, condition.Type)

        if oldCondition == nil {
                // We are adding new pod condition.
                return true
        }
        // We are updating an existing condition, so we need to check if it has changed.
        if condition.Status == oldCondition.Status {
                lastTransitionTime = oldCondition.LastTransitionTime
        }

        isEqual := condition.Status == oldCondition.Status &&
                condition.Reason == oldCondition.Reason &&
                condition.Message == oldCondition.Message &&
                condition.LastProbeTime.Equal(&oldCondition.LastProbeTime) &&
                lastTransitionTime.Equal(&oldCondition.LastTransitionTime)

        // Return true if one of the fields have changed.
        return !isEqual
}

// UpdatePodCondition will Update pod with podCondition
func (su *defaultStatusUpdater) UpdatePodCondition(pod *v1.Pod, condition *v1.PodCondition) (*v1.Pod, error) {
        klog.V(3).Infof("Updating pod condition for %s/%s to (%s==%s)", pod.Namespace, pod.Name, condition.Type, condition.Status)
        if podutil.UpdatePodCondition(&pod.Status, condition) {
                return su.kubeclient.CoreV1().Pods(pod.Namespace).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{})
        }
        return pod, nil
}

// UpdatePodGroup will Update pod with podCondition
func (su *defaultStatusUpdater) UpdatePodGroup(pg *schedulingapi.PodGroup) (*schedulingapi.PodGroup, error) {
        podgroup := &vcv1beta1.PodGroup{}
        if err := schedulingscheme.Scheme.Convert(&pg.PodGroup, podgroup, nil); err != nil {
                klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err)
                return nil, err
        }

        updated, err := su.vcclient.SchedulingV1beta1().PodGroups(podgroup.Namespace).Update(context.TODO(), podgroup, metav1.UpdateOptions{})
        if err != nil {
                klog.Errorf("Error while updating PodGroup with error: %v", err)
                return nil, err
        }

        podGroupInfo := &schedulingapi.PodGroup{Version: schedulingapi.PodGroupVersionV1Beta1}
        if err := schedulingscheme.Scheme.Convert(updated, &podGroupInfo.PodGroup, nil); err != nil {
                klog.Errorf("Error while converting v1alpha.PodGroup to api.PodGroup with error: %v", err)
                return nil, err
        }

        return podGroupInfo, nil
}

type defaultVolumeBinder struct {
        volumeBinder volumescheduling.SchedulerVolumeBinder
}

// AllocateVolumes allocates volume on the host to the task
func (dvb *defaultVolumeBinder) AllocateVolumes(task *schedulingapi.TaskInfo, hostname string, podVolumes *volumescheduling.PodVolumes) error {
        allBound, err := dvb.volumeBinder.AssumePodVolumes(task.Pod, hostname, podVolumes)
        task.VolumeReady = allBound

        return err
}

// GetPodVolumes get pod volume on the host
func (dvb *defaultVolumeBinder) GetPodVolumes(task *schedulingapi.TaskInfo,
        node *v1.Node) (podVolumes *volumescheduling.PodVolumes, err error) {
        boundClaims, claimsToBind, _, err := dvb.volumeBinder.GetPodVolumes(task.Pod)
        if err != nil {
                return nil, err
        }

        podVolumes, _, err = dvb.volumeBinder.FindPodVolumes(task.Pod, boundClaims, claimsToBind, node)
        return podVolumes, err
}

// BindVolumes binds volumes to the task
func (dvb *defaultVolumeBinder) BindVolumes(task *schedulingapi.TaskInfo, podVolumes *volumescheduling.PodVolumes) error {
        // If task's volumes are ready, did not bind them again.
        if task.VolumeReady {
                return nil
        }

        return dvb.volumeBinder.BindPodVolumes(task.Pod, podVolumes)
}

type podgroupBinder struct {
        kubeclient *kubernetes.Clientset
        vcclient   *vcclient.Clientset
}

// Bind will add silo cluster annotaion on pod and podgroup
func (pgb *podgroupBinder) Bind(job *schedulingapi.JobInfo, cluster string) (*schedulingapi.JobInfo, error) {
        if len(job.Tasks) == 0 {
                klog.V(4).Infof("Job pods have not been created yet")
                return job, nil
        }
        for _, task := range job.Tasks {
                pod := task.Pod
                pod.Annotations[batch.ForwardClusterKey] = cluster
                pod.ResourceVersion = ""
                _, err := pgb.kubeclient.CoreV1().Pods(pod.Namespace).UpdateStatus(context.TODO(), pod, metav1.UpdateOptions{})
                if err != nil {
                        klog.Errorf("Error while update pod annotation with error: %v", err)
                        return nil, err
                }
        }

        pg := job.PodGroup
        pg.Annotations[batch.ForwardClusterKey] = cluster
        podgroup := &vcv1beta1.PodGroup{}
        if err := schedulingscheme.Scheme.Convert(&pg.PodGroup, podgroup, nil); err != nil {
                klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err)
                return nil, err
        }
        newPg, err := pgb.vcclient.SchedulingV1beta1().PodGroups(pg.Namespace).Update(context.TODO(), podgroup, metav1.UpdateOptions{})
        if err != nil {
                klog.Errorf("Error while update PodGroup annotation with error: %v", err)
                return nil, err
        }
        job.PodGroup.ResourceVersion = newPg.ResourceVersion
        klog.V(4).Infof("Bind PodGroup <%s> successfully", job.PodGroup.Name)
        return job, nil
}

func newSchedulerCache(config *rest.Config, schedulerName string, defaultQueue string) *SchedulerCache {
        kubeClient, err := kubernetes.NewForConfig(config)
        if err != nil {
                panic(fmt.Sprintf("failed init kubeClient, with err: %v", err))
        }
        vcClient, err := vcclient.NewForConfig(config)
        if err != nil {
                panic(fmt.Sprintf("failed init vcClient, with err: %v", err))
        }
        eventClient, err := kubernetes.NewForConfig(config)
        if err != nil {
                panic(fmt.Sprintf("failed init eventClient, with err: %v", err))
        }

        // create default queue
        reclaimable := true
        defaultQue := vcv1beta1.Queue{
                ObjectMeta: metav1.ObjectMeta{
                        Name: defaultQueue,
                },
                Spec: vcv1beta1.QueueSpec{
                        Reclaimable: &reclaimable,
                        Weight:      1,
                },
        }
        if _, err := vcClient.SchedulingV1beta1().Queues().Create(context.TODO(), &defaultQue, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
                panic(fmt.Sprintf("failed init default queue, with err: %v", err))
        }

        sc := &SchedulerCache{
                Jobs:            make(map[schedulingapi.JobID]*schedulingapi.JobInfo),
                Nodes:           make(map[string]*schedulingapi.NodeInfo),
                Queues:          make(map[schedulingapi.QueueID]*schedulingapi.QueueInfo),
                PriorityClasses: make(map[string]*v1beta1.PriorityClass),
                errTasks:        workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
                deletedJobs:     workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
                kubeClient:      kubeClient,
                vcClient:        vcClient,
                defaultQueue:    defaultQueue,
                schedulerName:   schedulerName,

                NamespaceCollection: make(map[string]*schedulingapi.NamespaceCollection),

                NodeList: []string{},
        }

        // Prepare event clients.
        broadcaster := record.NewBroadcaster()
        broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: eventClient.CoreV1().Events("")})
        sc.Recorder = broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: schedulerName})

        sc.Binder = &defaultBinder{
                kubeclient: sc.kubeClient,
        }

        sc.Evictor = &defaultEvictor{
                kubeclient: sc.kubeClient,
                recorder:   sc.Recorder,
        }

        sc.StatusUpdater = &defaultStatusUpdater{
                kubeclient: sc.kubeClient,
                vcclient:   sc.vcClient,
        }

        sc.PodGroupBinder = &podgroupBinder{
                kubeclient: sc.kubeClient,
                vcclient:   sc.vcClient,
        }

        informerFactory := informers.NewSharedInformerFactory(sc.kubeClient, 0)
        sc.informerFactory = informerFactory

        // create informer for node information
        sc.nodeInformer = informerFactory.Core().V1().Nodes()
        sc.nodeInformer.Informer().AddEventHandlerWithResyncPeriod(
                cache.ResourceEventHandlerFuncs{
                        AddFunc:    sc.AddNode,
                        UpdateFunc: sc.UpdateNode,
                        DeleteFunc: sc.DeleteNode,
                },
                0,
        )

        sc.podInformer = informerFactory.Core().V1().Pods()
        sc.pvcInformer = informerFactory.Core().V1().PersistentVolumeClaims()
        sc.pvInformer = informerFactory.Core().V1().PersistentVolumes()
        sc.scInformer = informerFactory.Storage().V1().StorageClasses()
        sc.csiNodeInformer = informerFactory.Storage().V1().CSINodes()
        sc.csiDriverInformer = informerFactory.Storage().V1().CSIDrivers()
        sc.csiStorageCapacityInformer = informerFactory.Storage().V1alpha1().CSIStorageCapacities()
        sc.VolumeBinder = &defaultVolumeBinder{
                volumeBinder: volumescheduling.NewVolumeBinder(
                        sc.kubeClient,
                        sc.podInformer,
                        sc.nodeInformer,
                        sc.csiNodeInformer,
                        sc.pvcInformer,
                        sc.pvInformer,
                        sc.scInformer,
                        &volumescheduling.CapacityCheck{
                                CSIDriverInformer:          sc.csiDriverInformer,
                                CSIStorageCapacityInformer: sc.csiStorageCapacityInformer,
                        },
                        30*time.Second,
                ),
        }

        // create informer for pod information
        sc.podInformer.Informer().AddEventHandler(
                cache.FilteringResourceEventHandler{
                        FilterFunc: func(obj interface{}) bool {
                                switch v := obj.(type) {
                                case *v1.Pod:
                                        if !responsibleForPod(v, schedulerName) {
                                                if len(v.Spec.NodeName) == 0 {
                                                        return false
                                                }
                                        }
                                        return true
                                default:
                                        return false
                                }
                        },
                        Handler: cache.ResourceEventHandlerFuncs{
                                AddFunc:    sc.AddPod,
                                UpdateFunc: sc.UpdatePod,
                                DeleteFunc: sc.DeletePod,
                        },
                })

        sc.pcInformer = informerFactory.Scheduling().V1beta1().PriorityClasses()
        sc.pcInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    sc.AddPriorityClass,
                UpdateFunc: sc.UpdatePriorityClass,
                DeleteFunc: sc.DeletePriorityClass,
        })

        sc.quotaInformer = informerFactory.Core().V1().ResourceQuotas()
        sc.quotaInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    sc.AddResourceQuota,
                UpdateFunc: sc.UpdateResourceQuota,
                DeleteFunc: sc.DeleteResourceQuota,
        })

        vcinformers := vcinformer.NewSharedInformerFactory(sc.vcClient, 0)

        // create informer for PodGroup(v1beta1) information
        sc.podGroupInformerV1beta1 = vcinformers.Scheduling().V1beta1().PodGroups()
        sc.podGroupInformerV1beta1.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    sc.AddPodGroupV1beta1,
                UpdateFunc: sc.UpdatePodGroupV1beta1,
                DeleteFunc: sc.DeletePodGroupV1beta1,
        })

        // create informer(v1beta1) for Queue information
        sc.queueInformerV1beta1 = vcinformers.Scheduling().V1beta1().Queues()
        sc.queueInformerV1beta1.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    sc.AddQueueV1beta1,
                UpdateFunc: sc.UpdateQueueV1beta1,
                DeleteFunc: sc.DeleteQueueV1beta1,
        })

        sc.cpuInformer = vcinformers.Nodeinfo().V1alpha1().Numatopologies()
        sc.cpuInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
                AddFunc:    sc.AddNumaInfoV1alpha1,
                UpdateFunc: sc.UpdateNumaInfoV1alpha1,
                DeleteFunc: sc.DeleteNumaInfoV1alpha1,
        })
        return sc
}

// Run  starts the schedulerCache
func (sc *SchedulerCache) Run(stopCh <-chan struct{}) {
        go sc.podInformer.Informer().Run(stopCh)
        go sc.nodeInformer.Informer().Run(stopCh)
        go sc.podGroupInformerV1beta1.Informer().Run(stopCh)
        go sc.pvInformer.Informer().Run(stopCh)
        go sc.pvcInformer.Informer().Run(stopCh)
        go sc.scInformer.Informer().Run(stopCh)
        go sc.queueInformerV1beta1.Informer().Run(stopCh)
        go sc.quotaInformer.Informer().Run(stopCh)
        go sc.cpuInformer.Informer().Run(stopCh)

        if options.ServerOpts.EnablePriorityClass {
                go sc.pcInformer.Informer().Run(stopCh)
        }

        // Re-sync error tasks.
        go wait.Until(sc.processResyncTask, 0, stopCh)

        // Cleanup jobs.
        go wait.Until(sc.processCleanupJob, 0, stopCh)
}

// WaitForCacheSync sync the cache with the api server
func (sc *SchedulerCache) WaitForCacheSync(stopCh <-chan struct{}) bool {
        return cache.WaitForCacheSync(stopCh,
                func() []cache.InformerSynced {
                        informerSynced := []cache.InformerSynced{
                                sc.podInformer.Informer().HasSynced,
                                sc.podGroupInformerV1beta1.Informer().HasSynced,
                                sc.nodeInformer.Informer().HasSynced,
                                sc.pvInformer.Informer().HasSynced,
                                sc.pvcInformer.Informer().HasSynced,
                                sc.scInformer.Informer().HasSynced,
                                sc.queueInformerV1beta1.Informer().HasSynced,
                                sc.quotaInformer.Informer().HasSynced,
                                sc.cpuInformer.Informer().HasSynced,
                        }
                        if options.ServerOpts.EnablePriorityClass {
                                informerSynced = append(informerSynced, sc.pcInformer.Informer().HasSynced)
                        }
                        return informerSynced
                }()...,
        )
}

// findJobAndTask returns job and the task info
func (sc *SchedulerCache) findJobAndTask(taskInfo *schedulingapi.TaskInfo) (*schedulingapi.JobInfo, *schedulingapi.TaskInfo, error) {
        job, found := sc.Jobs[taskInfo.Job]
        if !found {
                return nil, nil, fmt.Errorf("failed to find Job %v for Task %v",
                        taskInfo.Job, taskInfo.UID)
        }

        task, found := job.Tasks[taskInfo.UID]
        if !found {
                return nil, nil, fmt.Errorf("failed to find task in status %v by id %v",
                        taskInfo.Status, taskInfo.UID)
        }

        return job, task, nil
}

// Evict will evict the pod.
//
// If error occurs both task and job are guaranteed to be in the original state.
func (sc *SchedulerCache) Evict(taskInfo *schedulingapi.TaskInfo, reason string) error {
        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        job, task, err := sc.findJobAndTask(taskInfo)

        if err != nil {
                return err
        }

        node, found := sc.Nodes[task.NodeName]
        if !found {
                return fmt.Errorf("failed to bind Task %v to host %v, host does not exist",
                        task.UID, task.NodeName)
        }

        originalStatus := task.Status
        if err := job.UpdateTaskStatus(task, schedulingapi.Releasing); err != nil {
                return err
        }

        // Add new task to node.
        if err := node.UpdateTask(task); err != nil {
                // After failing to update task to a node we need to revert task status from Releasing,
                // otherwise task might be stuck in the Releasing state indefinitely.
                if err := job.UpdateTaskStatus(task, originalStatus); err != nil {
                        klog.Errorf("Task <%s/%s> will be resynchronized after failing to revert status "+
                                "from %s to %s after failing to update Task on Node <%s>: %v",
                                task.Namespace, task.Name, task.Status, originalStatus, node.Name, err)
                        sc.resyncTask(task)
                }
                return err
        }

        p := task.Pod

        go func() {
                err := sc.Evictor.Evict(p, reason)
                if err != nil {
                        sc.resyncTask(task)
                }
        }()

        podgroup := &vcv1beta1.PodGroup{}
        if err := schedulingscheme.Scheme.Convert(&job.PodGroup.PodGroup, podgroup, nil); err != nil {
                klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err)
                return err
        }
        sc.Recorder.Eventf(podgroup, v1.EventTypeNormal, "Evict", reason)
        return nil
}

// Bind binds task to the target host.
func (sc *SchedulerCache) Bind(taskInfo *schedulingapi.TaskInfo, hostname string) error {
        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        job, task, err := sc.findJobAndTask(taskInfo)

        if err != nil {
                return err
        }

        node, found := sc.Nodes[hostname]
        if !found {
                return fmt.Errorf("failed to bind Task %v to host %v, host does not exist",
                        task.UID, hostname)
        }

        originalStatus := task.Status
        if err := job.UpdateTaskStatus(task, schedulingapi.Binding); err != nil {
                return err
        }

        // Add task to the node.
        if err := node.AddTask(task); err != nil {
                // After failing to update task to a node we need to revert task status from Releasing,
                // otherwise task might be stuck in the Releasing state indefinitely.
                if err := job.UpdateTaskStatus(task, originalStatus); err != nil {
                        klog.Errorf("Task <%s/%s> will be resynchronized after failing to revert status "+
                                "from %s to %s after failing to update Task on Node <%s>: %v",
                                task.Namespace, task.Name, task.Status, originalStatus, node.Name, err)
                        sc.resyncTask(task)
                }
                return err
        }

        p := task.Pod
        if !(task.TopologyPolicy == "" || task.TopologyPolicy == "none") {
                if err := sc.Binder.Bind(p, hostname); err != nil {
                        sc.resyncTask(task)
                } else {
                        sc.Recorder.Eventf(p, v1.EventTypeNormal, "Scheduled", "Successfully assigned %v/%v to %v", p.Namespace, p.Name, hostname)
                }
        } else {
                go func() {
                        if err := sc.Binder.Bind(p, hostname); err != nil {
                                sc.resyncTask(task)
                        } else {
                                sc.Recorder.Eventf(p, v1.EventTypeNormal, "Scheduled", "Successfully assigned %v/%v to %v", p.Namespace, p.Name, hostname)
                        }
                }()
        }

        return nil
}

// BindPodGroup binds job to silo cluster
func (sc *SchedulerCache) BindPodGroup(job *schedulingapi.JobInfo, cluster string) error {
        if _, err := sc.PodGroupBinder.Bind(job, cluster); err != nil {
                klog.Errorf("Bind job <%s> to cluster <%s> failed: %v", job.Name, cluster, err)
                return err
        }
        return nil
}

// GetPodVolumes get pod volume on the host
func (sc *SchedulerCache) GetPodVolumes(task *schedulingapi.TaskInfo, node *v1.Node) (*volumescheduling.PodVolumes, error) {
        return sc.VolumeBinder.GetPodVolumes(task, node)
}

// AllocateVolumes allocates volume on the host to the task
func (sc *SchedulerCache) AllocateVolumes(task *schedulingapi.TaskInfo, hostname string, podVolumes *volumescheduling.PodVolumes) error {
        return sc.VolumeBinder.AllocateVolumes(task, hostname, podVolumes)
}

// BindVolumes binds volumes to the task
func (sc *SchedulerCache) BindVolumes(task *schedulingapi.TaskInfo, podVolumes *volumescheduling.PodVolumes) error {
        return sc.VolumeBinder.BindVolumes(task, podVolumes)
}

// Client returns the kubernetes clientSet
func (sc *SchedulerCache) Client() kubernetes.Interface {
        return sc.kubeClient
}

// SharedInformerFactory returns the scheduler SharedInformerFactory
func (sc *SchedulerCache) SharedInformerFactory() informers.SharedInformerFactory {
        return sc.informerFactory
}

// UpdateSchedulerNumaInfo used to update scheduler node cache NumaSchedulerInfo
func (sc *SchedulerCache) UpdateSchedulerNumaInfo(AllocatedSets map[string]schedulingapi.ResNumaSets) error {
        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        for nodeName, sets := range AllocatedSets {
                if _, found := sc.Nodes[nodeName]; !found {
                        continue
                }

                numaInfo := sc.Nodes[nodeName].NumaSchedulerInfo
                if numaInfo == nil {
                        continue
                }

                numaInfo.Allocate(sets)
        }
        return nil
}

// taskUnschedulable updates pod status of pending task
func (sc *SchedulerCache) taskUnschedulable(task *schedulingapi.TaskInfo, reason, message string) error {
        pod := task.Pod

        condition := &v1.PodCondition{
                Type:    v1.PodScheduled,
                Status:  v1.ConditionFalse,
                Reason:  reason, // Add more reasons in order to distinguish more specific scenario of pending tasks
                Message: message,
        }

        if podConditionHaveUpdate(&pod.Status, condition) {
                pod = pod.DeepCopy()

                // The reason field in 'Events' should be "FailedScheduling", there is not constants defined for this in
                // k8s core, so using the same string here.
                // The reason field in PodCondition can be "Unschedulable"
                sc.Recorder.Eventf(pod, v1.EventTypeWarning, "FailedScheduling", message)
                if _, err := sc.StatusUpdater.UpdatePodCondition(pod, condition); err != nil {
                        return err
                }
        } else {
                klog.V(4).Infof("task unscheduleable %s/%s, message: %s, skip by no condition update", pod.Namespace, pod.Name, message)
        }

        return nil
}

func (sc *SchedulerCache) deleteJob(job *schedulingapi.JobInfo) {
        klog.V(3).Infof("Try to delete Job <%v:%v/%v>", job.UID, job.Namespace, job.Name)

        sc.deletedJobs.AddRateLimited(job)
}

func (sc *SchedulerCache) processCleanupJob() {
        obj, shutdown := sc.deletedJobs.Get()
        if shutdown {
                return
        }

        defer sc.deletedJobs.Done(obj)

        job, found := obj.(*schedulingapi.JobInfo)
        if !found {
                klog.Errorf("Failed to convert <%v> to *JobInfo", obj)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        if schedulingapi.JobTerminated(job) {
                delete(sc.Jobs, job.UID)
                klog.V(3).Infof("Job <%v:%v/%v> was deleted.", job.UID, job.Namespace, job.Name)
        } else {
                // Retry
                sc.deleteJob(job)
        }
}

func (sc *SchedulerCache) resyncTask(task *schedulingapi.TaskInfo) {
        sc.errTasks.AddRateLimited(task)
}

func (sc *SchedulerCache) processResyncTask() {
        obj, shutdown := sc.errTasks.Get()
        if shutdown {
                return
        }

        defer sc.errTasks.Done(obj)

        task, ok := obj.(*schedulingapi.TaskInfo)
        if !ok {
                klog.Errorf("failed to convert %v to *schedulingapi.TaskInfo", obj)
                return
        }

        if err := sc.syncTask(task); err != nil {
                klog.Errorf("Failed to sync pod <%v/%v>, retry it.", task.Namespace, task.Name)
                sc.resyncTask(task)
        }
}

// Snapshot returns the complete snapshot of the cluster from cache
func (sc *SchedulerCache) Snapshot() *schedulingapi.ClusterInfo {
        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        snapshot := &schedulingapi.ClusterInfo{
                Nodes:          make(map[string]*schedulingapi.NodeInfo),
                Jobs:           make(map[schedulingapi.JobID]*schedulingapi.JobInfo),
                Queues:         make(map[schedulingapi.QueueID]*schedulingapi.QueueInfo),
                NamespaceInfo:  make(map[schedulingapi.NamespaceName]*schedulingapi.NamespaceInfo),
                RevocableNodes: make(map[string]*schedulingapi.NodeInfo),
                NodeList:       make([]string, len(sc.NodeList)),
        }

        copy(snapshot.NodeList, sc.NodeList)
        for _, value := range sc.Nodes {
                value.RefreshNumaSchedulerInfoByCrd()
        }

        for _, value := range sc.Nodes {
                if !value.Ready() {
                        continue
                }

                snapshot.Nodes[value.Name] = value.Clone()

                if value.RevocableZone != "" {
                        snapshot.RevocableNodes[value.Name] = snapshot.Nodes[value.Name]
                }
        }

        for _, value := range sc.Queues {
                snapshot.Queues[value.UID] = value.Clone()
        }

        var cloneJobLock sync.Mutex
        var wg sync.WaitGroup

        cloneJob := func(value *schedulingapi.JobInfo) {
                defer wg.Done()
                if value.PodGroup != nil {
                        value.Priority = sc.defaultPriority

                        priName := value.PodGroup.Spec.PriorityClassName
                        if priorityClass, found := sc.PriorityClasses[priName]; found {
                                value.Priority = priorityClass.Value
                        }

                        klog.V(4).Infof("The priority of job <%s/%s> is <%s/%d>",
                                value.Namespace, value.Name, priName, value.Priority)
                }

                clonedJob := value.Clone()

                cloneJobLock.Lock()
                snapshot.Jobs[value.UID] = clonedJob
                cloneJobLock.Unlock()
        }

        for _, value := range sc.NamespaceCollection {
                info := value.Snapshot()
                snapshot.NamespaceInfo[info.Name] = info
                klog.V(4).Infof("Namespace %s has weight %v",
                        value.Name, info.GetWeight())
        }

        for _, value := range sc.Jobs {
                // If no scheduling spec, does not handle it.
                if value.PodGroup == nil {
                        klog.V(4).Infof("The scheduling spec of Job <%v:%s/%s> is nil, ignore it.",
                                value.UID, value.Namespace, value.Name)

                        continue
                }

                if _, found := snapshot.Queues[value.Queue]; !found {
                        klog.V(3).Infof("The Queue <%v> of Job <%v/%v> does not exist, ignore it.",
                                value.Queue, value.Namespace, value.Name)
                        continue
                }

                wg.Add(1)
                go cloneJob(value)
        }
        wg.Wait()

        klog.V(3).Infof("There are <%d> Jobs, <%d> Queues and <%d> Nodes in total for scheduling.",
                len(snapshot.Jobs), len(snapshot.Queues), len(snapshot.Nodes))

        return snapshot
}

// String returns information about the cache in a string format
func (sc *SchedulerCache) String() string {
        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        str := "Cache:\n"

        if len(sc.Nodes) != 0 {
                str += "Nodes:\n"
                for _, n := range sc.Nodes {
                        str += fmt.Sprintf("\t %s: idle(%v) used(%v) allocatable(%v) pods(%d)\n",
                                n.Name, n.Idle, n.Used, n.Allocatable, len(n.Tasks))

                        i := 0
                        for _, p := range n.Tasks {
                                str += fmt.Sprintf("\t\t %d: %v\n", i, p)
                                i++
                        }
                }
        }

        if len(sc.Jobs) != 0 {
                str += "Jobs:\n"
                for _, job := range sc.Jobs {
                        str += fmt.Sprintf("\t %s\n", job)
                }
        }

        if len(sc.NamespaceCollection) != 0 {
                str += "Namespaces:\n"
                for _, ns := range sc.NamespaceCollection {
                        info := ns.Snapshot()
                        str += fmt.Sprintf("\t Namespace(%s) Weight(%v)\n",
                                info.Name, info.Weight)
                }
        }

        if len(sc.NodeList) != 0 {
                str += fmt.Sprintf("NodeList: %v\n", sc.NodeList)
        }

        return str
}

// RecordJobStatusEvent records related events according to job status.
func (sc *SchedulerCache) RecordJobStatusEvent(job *schedulingapi.JobInfo) {
        pgUnschedulable := job.PodGroup != nil &&
                (job.PodGroup.Status.Phase == scheduling.PodGroupUnknown ||
                        job.PodGroup.Status.Phase == scheduling.PodGroupPending ||
                        job.PodGroup.Status.Phase == scheduling.PodGroupInqueue)

        // If pending or unschedulable, record unschedulable event.
        if pgUnschedulable {
                msg := fmt.Sprintf("%v/%v tasks in gang unschedulable: %v",
                        len(job.TaskStatusIndex[schedulingapi.Pending]),
                        len(job.Tasks),
                        job.FitError())
                sc.recordPodGroupEvent(job.PodGroup, v1.EventTypeWarning, string(scheduling.PodGroupUnschedulableType), msg)
        } else {
                sc.recordPodGroupEvent(job.PodGroup, v1.EventTypeNormal, string(scheduling.PodGroupScheduled), string(scheduling.PodGroupReady))
        }

        baseErrorMessage := job.JobFitErrors
        if baseErrorMessage == "" {
                baseErrorMessage = schedulingapi.AllNodeUnavailableMsg
        }
        // Update podCondition for tasks Allocated and Pending before job discarded
        for _, status := range []schedulingapi.TaskStatus{schedulingapi.Allocated, schedulingapi.Pending, schedulingapi.Pipelined} {
                for _, taskInfo := range job.TaskStatusIndex[status] {
                        reason, msg := job.TaskSchedulingReason(taskInfo.UID)
                        if len(msg) == 0 {
                                msg = baseErrorMessage
                        }
                        if err := sc.taskUnschedulable(taskInfo, reason, msg); err != nil {
                                klog.Errorf("Failed to update unschedulable task status <%s/%s>: %v",
                                        taskInfo.Namespace, taskInfo.Name, err)
                        }
                }
        }
}

// UpdateJobStatus update the status of job and its tasks.
func (sc *SchedulerCache) UpdateJobStatus(job *schedulingapi.JobInfo, updatePG bool) (*schedulingapi.JobInfo, error) {
        if updatePG {
                pg, err := sc.StatusUpdater.UpdatePodGroup(job.PodGroup)
                if err != nil {
                        return nil, err
                }
                job.PodGroup = pg
        }

        sc.RecordJobStatusEvent(job)

        return job, nil
}

func (sc *SchedulerCache) recordPodGroupEvent(podGroup *schedulingapi.PodGroup, eventType, reason, msg string) {
        if podGroup == nil {
                return
        }

        pg := &vcv1beta1.PodGroup{}
        if err := schedulingscheme.Scheme.Convert(&podGroup.PodGroup, pg, nil); err != nil {
                klog.Errorf("Error while converting PodGroup to v1alpha1.PodGroup with error: %v", err)
                return
        }
        sc.Recorder.Eventf(pg, eventType, reason, msg)
}

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
        "context"
        "fmt"
        "strconv"

        v1 "k8s.io/api/core/v1"
        "k8s.io/api/scheduling/v1beta1"
        "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/client-go/tools/cache"
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"

        nodeinfov1alpha1 "volcano.sh/apis/pkg/apis/nodeinfo/v1alpha1"
        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/apis/pkg/apis/scheduling/scheme"
        schedulingv1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/apis/pkg/apis/utils"
        schedulingapi "volcano.sh/volcano/pkg/scheduler/api"
)

func isTerminated(status schedulingapi.TaskStatus) bool {
        return status == schedulingapi.Succeeded || status == schedulingapi.Failed
}

// getOrCreateJob will return corresponding Job for pi if it exists, or it will create a Job and return it if
// pi.Pod.Spec.SchedulerName is same as volcano scheduler's name, otherwise it will return nil.
func (sc *SchedulerCache) getOrCreateJob(pi *schedulingapi.TaskInfo) *schedulingapi.JobInfo {
        if len(pi.Job) == 0 {
                if pi.Pod.Spec.SchedulerName != sc.schedulerName {
                        klog.V(4).Infof("Pod %s/%s will not not scheduled by %s, skip creating PodGroup and Job for it",
                                pi.Pod.Namespace, pi.Pod.Name, sc.schedulerName)
                }
                return nil
        }

        if _, found := sc.Jobs[pi.Job]; !found {
                sc.Jobs[pi.Job] = schedulingapi.NewJobInfo(pi.Job)
        }

        return sc.Jobs[pi.Job]
}

func (sc *SchedulerCache) addTask(pi *schedulingapi.TaskInfo) error {
        if len(pi.NodeName) != 0 {
                if _, found := sc.Nodes[pi.NodeName]; !found {
                        sc.Nodes[pi.NodeName] = schedulingapi.NewNodeInfo(nil)
                        sc.Nodes[pi.NodeName].Name = pi.NodeName
                }

                node := sc.Nodes[pi.NodeName]
                if !isTerminated(pi.Status) {
                        if err := node.AddTask(pi); err != nil {
                                return err
                        }
                } else {
                        klog.V(4).Infof("Pod <%v/%v> is in status %s.", pi.Namespace, pi.Name, pi.Status.String())
                }
        }

        job := sc.getOrCreateJob(pi)
        if job != nil {
                job.AddTaskInfo(pi)
        }

        return nil
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) addPod(pod *v1.Pod) error {
        pi := schedulingapi.NewTaskInfo(pod)

        return sc.addTask(pi)
}

func (sc *SchedulerCache) syncTask(oldTask *schedulingapi.TaskInfo) error {
        newPod, err := sc.kubeClient.CoreV1().Pods(oldTask.Namespace).Get(context.TODO(), oldTask.Name, metav1.GetOptions{})
        if err != nil {
                if errors.IsNotFound(err) {
                        err := sc.deleteTask(oldTask)
                        if err != nil {
                                klog.Errorf("Failed to delete Pod <%v/%v> and remove from cache: %s", oldTask.Namespace, oldTask.Name, err.Error())
                                return err
                        }
                        klog.V(3).Infof("Pod <%v/%v> was deleted, removed from cache.", oldTask.Namespace, oldTask.Name)

                        return nil
                }
                return fmt.Errorf("failed to get Pod <%v/%v>: err %v", oldTask.Namespace, oldTask.Name, err)
        }

        newTask := schedulingapi.NewTaskInfo(newPod)

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()
        return sc.updateTask(oldTask, newTask)
}

func (sc *SchedulerCache) updateTask(oldTask, newTask *schedulingapi.TaskInfo) error {
        if err := sc.deleteTask(oldTask); err != nil {
                klog.Warningf("Failed to delete task: %v", err)
        }

        return sc.addTask(newTask)
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) updatePod(oldPod, newPod *v1.Pod) error {
        if err := sc.deletePod(oldPod); err != nil {
                return err
        }
        //when delete pod, the ownerreference of pod will be set nil,just as orphan pod
        if len(utils.GetController(newPod)) == 0 {
                newPod.OwnerReferences = oldPod.OwnerReferences
        }
        return sc.addPod(newPod)
}

func (sc *SchedulerCache) deleteTask(pi *schedulingapi.TaskInfo) error {
        var jobErr, nodeErr, numaErr error

        if len(pi.Job) != 0 {
                if job, found := sc.Jobs[pi.Job]; found {
                        jobErr = job.DeleteTaskInfo(pi)
                } else {
                        jobErr = fmt.Errorf("failed to find Job <%v> for Task %v/%v",
                                pi.Job, pi.Namespace, pi.Name)
                }
        }

        if len(pi.NodeName) != 0 {
                node := sc.Nodes[pi.NodeName]
                if node != nil {
                        nodeErr = node.RemoveTask(pi)
                }
        }

        if jobErr != nil || nodeErr != nil {
                return schedulingapi.MergeErrors(jobErr, nodeErr, numaErr)
        }

        return nil
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) deletePod(pod *v1.Pod) error {
        pi := schedulingapi.NewTaskInfo(pod)

        // Delete the Task in cache to handle Binding status.
        task := pi
        if job, found := sc.Jobs[pi.Job]; found {
                if t, found := job.Tasks[pi.UID]; found {
                        task = t
                }
        }
        if err := sc.deleteTask(task); err != nil {
                klog.Warningf("Failed to delete task: %v", err)
        }

        // If job was terminated, delete it.
        if job, found := sc.Jobs[pi.Job]; found && schedulingapi.JobTerminated(job) {
                sc.deleteJob(job)
        }

        return nil
}

// AddPod add pod to scheduler cache
func (sc *SchedulerCache) AddPod(obj interface{}) {
        pod, ok := obj.(*v1.Pod)
        if !ok {
                klog.Errorf("Cannot convert to *v1.Pod: %v", obj)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.addPod(pod)
        if err != nil {
                klog.Errorf("Failed to add pod <%s/%s> into cache: %v",
                        pod.Namespace, pod.Name, err)
                return
        }
        klog.V(3).Infof("Added pod <%s/%v> into cache.", pod.Namespace, pod.Name)
}

// UpdatePod update pod to scheduler cache
func (sc *SchedulerCache) UpdatePod(oldObj, newObj interface{}) {
        oldPod, ok := oldObj.(*v1.Pod)
        if !ok {
                klog.Errorf("Cannot convert oldObj to *v1.Pod: %v", oldObj)
                return
        }
        newPod, ok := newObj.(*v1.Pod)
        if !ok {
                klog.Errorf("Cannot convert newObj to *v1.Pod: %v", newObj)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.updatePod(oldPod, newPod)
        if err != nil {
                klog.Errorf("Failed to update pod %v in cache: %v", oldPod.Name, err)
                return
        }

        klog.V(4).Infof("Updated pod <%s/%v> in cache.", oldPod.Namespace, oldPod.Name)
}

// DeletePod delete pod from scheduler cache
func (sc *SchedulerCache) DeletePod(obj interface{}) {
        var pod *v1.Pod
        switch t := obj.(type) {
        case *v1.Pod:
                pod = t
        case cache.DeletedFinalStateUnknown:
                var ok bool
                pod, ok = t.Obj.(*v1.Pod)
                if !ok {
                        klog.Errorf("Cannot convert to *v1.Pod: %v", t.Obj)
                        return
                }
        default:
                klog.Errorf("Cannot convert to *v1.Pod: %v", t)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.deletePod(pod)
        if err != nil {
                klog.Errorf("Failed to delete pod %v from cache: %v", pod.Name, err)
                return
        }

        klog.V(3).Infof("Deleted pod <%s/%v> from cache.", pod.Namespace, pod.Name)
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) addNode(node *v1.Node) error {
        if sc.Nodes[node.Name] != nil {
                sc.Nodes[node.Name].SetNode(node)
        } else {
                sc.Nodes[node.Name] = schedulingapi.NewNodeInfo(node)
        }
        return nil
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) updateNode(oldNode, newNode *v1.Node) error {
        if sc.Nodes[newNode.Name] != nil {
                sc.Nodes[newNode.Name].SetNode(newNode)
                return nil
        }

        return fmt.Errorf("node <%s> does not exist", newNode.Name)
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) deleteNode(node *v1.Node) error {
        if _, ok := sc.Nodes[node.Name]; !ok {
                return fmt.Errorf("node <%s> does not exist", node.Name)
        }

        numaInfo := sc.Nodes[node.Name].NumaInfo
        if numaInfo != nil {
                klog.V(3).Infof("delete numatopo <%s/%s>", numaInfo.Namespace, numaInfo.Name)
                err := sc.vcClient.NodeinfoV1alpha1().Numatopologies().Delete(context.TODO(), numaInfo.Name, metav1.DeleteOptions{})
                if err != nil {
                        klog.Errorf("delete numatopo <%s/%s> failed.", numaInfo.Namespace, numaInfo.Name)
                }
        }

        delete(sc.Nodes, node.Name)

        return nil
}

// AddNode add node to scheduler cache
func (sc *SchedulerCache) AddNode(obj interface{}) {
        node, ok := obj.(*v1.Node)
        if !ok {
                klog.Errorf("Cannot convert to *v1.Node: %v", obj)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.addNode(node)
        if err != nil {
                klog.Errorf("Failed to add node %s into cache: %v", node.Name, err)
                return
        }
        sc.NodeList = append(sc.NodeList, node.Name)
}

// UpdateNode update node to scheduler cache
func (sc *SchedulerCache) UpdateNode(oldObj, newObj interface{}) {
        oldNode, ok := oldObj.(*v1.Node)
        if !ok {
                klog.Errorf("Cannot convert oldObj to *v1.Node: %v", oldObj)
                return
        }
        newNode, ok := newObj.(*v1.Node)
        if !ok {
                klog.Errorf("Cannot convert newObj to *v1.Node: %v", newObj)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.updateNode(oldNode, newNode)
        if err != nil {
                klog.Errorf("Failed to update node %v in cache: %v", oldNode.Name, err)
                return
        }
}

// DeleteNode delete node from scheduler cache
func (sc *SchedulerCache) DeleteNode(obj interface{}) {
        var node *v1.Node
        switch t := obj.(type) {
        case *v1.Node:
                node = t
        case cache.DeletedFinalStateUnknown:
                var ok bool
                node, ok = t.Obj.(*v1.Node)
                if !ok {
                        klog.Errorf("Cannot convert to *v1.Node: %v", t.Obj)
                        return
                }
        default:
                klog.Errorf("Cannot convert to *v1.Node: %v", t)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        err := sc.deleteNode(node)
        if err != nil {
                klog.Errorf("Failed to delete node %s from cache: %v", node.Name, err)
                return
        }

        for i, name := range sc.NodeList {
                if name == node.Name {
                        sc.NodeList = append(sc.NodeList[:i], sc.NodeList[i+1:]...)
                        break
                }
        }
}

func getJobID(pg *schedulingapi.PodGroup) schedulingapi.JobID {
        return schedulingapi.JobID(fmt.Sprintf("%s/%s", pg.Namespace, pg.Name))
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) setPodGroup(ss *schedulingapi.PodGroup) error {
        job := getJobID(ss)
        if _, found := sc.Jobs[job]; !found {
                sc.Jobs[job] = schedulingapi.NewJobInfo(job)
        }

        sc.Jobs[job].SetPodGroup(ss)

        // TODO(k82cn): set default queue in admission.
        if len(ss.Spec.Queue) == 0 {
                sc.Jobs[job].Queue = schedulingapi.QueueID(sc.defaultQueue)
        }

        return nil
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) updatePodGroup(newPodGroup *schedulingapi.PodGroup) error {
        return sc.setPodGroup(newPodGroup)
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) deletePodGroup(id schedulingapi.JobID) error {
        job, found := sc.Jobs[id]
        if !found {
                return fmt.Errorf("can not found job %v", id)
        }

        // Unset SchedulingSpec
        job.UnsetPodGroup()

        sc.deleteJob(job)

        return nil
}

// AddPodGroupV1beta1 add podgroup to scheduler cache
func (sc *SchedulerCache) AddPodGroupV1beta1(obj interface{}) {
        ss, ok := obj.(*schedulingv1.PodGroup)
        if !ok {
                klog.Errorf("Cannot convert to *schedulingv1.PodGroup: %v", obj)
                return
        }

        podgroup := scheduling.PodGroup{}
        if err := scheme.Scheme.Convert(ss, &podgroup, nil); err != nil {
                klog.Errorf("Failed to convert podgroup from %T to %T", ss, podgroup)
                return
        }

        pg := &schedulingapi.PodGroup{PodGroup: podgroup, Version: schedulingapi.PodGroupVersionV1Beta1}
        klog.V(4).Infof("Add PodGroup(%s) into cache, spec(%#v)", ss.Name, ss.Spec)

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        if err := sc.setPodGroup(pg); err != nil {
                klog.Errorf("Failed to add PodGroup %s into cache: %v", ss.Name, err)
                return
        }
}

// UpdatePodGroupV1beta1 add podgroup to scheduler cache
func (sc *SchedulerCache) UpdatePodGroupV1beta1(oldObj, newObj interface{}) {
        oldSS, ok := oldObj.(*schedulingv1.PodGroup)
        if !ok {
                klog.Errorf("Cannot convert oldObj to *schedulingv1.SchedulingSpec: %v", oldObj)
                return
        }
        newSS, ok := newObj.(*schedulingv1.PodGroup)
        if !ok {
                klog.Errorf("Cannot convert newObj to *schedulingv1.SchedulingSpec: %v", newObj)
                return
        }

        if oldSS.ResourceVersion == newSS.ResourceVersion {
                return
        }

        podgroup := scheduling.PodGroup{}
        if err := scheme.Scheme.Convert(newSS, &podgroup, nil); err != nil {
                klog.Errorf("Failed to convert podgroup from %T to %T", newSS, podgroup)
                return
        }

        pg := &schedulingapi.PodGroup{PodGroup: podgroup, Version: schedulingapi.PodGroupVersionV1Beta1}

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        if err := sc.updatePodGroup(pg); err != nil {
                klog.Errorf("Failed to update SchedulingSpec %s into cache: %v", pg.Name, err)
                return
        }
}

// DeletePodGroupV1beta1 delete podgroup from scheduler cache
func (sc *SchedulerCache) DeletePodGroupV1beta1(obj interface{}) {
        var ss *schedulingv1.PodGroup
        switch t := obj.(type) {
        case *schedulingv1.PodGroup:
                ss = t
        case cache.DeletedFinalStateUnknown:
                var ok bool
                ss, ok = t.Obj.(*schedulingv1.PodGroup)
                if !ok {
                        klog.Errorf("Cannot convert to podgroup: %v", t.Obj)
                        return
                }
        default:
                klog.Errorf("Cannot convert to podgroup: %v", t)
                return
        }

        jobID := schedulingapi.JobID(fmt.Sprintf("%s/%s", ss.Namespace, ss.Name))

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        if err := sc.deletePodGroup(jobID); err != nil {
                klog.Errorf("Failed to delete podgroup %s from cache: %v", ss.Name, err)
                return
        }
}

// AddQueueV1beta1 add queue to scheduler cache
func (sc *SchedulerCache) AddQueueV1beta1(obj interface{}) {
        ss, ok := obj.(*schedulingv1.Queue)
        if !ok {
                klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", obj)
                return
        }

        queue := &scheduling.Queue{}
        if err := scheme.Scheme.Convert(ss, queue, nil); err != nil {
                klog.Errorf("Failed to convert queue from %T to %T", ss, queue)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        klog.V(4).Infof("Add Queue(%s) into cache, spec(%#v)", ss.Name, ss.Spec)
        sc.addQueue(queue)
}

// UpdateQueueV1beta1 update queue to scheduler cache
func (sc *SchedulerCache) UpdateQueueV1beta1(oldObj, newObj interface{}) {
        oldSS, ok := oldObj.(*schedulingv1.Queue)
        if !ok {
                klog.Errorf("Cannot convert oldObj to *schedulingv1.Queue: %v", oldObj)
                return
        }
        newSS, ok := newObj.(*schedulingv1.Queue)
        if !ok {
                klog.Errorf("Cannot convert newObj to *schedulingv1.Queue: %v", newObj)
                return
        }

        if oldSS.ResourceVersion == newSS.ResourceVersion {
                return
        }

        newQueue := &scheduling.Queue{}
        if err := scheme.Scheme.Convert(newSS, newQueue, nil); err != nil {
                klog.Errorf("Failed to convert queue from %T to %T", newSS, newQueue)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()
        sc.updateQueue(newQueue)
}

// DeleteQueueV1beta1 delete queue from the scheduler cache
func (sc *SchedulerCache) DeleteQueueV1beta1(obj interface{}) {
        var ss *schedulingv1.Queue
        switch t := obj.(type) {
        case *schedulingv1.Queue:
                ss = t
        case cache.DeletedFinalStateUnknown:
                var ok bool
                ss, ok = t.Obj.(*schedulingv1.Queue)
                if !ok {
                        klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t.Obj)
                        return
                }
        default:
                klog.Errorf("Cannot convert to *schedulingv1.Queue: %v", t)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()
        sc.deleteQueue(schedulingapi.QueueID(ss.Name))
}

func (sc *SchedulerCache) addQueue(queue *scheduling.Queue) {
        qi := schedulingapi.NewQueueInfo(queue)
        sc.Queues[qi.UID] = qi
}

func (sc *SchedulerCache) updateQueue(queue *scheduling.Queue) {
        sc.addQueue(queue)
}

func (sc *SchedulerCache) deleteQueue(id schedulingapi.QueueID) {
        delete(sc.Queues, id)
}

//DeletePriorityClass delete priorityclass from the scheduler cache
func (sc *SchedulerCache) DeletePriorityClass(obj interface{}) {
        var ss *v1beta1.PriorityClass
        switch t := obj.(type) {
        case *v1beta1.PriorityClass:
                ss = t
        case cache.DeletedFinalStateUnknown:
                var ok bool
                ss, ok = t.Obj.(*v1beta1.PriorityClass)
                if !ok {
                        klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj)
                        return
                }
        default:
                klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        sc.deletePriorityClass(ss)
}

//UpdatePriorityClass update priorityclass to scheduler cache
func (sc *SchedulerCache) UpdatePriorityClass(oldObj, newObj interface{}) {
        oldSS, ok := oldObj.(*v1beta1.PriorityClass)
        if !ok {
                klog.Errorf("Cannot convert oldObj to *v1beta1.PriorityClass: %v", oldObj)

                return
        }

        newSS, ok := newObj.(*v1beta1.PriorityClass)
        if !ok {
                klog.Errorf("Cannot convert newObj to *v1beta1.PriorityClass: %v", newObj)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        sc.deletePriorityClass(oldSS)
        sc.addPriorityClass(newSS)
}

//AddPriorityClass add priorityclass to scheduler cache
func (sc *SchedulerCache) AddPriorityClass(obj interface{}) {
        var ss *v1beta1.PriorityClass
        switch t := obj.(type) {
        case *v1beta1.PriorityClass:
                ss = t
        case cache.DeletedFinalStateUnknown:
                var ok bool
                ss, ok = t.Obj.(*v1beta1.PriorityClass)
                if !ok {
                        klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t.Obj)
                        return
                }
        default:
                klog.Errorf("Cannot convert to *v1beta1.PriorityClass: %v", t)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        sc.addPriorityClass(ss)
}

func (sc *SchedulerCache) deletePriorityClass(pc *v1beta1.PriorityClass) {
        if pc.GlobalDefault {
                sc.defaultPriorityClass = nil
                sc.defaultPriority = 0
        }

        delete(sc.PriorityClasses, pc.Name)
}

func (sc *SchedulerCache) addPriorityClass(pc *v1beta1.PriorityClass) {
        if pc.GlobalDefault {
                if sc.defaultPriorityClass != nil {
                        klog.Errorf("Updated default priority class from <%s> to <%s> forcefully.",
                                sc.defaultPriorityClass.Name, pc.Name)
                }
                sc.defaultPriorityClass = pc
                sc.defaultPriority = pc.Value
        }

        sc.PriorityClasses[pc.Name] = pc
}

func (sc *SchedulerCache) updateResourceQuota(quota *v1.ResourceQuota) {
        collection, ok := sc.NamespaceCollection[quota.Namespace]
        if !ok {
                collection = schedulingapi.NewNamespaceCollection(quota.Namespace)
                sc.NamespaceCollection[quota.Namespace] = collection
        }

        collection.Update(quota)
}

func (sc *SchedulerCache) deleteResourceQuota(quota *v1.ResourceQuota) {
        collection, ok := sc.NamespaceCollection[quota.Namespace]
        if !ok {
                return
        }

        collection.Delete(quota)
}

// DeleteResourceQuota delete ResourceQuota from the scheduler cache
func (sc *SchedulerCache) DeleteResourceQuota(obj interface{}) {
        var r *v1.ResourceQuota
        switch t := obj.(type) {
        case *v1.ResourceQuota:
                r = t
        case cache.DeletedFinalStateUnknown:
                var ok bool
                r, ok = t.Obj.(*v1.ResourceQuota)
                if !ok {
                        klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t.Obj)
                        return
                }
        default:
                klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        klog.V(3).Infof("Delete ResourceQuota <%s/%v> in cache", r.Namespace, r.Name)
        sc.deleteResourceQuota(r)
}

// UpdateResourceQuota update ResourceQuota to scheduler cache
func (sc *SchedulerCache) UpdateResourceQuota(oldObj, newObj interface{}) {
        newR, ok := newObj.(*v1.ResourceQuota)
        if !ok {
                klog.Errorf("Cannot convert newObj to *v1.ResourceQuota: %v", newObj)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        klog.V(3).Infof("Update ResourceQuota <%s/%v> in cache, with spec: %v.", newR.Namespace, newR.Name, newR.Spec.Hard)
        sc.updateResourceQuota(newR)
}

// AddResourceQuota add ResourceQuota to scheduler cache
func (sc *SchedulerCache) AddResourceQuota(obj interface{}) {
        var r *v1.ResourceQuota
        switch t := obj.(type) {
        case *v1.ResourceQuota:
                r = t
        default:
                klog.Errorf("Cannot convert to *v1.ResourceQuota: %v", t)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        klog.V(3).Infof("Add ResourceQuota <%s/%v> in cache, with spec: %v.", r.Namespace, r.Name, r.Spec.Hard)
        sc.updateResourceQuota(r)
}

func getNumaInfo(srcInfo *nodeinfov1alpha1.Numatopology) *schedulingapi.NumatopoInfo {
        numaInfo := &schedulingapi.NumatopoInfo{
                Namespace:   srcInfo.Namespace,
                Name:        srcInfo.Name,
                Policies:    make(map[nodeinfov1alpha1.PolicyName]string),
                NumaResMap:  make(map[string]*schedulingapi.ResourceInfo),
                CPUDetail:   topology.CPUDetails{},
                ResReserved: make(v1.ResourceList),
        }

        policies := srcInfo.Spec.Policies
        for name, policy := range policies {
                numaInfo.Policies[name] = policy
        }

        numaResMap := srcInfo.Spec.NumaResMap
        for name, resInfo := range numaResMap {
                tmp := schedulingapi.ResourceInfo{}
                tmp.Capacity = resInfo.Capacity
                tmp.Allocatable = cpuset.MustParse(resInfo.Allocatable)
                numaInfo.NumaResMap[name] = &tmp
        }

        cpuDetail := srcInfo.Spec.CPUDetail
        for key, detail := range cpuDetail {
                cpuID, _ := strconv.Atoi(key)
                numaInfo.CPUDetail[cpuID] = topology.CPUInfo{
                        NUMANodeID: detail.NUMANodeID,
                        SocketID:   detail.SocketID,
                        CoreID:     detail.CoreID,
                }
        }

        resReserved, err := schedulingapi.ParseResourceList(srcInfo.Spec.ResReserved)
        if err != nil {
                klog.Errorf("ParseResourceList failed, err=%v", err)
        } else {
                numaInfo.ResReserved = resReserved
        }

        return numaInfo
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) addNumaInfo(info *nodeinfov1alpha1.Numatopology) error {
        if sc.Nodes[info.Name] == nil {
                sc.Nodes[info.Name] = schedulingapi.NewNodeInfo(nil)
                sc.Nodes[info.Name].Name = info.Name
        }

        if sc.Nodes[info.Name].NumaInfo == nil {
                sc.Nodes[info.Name].NumaInfo = getNumaInfo(info)
        }

        newLocalInfo := getNumaInfo(info)
        if sc.Nodes[info.Name].NumaInfo.Compare(newLocalInfo) {
                sc.Nodes[info.Name].NumaChgFlag = schedulingapi.NumaInfoMoreFlag
        } else {
                sc.Nodes[info.Name].NumaChgFlag = schedulingapi.NumaInfoLessFlag
        }

        sc.Nodes[info.Name].NumaInfo = newLocalInfo

        for resName, NumaResInfo := range sc.Nodes[info.Name].NumaInfo.NumaResMap {
                klog.V(3).Infof("resource %s Allocatable %v on node[%s] into cache", resName, NumaResInfo, info.Name)
        }

        klog.V(3).Infof("Policies %v on node[%s] into cache, change= %v",
                sc.Nodes[info.Name].NumaInfo.Policies, info.Name, sc.Nodes[info.Name].NumaChgFlag)
        return nil
}

// Assumes that lock is already acquired.
func (sc *SchedulerCache) deleteNumaInfo(info *nodeinfov1alpha1.Numatopology) {
        if sc.Nodes[info.Name] != nil {
                sc.Nodes[info.Name].NumaInfo = nil
                sc.Nodes[info.Name].NumaChgFlag = schedulingapi.NumaInfoResetFlag
                klog.V(3).Infof("delete numainfo in cahce for node<%s>", info.Name)
        }
}

// AddNumaInfoV1alpha1 add numa information to scheduler cache
func (sc *SchedulerCache) AddNumaInfoV1alpha1(obj interface{}) {
        ss, ok := obj.(*nodeinfov1alpha1.Numatopology)
        if !ok {
                klog.Errorf("Cannot convert oldObj to *nodeinfov1alpha1.Numatopology: %v", obj)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        sc.addNumaInfo(ss)
}

// UpdateNumaInfoV1alpha1 update numa information to scheduler cache
func (sc *SchedulerCache) UpdateNumaInfoV1alpha1(oldObj, newObj interface{}) {
        ss, ok := newObj.(*nodeinfov1alpha1.Numatopology)
        if !ok {
                klog.Errorf("Cannot convert oldObj to *nodeinfov1alpha1.Numatopology: %v", newObj)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()
        sc.addNumaInfo(ss)
        klog.V(3).Infof("update numaInfo<%s> in cahce, with spec: Policy: %v, resMap: %v", ss.Name, ss.Spec.Policies, ss.Spec.NumaResMap)
}

// DeleteNumaInfoV1alpha1 delete numa information from scheduler cache
func (sc *SchedulerCache) DeleteNumaInfoV1alpha1(obj interface{}) {
        var ss *nodeinfov1alpha1.Numatopology
        switch t := obj.(type) {
        case *nodeinfov1alpha1.Numatopology:
                ss = t
        case cache.DeletedFinalStateUnknown:
                var ok bool
                ss, ok = t.Obj.(*nodeinfov1alpha1.Numatopology)
                if !ok {
                        klog.Errorf("Cannot convert to Numatopo: %v", t.Obj)
                        return
                }
        default:
                klog.Errorf("Cannot convert to Numatopo: %v", t)
                return
        }

        sc.Mutex.Lock()
        defer sc.Mutex.Unlock()

        sc.deleteNumaInfo(ss)
        klog.V(3).Infof("Delete numaInfo<%s> from cahce, with spec: Policy: %v, resMap: %v", ss.Name, ss.Spec.Policies, ss.Spec.NumaResMap)
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import v1 "k8s.io/api/core/v1"

// responsibleForPod returns true if the pod has asked to be scheduled by the given scheduler.
func responsibleForPod(pod *v1.Pod, schedulerName string) bool {
        return schedulerName == pod.Spec.SchedulerName
}

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        "strconv"

        "volcano.sh/volcano/pkg/scheduler/conf"

        "k8s.io/klog"
)

// Arguments map
type Arguments map[string]string

// GetInt get the integer value from string
func (a Arguments) GetInt(ptr *int, key string) {
        if ptr == nil {
                return
        }

        argv, ok := a[key]
        if !ok || argv == "" {
                return
        }

        value, err := strconv.Atoi(argv)
        if err != nil {
                klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err)
                return
        }

        *ptr = value
}

// GetFloat64 get the float64 value from string
func (a Arguments) GetFloat64(ptr *float64, key string) {
        if ptr == nil {
                return
        }

        argv, ok := a[key]
        if !ok || len(argv) == 0 {
                return
        }

        value, err := strconv.ParseFloat(argv, 64)
        if err != nil {
                klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err)
                return
        }

        *ptr = value
}

// GetBool get the bool value from string
func (a Arguments) GetBool(ptr *bool, key string) {
        if ptr == nil {
                return
        }

        argv, ok := a[key]
        if !ok || argv == "" {
                return
        }

        value, err := strconv.ParseBool(argv)
        if err != nil {
                klog.Warningf("Could not parse argument: %s for key %s, with err %v", argv, key, err)
                return
        }

        *ptr = value
}

// GetArgOfActionFromConf return argument of action reading from configuration of schedule
func GetArgOfActionFromConf(configurations []conf.Configuration, actionName string) Arguments {
        for _, c := range configurations {
                if c.Name == actionName {
                        return c.Arguments
                }
        }

        return nil
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        "time"

        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/cache"
        "volcano.sh/volcano/pkg/scheduler/conf"
        "volcano.sh/volcano/pkg/scheduler/metrics"
)

// OpenSession start the session
func OpenSession(cache cache.Cache, tiers []conf.Tier, configurations []conf.Configuration) *Session {
        ssn := openSession(cache)
        ssn.Tiers = tiers
        ssn.Configurations = configurations

        for _, tier := range tiers {
                for _, plugin := range tier.Plugins {
                        if pb, found := GetPluginBuilder(plugin.Name); !found {
                                klog.Errorf("Failed to get plugin %s.", plugin.Name)
                        } else {
                                plugin := pb(plugin.Arguments)
                                ssn.plugins[plugin.Name()] = plugin
                                onSessionOpenStart := time.Now()
                                plugin.OnSessionOpen(ssn)
                                metrics.UpdatePluginDuration(plugin.Name(), metrics.OnSessionOpen, metrics.Duration(onSessionOpenStart))
                        }
                }
        }
        return ssn
}

// CloseSession close the session
func CloseSession(ssn *Session) {
        for _, plugin := range ssn.plugins {
                onSessionCloseStart := time.Now()
                plugin.OnSessionClose(ssn)
                metrics.UpdatePluginDuration(plugin.Name(), metrics.OnSessionClose, metrics.Duration(onSessionCloseStart))
        }

        closeSession(ssn)
}

package framework

import (
        "context"
        "math/rand"
        "reflect"
        "time"

        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/volcano/pkg/scheduler/api"
)

const (
        jobUpdaterWorker = 16

        jobConditionUpdateTime       = time.Minute
        jobConditionUpdateTimeJitter = 30 * time.Second
)

// TimeJitterAfter means: new after old + duration + jitter
func TimeJitterAfter(new, old time.Time, duration, maxJitter time.Duration) bool {
        var jitter int64
        if maxJitter > 0 {
                jitter = rand.Int63n(int64(maxJitter))
        }
        return new.After(old.Add(duration + time.Duration(jitter)))
}

type jobUpdater struct {
        ssn      *Session
        jobQueue []*api.JobInfo
}

func newJobUpdater(ssn *Session) *jobUpdater {
        queue := make([]*api.JobInfo, 0, len(ssn.Jobs))
        for _, job := range ssn.Jobs {
                queue = append(queue, job)
        }

        ju := &jobUpdater{
                ssn:      ssn,
                jobQueue: queue,
        }
        return ju
}

func (ju *jobUpdater) UpdateAll() {
        workqueue.ParallelizeUntil(context.TODO(), jobUpdaterWorker, len(ju.jobQueue), ju.updateJob)
}

func isPodGroupConditionsUpdated(newCondition, oldCondition []scheduling.PodGroupCondition) bool {
        if len(newCondition) != len(oldCondition) {
                return true
        }

        for index, newCond := range newCondition {
                oldCond := oldCondition[index]

                newTime := newCond.LastTransitionTime
                oldTime := oldCond.LastTransitionTime
                if TimeJitterAfter(newTime.Time, oldTime.Time, jobConditionUpdateTime, jobConditionUpdateTimeJitter) {
                        return true
                }

                // if newCond is not new enough, we treat it the same as the old one
                newCond.LastTransitionTime = oldTime

                // comparing should ignore the TransitionID
                newTransitionID := newCond.TransitionID
                newCond.TransitionID = oldCond.TransitionID

                shouldUpdate := !reflect.DeepEqual(&newCond, &oldCond)

                newCond.LastTransitionTime = newTime
                newCond.TransitionID = newTransitionID
                if shouldUpdate {
                        return true
                }
        }

        return false
}

func isPodGroupStatusUpdated(newStatus, oldStatus scheduling.PodGroupStatus) bool {
        newCondition := newStatus.Conditions
        newStatus.Conditions = nil
        oldCondition := oldStatus.Conditions
        oldStatus.Conditions = nil

        return !reflect.DeepEqual(newStatus, oldStatus) || isPodGroupConditionsUpdated(newCondition, oldCondition)
}

// updateJob update specified job
func (ju *jobUpdater) updateJob(index int) {
        job := ju.jobQueue[index]
        ssn := ju.ssn

        job.PodGroup.Status = jobStatus(ssn, job)
        oldStatus, found := ssn.podGroupStatus[job.UID]
        updatePG := !found || isPodGroupStatusUpdated(job.PodGroup.Status, oldStatus)
        if _, err := ssn.cache.UpdateJobStatus(job, updatePG); err != nil {
                klog.Errorf("Failed to update job <%s/%s>: %v",
                        job.Namespace, job.Name, err)
        }
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        "fmt"
        "path/filepath"
        "plugin"
        "strings"
        "sync"

        "k8s.io/klog"
)

var pluginMutex sync.Mutex

// PluginBuilder plugin management
type PluginBuilder = func(Arguments) Plugin

// Plugin management
var pluginBuilders = map[string]PluginBuilder{}

// RegisterPluginBuilder register the plugin
func RegisterPluginBuilder(name string, pc PluginBuilder) {
        pluginMutex.Lock()
        defer pluginMutex.Unlock()

        pluginBuilders[name] = pc
}

// CleanupPluginBuilders cleans up all the plugin
func CleanupPluginBuilders() {
        pluginMutex.Lock()
        defer pluginMutex.Unlock()

        pluginBuilders = map[string]PluginBuilder{}
}

// GetPluginBuilder get the pluginbuilder by name
func GetPluginBuilder(name string) (PluginBuilder, bool) {
        pluginMutex.Lock()
        defer pluginMutex.Unlock()

        pb, found := pluginBuilders[name]
        return pb, found
}

// LoadCustomPlugins loads custom implement plugins
func LoadCustomPlugins(pluginsDir string) error {
        pluginPaths, _ := filepath.Glob(fmt.Sprintf("%s/*.so", pluginsDir))
        for _, pluginPath := range pluginPaths {
                pluginBuilder, err := loadPluginBuilder(pluginPath)
                if err != nil {
                        return err
                }
                pluginName := getPluginName(pluginPath)
                RegisterPluginBuilder(pluginName, pluginBuilder)
                klog.V(4).Infof("Custom plugin %s loaded", pluginName)
        }

        return nil
}

func getPluginName(pluginPath string) string {
        return strings.TrimSuffix(filepath.Base(pluginPath), filepath.Ext(pluginPath))
}

func loadPluginBuilder(pluginPath string) (PluginBuilder, error) {
        plug, err := plugin.Open(pluginPath)
        if err != nil {
                return nil, err
        }

        symBuilder, err := plug.Lookup("New")
        if err != nil {
                return nil, err
        }

        builder, ok := symBuilder.(PluginBuilder)
        if !ok {
                return nil, fmt.Errorf("unexpected plugin: %s, failed to convert PluginBuilder `New`", pluginPath)
        }

        return builder, nil
}

// Action management
var actionMap = map[string]Action{}

// RegisterAction register action
func RegisterAction(act Action) {
        pluginMutex.Lock()
        defer pluginMutex.Unlock()

        actionMap[act.Name()] = act
}

// GetAction get the action by name
func GetAction(name string) (Action, bool) {
        pluginMutex.Lock()
        defer pluginMutex.Unlock()

        act, found := actionMap[name]
        return act, found
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        "fmt"

        v1 "k8s.io/api/core/v1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/apimachinery/pkg/util/uuid"
        "k8s.io/client-go/informers"
        "k8s.io/client-go/kubernetes"
        "k8s.io/klog"
        volumescheduling "k8s.io/kubernetes/pkg/controller/volume/scheduling"

        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/cache"
        "volcano.sh/volcano/pkg/scheduler/conf"
        "volcano.sh/volcano/pkg/scheduler/metrics"
        "volcano.sh/volcano/pkg/scheduler/util"
)

// Session information for the current session
type Session struct {
        UID types.UID

        kubeClient      kubernetes.Interface
        cache           cache.Cache
        informerFactory informers.SharedInformerFactory

        TotalResource *api.Resource
        // podGroupStatus cache podgroup status during schedule
        // This should not be mutated after initiated
        podGroupStatus map[api.JobID]scheduling.PodGroupStatus

        Jobs           map[api.JobID]*api.JobInfo
        Nodes          map[string]*api.NodeInfo
        RevocableNodes map[string]*api.NodeInfo
        Queues         map[api.QueueID]*api.QueueInfo
        NamespaceInfo  map[api.NamespaceName]*api.NamespaceInfo

        Tiers          []conf.Tier
        Configurations []conf.Configuration
        NodeList       []*api.NodeInfo

        plugins           map[string]Plugin
        eventHandlers     []*EventHandler
        jobOrderFns       map[string]api.CompareFn
        queueOrderFns     map[string]api.CompareFn
        taskOrderFns      map[string]api.CompareFn
        namespaceOrderFns map[string]api.CompareFn
        clusterOrderFns   map[string]api.CompareFn
        predicateFns      map[string]api.PredicateFn
        bestNodeFns       map[string]api.BestNodeFn
        nodeOrderFns      map[string]api.NodeOrderFn
        batchNodeOrderFns map[string]api.BatchNodeOrderFn
        nodeMapFns        map[string]api.NodeMapFn
        nodeReduceFns     map[string]api.NodeReduceFn
        preemptableFns    map[string]api.EvictableFn
        reclaimableFns    map[string]api.EvictableFn
        overusedFns       map[string]api.ValidateFn
        underUsedFns      map[string]api.UnderUsedResourceFn
        jobReadyFns       map[string]api.ValidateFn
        jobPipelinedFns   map[string]api.VoteFn
        jobValidFns       map[string]api.ValidateExFn
        jobEnqueueableFns map[string]api.VoteFn
        jobEnqueuedFns    map[string]api.JobEnqueuedFn
        targetJobFns      map[string]api.TargetJobFn
        reservedNodesFns  map[string]api.ReservedNodesFn
        victimTasksFns    map[string]api.VictimTasksFn
        jobStarvingFns    map[string]api.ValidateFn
}

func openSession(cache cache.Cache) *Session {
        ssn := &Session{
                UID:             uuid.NewUUID(),
                kubeClient:      cache.Client(),
                cache:           cache,
                informerFactory: cache.SharedInformerFactory(),

                TotalResource:  api.EmptyResource(),
                podGroupStatus: map[api.JobID]scheduling.PodGroupStatus{},

                Jobs:           map[api.JobID]*api.JobInfo{},
                Nodes:          map[string]*api.NodeInfo{},
                RevocableNodes: map[string]*api.NodeInfo{},
                Queues:         map[api.QueueID]*api.QueueInfo{},

                plugins:           map[string]Plugin{},
                jobOrderFns:       map[string]api.CompareFn{},
                queueOrderFns:     map[string]api.CompareFn{},
                taskOrderFns:      map[string]api.CompareFn{},
                namespaceOrderFns: map[string]api.CompareFn{},
                clusterOrderFns:   map[string]api.CompareFn{},
                predicateFns:      map[string]api.PredicateFn{},
                bestNodeFns:       map[string]api.BestNodeFn{},
                nodeOrderFns:      map[string]api.NodeOrderFn{},
                batchNodeOrderFns: map[string]api.BatchNodeOrderFn{},
                nodeMapFns:        map[string]api.NodeMapFn{},
                nodeReduceFns:     map[string]api.NodeReduceFn{},
                preemptableFns:    map[string]api.EvictableFn{},
                reclaimableFns:    map[string]api.EvictableFn{},
                overusedFns:       map[string]api.ValidateFn{},
                underUsedFns:      map[string]api.UnderUsedResourceFn{},
                jobReadyFns:       map[string]api.ValidateFn{},
                jobPipelinedFns:   map[string]api.VoteFn{},
                jobValidFns:       map[string]api.ValidateExFn{},
                jobEnqueueableFns: map[string]api.VoteFn{},
                jobEnqueuedFns:    map[string]api.JobEnqueuedFn{},
                targetJobFns:      map[string]api.TargetJobFn{},
                reservedNodesFns:  map[string]api.ReservedNodesFn{},
                victimTasksFns:    map[string]api.VictimTasksFn{},
                jobStarvingFns:    map[string]api.ValidateFn{},
        }

        snapshot := cache.Snapshot()

        ssn.Jobs = snapshot.Jobs
        for _, job := range ssn.Jobs {
                // only conditions will be updated periodically
                if job.PodGroup != nil && job.PodGroup.Status.Conditions != nil {
                        ssn.podGroupStatus[job.UID] = job.PodGroup.Status
                }

                if vjr := ssn.JobValid(job); vjr != nil {
                        if !vjr.Pass {
                                jc := &scheduling.PodGroupCondition{
                                        Type:               scheduling.PodGroupUnschedulableType,
                                        Status:             v1.ConditionTrue,
                                        LastTransitionTime: metav1.Now(),
                                        TransitionID:       string(ssn.UID),
                                        Reason:             vjr.Reason,
                                        Message:            vjr.Message,
                                }

                                if err := ssn.UpdatePodGroupCondition(job, jc); err != nil {
                                        klog.Errorf("Failed to update job condition: %v", err)
                                }
                        }

                        delete(ssn.Jobs, job.UID)
                }
        }
        ssn.NodeList = util.GetNodeList(snapshot.Nodes, snapshot.NodeList)
        ssn.Nodes = snapshot.Nodes
        ssn.RevocableNodes = snapshot.RevocableNodes
        ssn.Queues = snapshot.Queues
        ssn.NamespaceInfo = snapshot.NamespaceInfo
        // calculate all nodes' resource only once in each schedule cycle, other plugins can clone it when need
        for _, n := range ssn.Nodes {
                ssn.TotalResource.Add(n.Allocatable)
        }

        klog.V(3).Infof("Open Session %v with <%d> Job and <%d> Queues",
                ssn.UID, len(ssn.Jobs), len(ssn.Queues))

        return ssn
}

func closeSession(ssn *Session) {
        ju := newJobUpdater(ssn)
        ju.UpdateAll()

        ssn.Jobs = nil
        ssn.Nodes = nil
        ssn.RevocableNodes = nil
        ssn.plugins = nil
        ssn.eventHandlers = nil
        ssn.jobOrderFns = nil
        ssn.namespaceOrderFns = nil
        ssn.queueOrderFns = nil
        ssn.clusterOrderFns = nil
        ssn.NodeList = nil
        ssn.TotalResource = nil

        klog.V(3).Infof("Close Session %v", ssn.UID)
}

func jobStatus(ssn *Session, jobInfo *api.JobInfo) scheduling.PodGroupStatus {
        status := jobInfo.PodGroup.Status

        unschedulable := false
        for _, c := range status.Conditions {
                if c.Type == scheduling.PodGroupUnschedulableType &&
                        c.Status == v1.ConditionTrue &&
                        c.TransitionID == string(ssn.UID) {
                        unschedulable = true
                        break
                }
        }

        // If running tasks && unschedulable, unknown phase
        if len(jobInfo.TaskStatusIndex[api.Running]) != 0 && unschedulable {
                status.Phase = scheduling.PodGroupUnknown
        } else {
                allocated := 0
                for status, tasks := range jobInfo.TaskStatusIndex {
                        if api.AllocatedStatus(status) || status == api.Succeeded {
                                allocated += len(tasks)
                        }
                }

                // If there're enough allocated resource, it's running
                if int32(allocated) >= jobInfo.PodGroup.Spec.MinMember {
                        status.Phase = scheduling.PodGroupRunning
                } else if jobInfo.PodGroup.Status.Phase != scheduling.PodGroupInqueue {
                        status.Phase = scheduling.PodGroupPending
                }
        }

        status.Running = int32(len(jobInfo.TaskStatusIndex[api.Running]))
        status.Failed = int32(len(jobInfo.TaskStatusIndex[api.Failed]))
        status.Succeeded = int32(len(jobInfo.TaskStatusIndex[api.Succeeded]))

        return status
}

// Statement returns new statement object
func (ssn *Session) Statement() *Statement {
        return &Statement{
                ssn: ssn,
        }
}

// Pipeline  the task to the node in the session
func (ssn *Session) Pipeline(task *api.TaskInfo, hostname string) error {
        // Only update status in session
        job, found := ssn.Jobs[task.Job]
        if found {
                if err := job.UpdateTaskStatus(task, api.Pipelined); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                task.Namespace, task.Name, api.Pipelined, ssn.UID, err)
                        return err
                }
        } else {
                klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.",
                        task.Job, ssn.UID)
                return fmt.Errorf("failed to find job %s when binding", task.Job)
        }

        task.NodeName = hostname

        if node, found := ssn.Nodes[hostname]; found {
                if err := node.AddTask(task); err != nil {
                        klog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v",
                                task.Namespace, task.Name, hostname, ssn.UID, err)
                        return err
                }
                klog.V(3).Infof("After added Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>",
                        task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing)
        } else {
                klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.",
                        hostname, ssn.UID)
                return fmt.Errorf("failed to find node %s", hostname)
        }

        for _, eh := range ssn.eventHandlers {
                if eh.AllocateFunc != nil {
                        eh.AllocateFunc(&Event{
                                Task: task,
                        })
                }
        }

        return nil
}

//Allocate the task to the node in the session
func (ssn *Session) Allocate(task *api.TaskInfo, nodeInfo *api.NodeInfo) error {
        podVolumes, err := ssn.cache.GetPodVolumes(task, nodeInfo.Node)
        if err != nil {
                return err
        }

        hostname := nodeInfo.Name
        if err := ssn.cache.AllocateVolumes(task, hostname, podVolumes); err != nil {
                return err
        }

        task.Pod.Spec.NodeName = hostname
        task.PodVolumes = podVolumes

        // Only update status in session
        job, found := ssn.Jobs[task.Job]
        if found {
                if err := job.UpdateTaskStatus(task, api.Allocated); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                task.Namespace, task.Name, api.Allocated, ssn.UID, err)
                        return err
                }
        } else {
                klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.",
                        task.Job, ssn.UID)
                return fmt.Errorf("failed to find job %s", task.Job)
        }

        task.NodeName = hostname

        if node, found := ssn.Nodes[hostname]; found {
                if err := node.AddTask(task); err != nil {
                        klog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v",
                                task.Namespace, task.Name, hostname, ssn.UID, err)
                        return err
                }
                klog.V(3).Infof("After allocated Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>",
                        task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing)
        } else {
                klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.",
                        hostname, ssn.UID)
                return fmt.Errorf("failed to find node %s", hostname)
        }

        // Callbacks
        for _, eh := range ssn.eventHandlers {
                if eh.AllocateFunc != nil {
                        eh.AllocateFunc(&Event{
                                Task: task,
                        })
                }
        }

        if ssn.JobReady(job) {
                for _, task := range job.TaskStatusIndex[api.Allocated] {
                        if err := ssn.dispatch(task, podVolumes); err != nil {
                                klog.Errorf("Failed to dispatch task <%v/%v>: %v",
                                        task.Namespace, task.Name, err)
                                return err
                        }
                }
        }

        return nil
}

func (ssn *Session) dispatch(task *api.TaskInfo, volumes *volumescheduling.PodVolumes) error {
        if err := ssn.cache.BindVolumes(task, volumes); err != nil {
                return err
        }

        if err := ssn.cache.Bind(task, task.NodeName); err != nil {
                return err
        }

        // Update status in session
        if job, found := ssn.Jobs[task.Job]; found {
                if err := job.UpdateTaskStatus(task, api.Binding); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                task.Namespace, task.Name, api.Binding, ssn.UID, err)
                        return err
                }
        } else {
                klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.",
                        task.Job, ssn.UID)
                return fmt.Errorf("failed to find job %s", task.Job)
        }

        metrics.UpdateTaskScheduleDuration(metrics.Duration(task.Pod.CreationTimestamp.Time))
        return nil
}

//Evict the task in the session
func (ssn *Session) Evict(reclaimee *api.TaskInfo, reason string) error {
        if err := ssn.cache.Evict(reclaimee, reason); err != nil {
                return err
        }

        // Update status in session
        job, found := ssn.Jobs[reclaimee.Job]
        if found {
                if err := job.UpdateTaskStatus(reclaimee, api.Releasing); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                reclaimee.Namespace, reclaimee.Name, api.Releasing, ssn.UID, err)
                        return err
                }
        } else {
                klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.",
                        reclaimee.Job, ssn.UID)
                return fmt.Errorf("failed to find job %s", reclaimee.Job)
        }

        // Update task in node.
        if node, found := ssn.Nodes[reclaimee.NodeName]; found {
                if err := node.UpdateTask(reclaimee); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> in Session <%v>: %v",
                                reclaimee.Namespace, reclaimee.Name, ssn.UID, err)
                        return err
                }
        }

        for _, eh := range ssn.eventHandlers {
                if eh.DeallocateFunc != nil {
                        eh.DeallocateFunc(&Event{
                                Task: reclaimee,
                        })
                }
        }

        return nil
}

// BindPodGroup bind PodGroup to specified cluster
func (ssn *Session) BindPodGroup(job *api.JobInfo, cluster string) error {
        return ssn.cache.BindPodGroup(job, cluster)
}

// UpdatePodGroupCondition update job condition accordingly.
func (ssn *Session) UpdatePodGroupCondition(jobInfo *api.JobInfo, cond *scheduling.PodGroupCondition) error {
        job, ok := ssn.Jobs[jobInfo.UID]
        if !ok {
                return fmt.Errorf("failed to find job <%s/%s>", jobInfo.Namespace, jobInfo.Name)
        }

        index := -1
        for i, c := range job.PodGroup.Status.Conditions {
                if c.Type == cond.Type {
                        index = i
                        break
                }
        }

        // Update condition to the new condition.
        if index < 0 {
                job.PodGroup.Status.Conditions = append(job.PodGroup.Status.Conditions, *cond)
        } else {
                job.PodGroup.Status.Conditions[index] = *cond
        }

        return nil
}

// AddEventHandler add event handlers
func (ssn *Session) AddEventHandler(eh *EventHandler) {
        ssn.eventHandlers = append(ssn.eventHandlers, eh)
}

// UpdateSchedulerNumaInfo update SchedulerNumaInfo
func (ssn *Session) UpdateSchedulerNumaInfo(AllocatedSets map[string]api.ResNumaSets) {
        ssn.cache.UpdateSchedulerNumaInfo(AllocatedSets)
}

// KubeClient returns the kubernetes client
func (ssn Session) KubeClient() kubernetes.Interface {
        return ssn.kubeClient
}

// InformerFactory returns the scheduler ShareInformerFactory
func (ssn Session) InformerFactory() informers.SharedInformerFactory {
        return ssn.informerFactory
}

//String return nodes and jobs information in the session
func (ssn Session) String() string {
        msg := fmt.Sprintf("Session %v: \n", ssn.UID)

        for _, job := range ssn.Jobs {
                msg = fmt.Sprintf("%s%v\n", msg, job)
        }

        for _, node := range ssn.Nodes {
                msg = fmt.Sprintf("%s%v\n", msg, node)
        }

        return msg
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        k8sframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/apis/pkg/apis/scheduling"
        "volcano.sh/volcano/pkg/scheduler/api"
)

// AddJobOrderFn add job order function
func (ssn *Session) AddJobOrderFn(name string, cf api.CompareFn) {
        ssn.jobOrderFns[name] = cf
}

// AddQueueOrderFn add queue order function
func (ssn *Session) AddQueueOrderFn(name string, qf api.CompareFn) {
        ssn.queueOrderFns[name] = qf
}

// AddClusterOrderFn add queue order function
func (ssn *Session) AddClusterOrderFn(name string, qf api.CompareFn) {
        ssn.clusterOrderFns[name] = qf
}

// AddTaskOrderFn add task order function
func (ssn *Session) AddTaskOrderFn(name string, cf api.CompareFn) {
        ssn.taskOrderFns[name] = cf
}

// AddNamespaceOrderFn add namespace order function
func (ssn *Session) AddNamespaceOrderFn(name string, cf api.CompareFn) {
        ssn.namespaceOrderFns[name] = cf
}

// AddPreemptableFn add preemptable function
func (ssn *Session) AddPreemptableFn(name string, cf api.EvictableFn) {
        ssn.preemptableFns[name] = cf
}

// AddReclaimableFn add Reclaimable function
func (ssn *Session) AddReclaimableFn(name string, rf api.EvictableFn) {
        ssn.reclaimableFns[name] = rf
}

// AddJobReadyFn add JobReady function
func (ssn *Session) AddJobReadyFn(name string, vf api.ValidateFn) {
        ssn.jobReadyFns[name] = vf
}

// AddJobPipelinedFn add pipelined function
func (ssn *Session) AddJobPipelinedFn(name string, vf api.VoteFn) {
        ssn.jobPipelinedFns[name] = vf
}

// AddPredicateFn add Predicate function
func (ssn *Session) AddPredicateFn(name string, pf api.PredicateFn) {
        ssn.predicateFns[name] = pf
}

// AddBestNodeFn add BestNode function
func (ssn *Session) AddBestNodeFn(name string, pf api.BestNodeFn) {
        ssn.bestNodeFns[name] = pf
}

// AddNodeOrderFn add Node order function
func (ssn *Session) AddNodeOrderFn(name string, pf api.NodeOrderFn) {
        ssn.nodeOrderFns[name] = pf
}

// AddBatchNodeOrderFn add Batch Node order function
func (ssn *Session) AddBatchNodeOrderFn(name string, pf api.BatchNodeOrderFn) {
        ssn.batchNodeOrderFns[name] = pf
}

// AddNodeMapFn add Node map function
func (ssn *Session) AddNodeMapFn(name string, pf api.NodeMapFn) {
        ssn.nodeMapFns[name] = pf
}

// AddNodeReduceFn add Node reduce function
func (ssn *Session) AddNodeReduceFn(name string, pf api.NodeReduceFn) {
        ssn.nodeReduceFns[name] = pf
}

// AddOverusedFn add overused function
func (ssn *Session) AddOverusedFn(name string, fn api.ValidateFn) {
        ssn.overusedFns[name] = fn
}

// AddUnderusedResourceFn add underused function
func (ssn *Session) AddUnderusedResourceFn(name string, fn api.UnderUsedResourceFn) {
        ssn.underUsedFns[name] = fn
}

// AddJobValidFn add jobvalid function
func (ssn *Session) AddJobValidFn(name string, fn api.ValidateExFn) {
        ssn.jobValidFns[name] = fn
}

// AddJobEnqueueableFn add jobenqueueable function
func (ssn *Session) AddJobEnqueueableFn(name string, fn api.VoteFn) {
        ssn.jobEnqueueableFns[name] = fn
}

// AddJobEnqueuedFn add jobEnqueued function
func (ssn *Session) AddJobEnqueuedFn(name string, fn api.JobEnqueuedFn) {
        ssn.jobEnqueuedFns[name] = fn
}

// AddTargetJobFn add targetjob function
func (ssn *Session) AddTargetJobFn(name string, fn api.TargetJobFn) {
        ssn.targetJobFns[name] = fn
}

// AddReservedNodesFn add reservedNodesFn function
func (ssn *Session) AddReservedNodesFn(name string, fn api.ReservedNodesFn) {
        ssn.reservedNodesFns[name] = fn
}

// AddVictimTasksFns add victimTasksFns function
func (ssn *Session) AddVictimTasksFns(name string, fn api.VictimTasksFn) {
        ssn.victimTasksFns[name] = fn
}

// AddJobStarvingFns add jobStarvingFns function
func (ssn *Session) AddJobStarvingFns(name string, fn api.ValidateFn) {
        ssn.jobStarvingFns[name] = fn
}

// Reclaimable invoke reclaimable function of the plugins
func (ssn *Session) Reclaimable(reclaimer *api.TaskInfo, reclaimees []*api.TaskInfo) []*api.TaskInfo {
        var victims []*api.TaskInfo
        var init bool

        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledReclaimable) {
                                continue
                        }
                        rf, found := ssn.reclaimableFns[plugin.Name]
                        if !found {
                                continue
                        }

                        candidates, abstain := rf(reclaimer, reclaimees)
                        if abstain == 0 {
                                continue
                        }
                        if len(candidates) == 0 {
                                victims = nil
                                break
                        }
                        if !init {
                                victims = candidates
                                init = true
                        } else {
                                var intersection []*api.TaskInfo
                                // Get intersection of victims and candidates.
                                for _, v := range victims {
                                        for _, c := range candidates {
                                                if v.UID == c.UID {
                                                        intersection = append(intersection, v)
                                                }
                                        }
                                }

                                // Update victims to intersection
                                victims = intersection
                        }
                }
                // Plugins in this tier made decision if victims is not nil
                if victims != nil {
                        return victims
                }
        }

        return victims
}

// Preemptable invoke preemptable function of the plugins
func (ssn *Session) Preemptable(preemptor *api.TaskInfo, preemptees []*api.TaskInfo) []*api.TaskInfo {
        var victims []*api.TaskInfo
        var init bool

        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledPreemptable) {
                                continue
                        }

                        pf, found := ssn.preemptableFns[plugin.Name]
                        if !found {
                                continue
                        }
                        candidates, abstain := pf(preemptor, preemptees)
                        if abstain == 0 {
                                continue
                        }
                        // intersection will be nil if length is 0, don't need to do any more check
                        if len(candidates) == 0 {
                                victims = nil
                                break
                        }

                        if !init {
                                victims = candidates
                                init = true
                        } else {
                                var intersection []*api.TaskInfo
                                // Get intersection of victims and candidates.
                                for _, v := range victims {
                                        for _, c := range candidates {
                                                if v.UID == c.UID {
                                                        intersection = append(intersection, v)
                                                }
                                        }
                                }

                                // Update victims to intersection
                                victims = intersection
                        }
                }
                // Plugins in this tier made decision if victims is not nil
                if victims != nil {
                        return victims
                }
        }

        return victims
}

// Overused invoke overused function of the plugins
func (ssn *Session) Overused(queue *api.QueueInfo) bool {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        of, found := ssn.overusedFns[plugin.Name]
                        if !found {
                                continue
                        }
                        if of(queue) {
                                return true
                        }
                }
        }

        return false
}

// UnderusedResources invoke underused function of the plugins
// Returns:
//  * nil if no `UnderUsedResourceFn` is registered
//  * [] if no under-used resources
func (ssn *Session) UnderusedResources(queue *api.QueueInfo) api.ResourceNameList {
        if len(ssn.underUsedFns) == 0 {
                return nil
        }
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        of, found := ssn.underUsedFns[plugin.Name]
                        if !found {
                                continue
                        }
                        underUsedResourceList := of(queue)
                        return underUsedResourceList
                }
        }

        return api.ResourceNameList{}
}

// JobReady invoke jobready function of the plugins
func (ssn *Session) JobReady(obj interface{}) bool {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledJobReady) {
                                continue
                        }
                        jrf, found := ssn.jobReadyFns[plugin.Name]
                        if !found {
                                continue
                        }

                        if !jrf(obj) {
                                return false
                        }
                }
        }

        return true
}

// JobPipelined invoke pipelined function of the plugins
// Check if job has get enough resource to run
func (ssn *Session) JobPipelined(obj interface{}) bool {
        var hasFound bool
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledJobPipelined) {
                                continue
                        }
                        jrf, found := ssn.jobPipelinedFns[plugin.Name]
                        if !found {
                                continue
                        }

                        res := jrf(obj)
                        if res < 0 {
                                return false
                        }
                        if res > 0 {
                                hasFound = true
                        }
                }
                // if plugin exists that votes permit, meanwhile other plugin votes abstention,
                // permit job to be pipelined, do not check next tier
                if hasFound {
                        return true
                }
        }

        return true
}

// JobStarving invoke jobStarving function of the plugins
// Check if job still need more resource
func (ssn *Session) JobStarving(obj interface{}) bool {
        var hasFound bool
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledJobStarving) {
                                continue
                        }
                        jrf, found := ssn.jobStarvingFns[plugin.Name]
                        if !found {
                                continue
                        }
                        hasFound = true

                        if !jrf(obj) {
                                return false
                        }
                }
                // this tier registered function
                if hasFound {
                        return true
                }
        }

        return false
}

// JobValid invoke jobvalid function of the plugins
func (ssn *Session) JobValid(obj interface{}) *api.ValidateResult {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        jrf, found := ssn.jobValidFns[plugin.Name]
                        if !found {
                                continue
                        }

                        if vr := jrf(obj); vr != nil && !vr.Pass {
                                return vr
                        }
                }
        }

        return nil
}

// JobEnqueueable invoke jobEnqueueableFns function of the plugins
func (ssn *Session) JobEnqueueable(obj interface{}) bool {
        var hasFound bool
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledJobEnqueued) {
                                continue
                        }
                        fn, found := ssn.jobEnqueueableFns[plugin.Name]
                        if !found {
                                continue
                        }

                        res := fn(obj)
                        if res < 0 {
                                return false
                        }
                        if res > 0 {
                                hasFound = true
                        }
                }
                // if plugin exists that votes permit, meanwhile other plugin votes abstention,
                // permit job to be enqueueable, do not check next tier
                if hasFound {
                        return true
                }
        }

        return true
}

// JobEnqueued invoke jobEnqueuedFns function of the plugins
func (ssn *Session) JobEnqueued(obj interface{}) {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledJobEnqueued) {
                                continue
                        }
                        fn, found := ssn.jobEnqueuedFns[plugin.Name]
                        if !found {
                                continue
                        }

                        fn(obj)
                }
        }
}

// TargetJob invoke targetJobFns function of the plugins
func (ssn *Session) TargetJob(jobs []*api.JobInfo) *api.JobInfo {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledTargetJob) {
                                continue
                        }
                        fn, found := ssn.targetJobFns[plugin.Name]
                        if !found {
                                continue
                        }
                        return fn(jobs)
                }
        }
        return nil
}

// VictimTasks invoke ReservedNodes function of the plugins
func (ssn *Session) VictimTasks() []*api.TaskInfo {
        var victims []*api.TaskInfo
        var init bool

        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledVictim) {
                                continue
                        }

                        pf, found := ssn.victimTasksFns[plugin.Name]
                        if !found {
                                continue
                        }
                        candidates := pf()
                        if !init {
                                victims = candidates
                                init = true
                        } else {
                                var intersection []*api.TaskInfo
                                // Get intersection of victims and candidates.
                                for _, v := range victims {
                                        for _, c := range candidates {
                                                if v.UID == c.UID {
                                                        intersection = append(intersection, v)
                                                }
                                        }
                                }

                                // Update victims to intersection
                                victims = intersection
                        }
                }
                // Plugins in this tier made decision if victims is not nil
                if victims != nil {
                        return victims
                }
        }

        return victims
}

// ReservedNodes invoke ReservedNodes function of the plugins
func (ssn *Session) ReservedNodes() {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledReservedNodes) {
                                continue
                        }
                        fn, found := ssn.reservedNodesFns[plugin.Name]
                        if !found {
                                continue
                        }
                        fn()
                }
        }
}

// JobOrderFn invoke joborder function of the plugins
func (ssn *Session) JobOrderFn(l, r interface{}) bool {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledJobOrder) {
                                continue
                        }
                        jof, found := ssn.jobOrderFns[plugin.Name]
                        if !found {
                                continue
                        }
                        if j := jof(l, r); j != 0 {
                                return j < 0
                        }
                }
        }

        // If no job order funcs, order job by CreationTimestamp first, then by UID.
        lv := l.(*api.JobInfo)
        rv := r.(*api.JobInfo)
        if lv.CreationTimestamp.Equal(&rv.CreationTimestamp) {
                return lv.UID < rv.UID
        }
        return lv.CreationTimestamp.Before(&rv.CreationTimestamp)
}

// NamespaceOrderFn invoke namespaceorder function of the plugins
func (ssn *Session) NamespaceOrderFn(l, r interface{}) bool {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledNamespaceOrder) {
                                continue
                        }
                        nof, found := ssn.namespaceOrderFns[plugin.Name]
                        if !found {
                                continue
                        }
                        if j := nof(l, r); j != 0 {
                                return j < 0
                        }
                }
        }

        // TODO(lminzhw): if all NamespaceOrderFn treat these two namespace as the same,
        // we should make the job order have its affect among namespaces.
        // or just schedule namespace one by one
        lv := l.(api.NamespaceName)
        rv := r.(api.NamespaceName)
        return lv < rv
}

// ClusterOrderFn invoke ClusterOrderFn function of the plugins
func (ssn *Session) ClusterOrderFn(l, r interface{}) bool {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledClusterOrder) {
                                continue
                        }
                        cof, found := ssn.clusterOrderFns[plugin.Name]
                        if !found {
                                continue
                        }
                        if j := cof(l, r); j != 0 {
                                return j < 0
                        }
                }
        }

        // If no cluster order funcs, order cluster by ClusterID
        lv := l.(*scheduling.Cluster)
        rv := r.(*scheduling.Cluster)
        return lv.Name < rv.Name
}

// QueueOrderFn invoke queueorder function of the plugins
func (ssn *Session) QueueOrderFn(l, r interface{}) bool {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledQueueOrder) {
                                continue
                        }
                        qof, found := ssn.queueOrderFns[plugin.Name]
                        if !found {
                                continue
                        }
                        if j := qof(l, r); j != 0 {
                                return j < 0
                        }
                }
        }

        // If no queue order funcs, order queue by CreationTimestamp first, then by UID.
        lv := l.(*api.QueueInfo)
        rv := r.(*api.QueueInfo)
        if lv.Queue.CreationTimestamp.Equal(&rv.Queue.CreationTimestamp) {
                return lv.UID < rv.UID
        }
        return lv.Queue.CreationTimestamp.Before(&rv.Queue.CreationTimestamp)
}

// TaskCompareFns invoke taskorder function of the plugins
func (ssn *Session) TaskCompareFns(l, r interface{}) int {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledTaskOrder) {
                                continue
                        }
                        tof, found := ssn.taskOrderFns[plugin.Name]
                        if !found {
                                continue
                        }
                        if j := tof(l, r); j != 0 {
                                return j
                        }
                }
        }

        return 0
}

// TaskOrderFn invoke taskorder function of the plugins
func (ssn *Session) TaskOrderFn(l, r interface{}) bool {
        if res := ssn.TaskCompareFns(l, r); res != 0 {
                return res < 0
        }

        // If no task order funcs, order task by CreationTimestamp first, then by UID.
        lv := l.(*api.TaskInfo)
        rv := r.(*api.TaskInfo)
        if lv.Pod.CreationTimestamp.Equal(&rv.Pod.CreationTimestamp) {
                return lv.UID < rv.UID
        }
        return lv.Pod.CreationTimestamp.Before(&rv.Pod.CreationTimestamp)
}

// PredicateFn invoke predicate function of the plugins
func (ssn *Session) PredicateFn(task *api.TaskInfo, node *api.NodeInfo) error {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledPredicate) {
                                continue
                        }
                        pfn, found := ssn.predicateFns[plugin.Name]
                        if !found {
                                continue
                        }
                        err := pfn(task, node)
                        if err != nil {
                                return err
                        }
                }
        }
        return nil
}

// BestNodeFn invoke bestNode function of the plugins
func (ssn *Session) BestNodeFn(task *api.TaskInfo, nodeScores map[float64][]*api.NodeInfo) *api.NodeInfo {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledBestNode) {
                                continue
                        }
                        pfn, found := ssn.bestNodeFns[plugin.Name]
                        if !found {
                                continue
                        }
                        // Only the first plugin that enables and realizes bestNodeFn is allowed to choose best node for task
                        if bestNode := pfn(task, nodeScores); bestNode != nil {
                                return bestNode
                        }
                }
        }
        return nil
}

// NodeOrderFn invoke node order function of the plugins
func (ssn *Session) NodeOrderFn(task *api.TaskInfo, node *api.NodeInfo) (float64, error) {
        priorityScore := 0.0
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledNodeOrder) {
                                continue
                        }
                        pfn, found := ssn.nodeOrderFns[plugin.Name]
                        if !found {
                                continue
                        }
                        score, err := pfn(task, node)
                        if err != nil {
                                return 0, err
                        }
                        priorityScore += score
                }
        }
        return priorityScore, nil
}

// BatchNodeOrderFn invoke node order function of the plugins
func (ssn *Session) BatchNodeOrderFn(task *api.TaskInfo, nodes []*api.NodeInfo) (map[string]float64, error) {
        priorityScore := make(map[string]float64, len(nodes))
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledNodeOrder) {
                                continue
                        }
                        pfn, found := ssn.batchNodeOrderFns[plugin.Name]
                        if !found {
                                continue
                        }
                        score, err := pfn(task, nodes)
                        if err != nil {
                                return nil, err
                        }
                        for nodeName, score := range score {
                                priorityScore[nodeName] += score
                        }
                }
        }
        return priorityScore, nil
}

func isEnabled(enabled *bool) bool {
        return enabled != nil && *enabled
}

// NodeOrderMapFn invoke node order function of the plugins
func (ssn *Session) NodeOrderMapFn(task *api.TaskInfo, node *api.NodeInfo) (map[string]float64, float64, error) {
        nodeScoreMap := map[string]float64{}
        var priorityScore float64
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledNodeOrder) {
                                continue
                        }
                        if pfn, found := ssn.nodeOrderFns[plugin.Name]; found {
                                score, err := pfn(task, node)
                                if err != nil {
                                        return nodeScoreMap, priorityScore, err
                                }
                                priorityScore += score
                        }
                        if pfn, found := ssn.nodeMapFns[plugin.Name]; found {
                                score, err := pfn(task, node)
                                if err != nil {
                                        return nodeScoreMap, priorityScore, err
                                }
                                nodeScoreMap[plugin.Name] = score
                        }
                }
        }
        return nodeScoreMap, priorityScore, nil
}

// NodeOrderReduceFn invoke node order function of the plugins
func (ssn *Session) NodeOrderReduceFn(task *api.TaskInfo, pluginNodeScoreMap map[string]k8sframework.NodeScoreList) (map[string]float64, error) {
        nodeScoreMap := map[string]float64{}
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if !isEnabled(plugin.EnabledNodeOrder) {
                                continue
                        }
                        pfn, found := ssn.nodeReduceFns[plugin.Name]
                        if !found {
                                continue
                        }
                        if err := pfn(task, pluginNodeScoreMap[plugin.Name]); err != nil {
                                return nodeScoreMap, err
                        }
                        for _, hp := range pluginNodeScoreMap[plugin.Name] {
                                nodeScoreMap[hp.Name] += float64(hp.Score)
                        }
                }
        }
        return nodeScoreMap, nil
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
        "fmt"

        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/metrics"
)

// Operation type
type Operation int8

const (
        // Evict op
        Evict = iota
        // Pipeline op
        Pipeline
        // Allocate op
        Allocate
)

type operation struct {
        name   Operation
        task   *api.TaskInfo
        reason string
}

// Statement structure
type Statement struct {
        operations []operation
        ssn        *Session
}

// NewStatement returns new statement object
func NewStatement(ssn *Session) *Statement {
        return &Statement{
                ssn: ssn,
        }
}

// Evict the pod
func (s *Statement) Evict(reclaimee *api.TaskInfo, reason string) error {
        // Update status in session
        if job, found := s.ssn.Jobs[reclaimee.Job]; found {
                if err := job.UpdateTaskStatus(reclaimee, api.Releasing); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                reclaimee.Namespace, reclaimee.Name, api.Releasing, s.ssn.UID, err)
                }
        } else {
                klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.",
                        reclaimee.Job, s.ssn.UID)
        }

        // Update task in node.
        if node, found := s.ssn.Nodes[reclaimee.NodeName]; found {
                err := node.UpdateTask(reclaimee)
                if err != nil {
                        klog.Errorf("Failed to update task <%v/%v> in node %v for: %s",
                                reclaimee.Namespace, reclaimee.Name, reclaimee.NodeName, err.Error())
                        return err
                }
        }

        for _, eh := range s.ssn.eventHandlers {
                if eh.DeallocateFunc != nil {
                        eh.DeallocateFunc(&Event{
                                Task: reclaimee,
                        })
                }
        }

        s.operations = append(s.operations, operation{
                name:   Evict,
                task:   reclaimee,
                reason: reason,
        })

        return nil
}

func (s *Statement) evict(reclaimee *api.TaskInfo, reason string) error {
        if err := s.ssn.cache.Evict(reclaimee, reason); err != nil {
                if e := s.unevict(reclaimee); e != nil {
                        klog.Errorf("Faled to unevict task <%v/%v>: %v.",
                                reclaimee.Namespace, reclaimee.Name, e)
                }
                return err
        }

        return nil
}

func (s *Statement) unevict(reclaimee *api.TaskInfo) error {
        // Update status in session
        job, found := s.ssn.Jobs[reclaimee.Job]
        if found {
                if err := job.UpdateTaskStatus(reclaimee, api.Running); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                reclaimee.Namespace, reclaimee.Name, api.Releasing, s.ssn.UID, err)
                }
        } else {
                klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.",
                        reclaimee.Job, s.ssn.UID)
        }

        // Update task in node.
        if node, found := s.ssn.Nodes[reclaimee.NodeName]; found {
                err := node.UpdateTask(reclaimee)
                if err != nil {
                        klog.Errorf("Failed to update task <%v/%v> in node %v for: %s",
                                reclaimee.Namespace, reclaimee.Name, reclaimee.NodeName, err.Error())
                        return err
                }
        }

        for _, eh := range s.ssn.eventHandlers {
                if eh.AllocateFunc != nil {
                        eh.AllocateFunc(&Event{
                                Task: reclaimee,
                        })
                }
        }

        return nil
}

// Pipeline the task for the node
func (s *Statement) Pipeline(task *api.TaskInfo, hostname string) error {
        job, found := s.ssn.Jobs[task.Job]
        if found {
                if err := job.UpdateTaskStatus(task, api.Pipelined); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                task.Namespace, task.Name, api.Pipelined, s.ssn.UID, err)
                }
        } else {
                klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.",
                        task.Job, s.ssn.UID)
        }

        task.NodeName = hostname

        if node, found := s.ssn.Nodes[hostname]; found {
                if err := node.AddTask(task); err != nil {
                        klog.Errorf("Failed to pipeline task <%v/%v> to node <%v> in Session <%v>: %v",
                                task.Namespace, task.Name, hostname, s.ssn.UID, err)
                }
                klog.V(3).Infof("After pipelined Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>",
                        task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing)
        } else {
                klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.",
                        hostname, s.ssn.UID)
        }

        for _, eh := range s.ssn.eventHandlers {
                if eh.AllocateFunc != nil {
                        eh.AllocateFunc(&Event{
                                Task: task,
                        })
                }
        }

        s.operations = append(s.operations, operation{
                name: Pipeline,
                task: task,
        })

        return nil
}

func (s *Statement) pipeline(task *api.TaskInfo) {
}

func (s *Statement) unpipeline(task *api.TaskInfo) error {
        job, found := s.ssn.Jobs[task.Job]
        if found {
                if err := job.UpdateTaskStatus(task, api.Pending); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                task.Namespace, task.Name, api.Pipelined, s.ssn.UID, err)
                }
        } else {
                klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.",
                        task.Job, s.ssn.UID)
        }

        if node, found := s.ssn.Nodes[task.NodeName]; found {
                if err := node.RemoveTask(task); err != nil {
                        klog.Errorf("Failed to pipeline task <%v/%v> to node <%v> in Session <%v>: %v",
                                task.Namespace, task.Name, task.NodeName, s.ssn.UID, err)
                }
                klog.V(3).Infof("After pipelined Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>",
                        task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing)
        } else {
                klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.",
                        task.NodeName, s.ssn.UID)
        }

        for _, eh := range s.ssn.eventHandlers {
                if eh.DeallocateFunc != nil {
                        eh.DeallocateFunc(&Event{
                                Task: task,
                        })
                }
        }
        task.NodeName = ""

        return nil
}

// Allocate the task to node
func (s *Statement) Allocate(task *api.TaskInfo, nodeInfo *api.NodeInfo) error {
        podVolumes, err := s.ssn.cache.GetPodVolumes(task, nodeInfo.Node)
        if err != nil {
                return err
        }

        hostname := nodeInfo.Name
        if err := s.ssn.cache.AllocateVolumes(task, hostname, podVolumes); err != nil {
                return err
        }

        task.Pod.Spec.NodeName = hostname
        task.PodVolumes = podVolumes

        // Only update status in session
        job, found := s.ssn.Jobs[task.Job]
        if found {
                if err := job.UpdateTaskStatus(task, api.Allocated); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                task.Namespace, task.Name, api.Allocated, s.ssn.UID, err)
                        return err
                }
        } else {
                klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.",
                        task.Job, s.ssn.UID)
                return fmt.Errorf("failed to find job %s", task.Job)
        }

        task.NodeName = hostname
        if node, found := s.ssn.Nodes[hostname]; found {
                if err := node.AddTask(task); err != nil {
                        klog.Errorf("Failed to add task <%v/%v> to node <%v> in Session <%v>: %v",
                                task.Namespace, task.Name, hostname, s.ssn.UID, err)
                        return err
                }
                klog.V(3).Infof("After allocated Task <%v/%v> to Node <%v>: idle <%v>, used <%v>, releasing <%v>",
                        task.Namespace, task.Name, node.Name, node.Idle, node.Used, node.Releasing)
        } else {
                klog.Errorf("Failed to found Node <%s> in Session <%s> index when binding.",
                        hostname, s.ssn.UID)
                return fmt.Errorf("failed to find node %s", hostname)
        }

        // Callbacks
        for _, eh := range s.ssn.eventHandlers {
                if eh.AllocateFunc != nil {
                        eh.AllocateFunc(&Event{
                                Task: task,
                        })
                }
        }

        // Update status in session
        klog.V(3).Info("Allocating operations ...")
        s.operations = append(s.operations, operation{
                name: Allocate,
                task: task,
        })

        return nil
}

func (s *Statement) allocate(task *api.TaskInfo) error {
        if err := s.ssn.cache.BindVolumes(task, task.PodVolumes); err != nil {
                return err
        }

        if err := s.ssn.cache.Bind(task, task.NodeName); err != nil {
                return err
        }

        // Update status in session
        if job, found := s.ssn.Jobs[task.Job]; found {
                if err := job.UpdateTaskStatus(task, api.Binding); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                task.Namespace, task.Name, api.Binding, s.ssn.UID, err)
                        return err
                }
        } else {
                klog.Errorf("Failed to found Job <%s> in Session <%s> index when binding.",
                        task.Job, s.ssn.UID)
                return fmt.Errorf("failed to find job %s", task.Job)
        }

        metrics.UpdateTaskScheduleDuration(metrics.Duration(task.Pod.CreationTimestamp.Time))
        return nil
}

// unallocate the pod for task
func (s *Statement) unallocate(task *api.TaskInfo) error {
        // Update status in session
        job, found := s.ssn.Jobs[task.Job]
        if found {
                if err := job.UpdateTaskStatus(task, api.Pending); err != nil {
                        klog.Errorf("Failed to update task <%v/%v> status to %v in Session <%v>: %v",
                                task.Namespace, task.Name, api.Pending, s.ssn.UID, err)
                }
        } else {
                klog.Errorf("Failed to find Job <%s> in Session <%s> index when unallocating.",
                        task.Job, s.ssn.UID)
        }

        if node, found := s.ssn.Nodes[task.NodeName]; found {
                klog.V(3).Infof("Remove Task <%v> on node <%v>", task.Name, task.NodeName)
                err := node.RemoveTask(task)
                if err != nil {
                        klog.Errorf("Failed to remove Task <%v> on node <%v>: %s", task.Name, task.NodeName, err.Error())
                }
        }

        for _, eh := range s.ssn.eventHandlers {
                if eh.DeallocateFunc != nil {
                        eh.DeallocateFunc(&Event{
                                Task: task,
                        })
                }
        }
        task.NodeName = ""

        return nil
}

// Discard operation for evict, pipeline and allocate
func (s *Statement) Discard() {
        klog.V(3).Info("Discarding operations ...")
        for i := len(s.operations) - 1; i >= 0; i-- {
                op := s.operations[i]
                op.task.GenerateLastTxContext()
                switch op.name {
                case Evict:
                        err := s.unevict(op.task)
                        if err != nil {
                                klog.Errorf("Failed to unevict task: %s", err.Error())
                        }
                case Pipeline:
                        err := s.unpipeline(op.task)
                        if err != nil {
                                klog.Errorf("Failed to unpipeline task: %s", err.Error())
                        }
                case Allocate:
                        err := s.unallocate(op.task)
                        if err != nil {
                                klog.Errorf("Failed to unallocate task: %s", err.Error())
                        }
                }
        }
}

// Commit operation for evict and pipeline
func (s *Statement) Commit() {
        klog.V(3).Info("Committing operations ...")
        for _, op := range s.operations {
                op.task.ClearLastTxContext()
                switch op.name {
                case Evict:
                        err := s.evict(op.task, op.reason)
                        if err != nil {
                                klog.Errorf("Failed to evict task: %s", err.Error())
                        }
                case Pipeline:
                        s.pipeline(op.task)
                case Allocate:
                        err := s.allocate(op.task)
                        if err != nil {
                                klog.Errorf("Failed to allocate task: for %s", err.Error())
                        }
                }
        }
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package binpack

import (
        "fmt"
        "strings"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
)

const (
        // PluginName indicates name of volcano scheduler plugin.
        PluginName = "binpack"
)

const (
        // BinpackWeight is the key for providing Binpack Priority Weight in YAML
        BinpackWeight = "binpack.weight"
        // BinpackCPU is the key for weight of cpu
        BinpackCPU = "binpack.cpu"
        // BinpackMemory is the key for weight of memory
        BinpackMemory = "binpack.memory"

        // BinpackResources is the key for additional resource key name
        BinpackResources = "binpack.resources"
        // BinpackResourcesPrefix is the key prefix for additional resource key name
        BinpackResourcesPrefix = BinpackResources + "."

        resourceFmt = "%s[%d]"
)

type priorityWeight struct {
        BinPackingWeight    int
        BinPackingCPU       int
        BinPackingMemory    int
        BinPackingResources map[v1.ResourceName]int
}

func (w *priorityWeight) String() string {
        length := 3
        if extendLength := len(w.BinPackingResources); extendLength == 0 {
                length++
        } else {
                length += extendLength
        }
        msg := make([]string, 0, length)
        msg = append(msg,
                fmt.Sprintf(resourceFmt, BinpackWeight, w.BinPackingWeight),
                fmt.Sprintf(resourceFmt, BinpackCPU, w.BinPackingCPU),
                fmt.Sprintf(resourceFmt, BinpackMemory, w.BinPackingMemory),
        )

        if len(w.BinPackingResources) == 0 {
                msg = append(msg, "no extend resources.")
        } else {
                for name, weight := range w.BinPackingResources {
                        msg = append(msg, fmt.Sprintf(resourceFmt, name, weight))
                }
        }
        return strings.Join(msg, ", ")
}

type binpackPlugin struct {
        // Arguments given for the plugin
        weight priorityWeight
}

//New function returns prioritizePlugin object
func New(aruguments framework.Arguments) framework.Plugin {
        weight := calculateWeight(aruguments)
        return &binpackPlugin{weight: weight}
}

func calculateWeight(args framework.Arguments) priorityWeight {
        /*
           User Should give priorityWeight in this format(binpack.weight, binpack.cpu, binpack.memory).
           Support change the weight about cpu, memory and additional resource by arguments.

           actions: "enqueue, reclaim, allocate, backfill, preempt"
           tiers:
           - plugins:
             - name: binpack
               arguments:
                 binpack.weight: 10
                 binpack.cpu: 5
                 binpack.memory: 1
                 binpack.resources: nvidia.com/gpu, example.com/foo
                 binpack.resources.nvidia.com/gpu: 2
                 binpack.resources.example.com/foo: 3
        */
        // Values are initialized to 1.
        weight := priorityWeight{
                BinPackingWeight:    1,
                BinPackingCPU:       1,
                BinPackingMemory:    1,
                BinPackingResources: make(map[v1.ResourceName]int),
        }

        // Checks whether binpack.weight is provided or not, if given, modifies the value in weight struct.
        args.GetInt(&weight.BinPackingWeight, BinpackWeight)
        // Checks whether binpack.cpu is provided or not, if given, modifies the value in weight struct.
        args.GetInt(&weight.BinPackingCPU, BinpackCPU)
        if weight.BinPackingCPU < 0 {
                weight.BinPackingCPU = 1
        }
        // Checks whether binpack.memory is provided or not, if given, modifies the value in weight struct.
        args.GetInt(&weight.BinPackingMemory, BinpackMemory)
        if weight.BinPackingMemory < 0 {
                weight.BinPackingMemory = 1
        }

        resourcesStr := args[BinpackResources]
        resources := strings.Split(resourcesStr, ",")
        for _, resource := range resources {
                resource = strings.TrimSpace(resource)
                if resource == "" {
                        continue
                }

                // binpack.resources.[ResourceName]
                resourceKey := BinpackResourcesPrefix + resource
                resourceWeight := 1
                args.GetInt(&resourceWeight, resourceKey)
                if resourceWeight < 0 {
                        resourceWeight = 1
                }
                weight.BinPackingResources[v1.ResourceName(resource)] = resourceWeight
        }

        return weight
}

func (bp *binpackPlugin) Name() string {
        return PluginName
}

func (bp *binpackPlugin) OnSessionOpen(ssn *framework.Session) {
        klog.V(4).Infof("Enter binpack plugin ...")
        if klog.V(4) {
                defer func() {
                        klog.V(4).Infof("Leaving binpack plugin. %s ...", bp.weight.String())
                }()

                notFoundResource := []string{}
                for resource := range bp.weight.BinPackingResources {
                        found := false
                        for _, nodeInfo := range ssn.Nodes {
                                if nodeInfo.Allocatable.Get(resource) > 0 {
                                        found = true
                                        break
                                }
                        }
                        if !found {
                                notFoundResource = append(notFoundResource, string(resource))
                        }
                }
                klog.V(4).Infof("resources [%s] record in weight but not found on any node", strings.Join(notFoundResource, ", "))
        }

        nodeOrderFn := func(task *api.TaskInfo, node *api.NodeInfo) (float64, error) {
                binPackingScore := BinPackingScore(task, node, bp.weight)

                klog.V(4).Infof("Binpack score for Task %s/%s on node %s is: %v", task.Namespace, task.Name, node.Name, binPackingScore)
                return binPackingScore, nil
        }
        if bp.weight.BinPackingWeight != 0 {
                ssn.AddNodeOrderFn(bp.Name(), nodeOrderFn)
        } else {
                klog.Infof("binpack weight is zero, skip node order function")
        }
}

func (bp *binpackPlugin) OnSessionClose(ssn *framework.Session) {
}

// BinPackingScore use the best fit polices during scheduling.
// Goals:
// - Schedule Jobs using BestFit Policy using Resource Bin Packing Priority Function
// - Reduce Fragmentation of scarce resources on the Cluster
func BinPackingScore(task *api.TaskInfo, node *api.NodeInfo, weight priorityWeight) float64 {
        score := 0.0
        weightSum := 0
        requested := task.Resreq
        allocatable := node.Allocatable
        used := node.Used

        for _, resource := range requested.ResourceNames() {
                request := requested.Get(resource)
                if request == 0 {
                        continue
                }
                allocate := allocatable.Get(resource)
                nodeUsed := used.Get(resource)

                resourceWeight := 0
                found := false
                switch resource {
                case v1.ResourceCPU:
                        resourceWeight = weight.BinPackingCPU
                        found = true
                case v1.ResourceMemory:
                        resourceWeight = weight.BinPackingMemory
                        found = true
                default:
                        resourceWeight, found = weight.BinPackingResources[resource]
                }
                if !found {
                        continue
                }

                resourceScore := ResourceBinPackingScore(request, allocate, nodeUsed, resourceWeight)
                klog.V(5).Infof("task %s/%s on node %s resource %s, need %f, used %f, allocatable %f, weight %d, score %f", task.Namespace, task.Name, node.Name, resource, request, nodeUsed, allocate, resourceWeight, resourceScore)

                score += resourceScore
                weightSum += resourceWeight
        }

        // mapping the result from [0, weightSum] to [0, 10(MaxPriority)]
        if weightSum > 0 {
                score /= float64(weightSum)
        }
        score *= float64(v1alpha1.MaxNodeScore * int64(weight.BinPackingWeight))

        return score
}

// ResourceBinPackingScore calculate the binpack score for resource with provided info
func ResourceBinPackingScore(requested, capacity, used float64, weight int) float64 {
        if capacity == 0 || weight == 0 {
                return 0
        }

        usedFinally := requested + used
        if usedFinally > capacity {
                return 0
        }

        score := usedFinally * float64(weight) / capacity
        return score
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package drf

import (
        "fmt"
        "math"
        "strconv"
        "strings"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/api/helpers"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/metrics"
        "volcano.sh/volcano/pkg/scheduler/plugins/util"
)

// PluginName indicates name of volcano scheduler plugin.
const PluginName = "drf"

var shareDelta = 0.000001

// hierarchicalNode represents the node hierarchy
// and the corresponding weight and drf attribute
type hierarchicalNode struct {
        parent *hierarchicalNode
        attr   *drfAttr
        // If the node is a leaf node,
        // request represents the request of the job.
        request   *api.Resource
        weight    float64
        saturated bool
        hierarchy string
        children  map[string]*hierarchicalNode
}

func (node *hierarchicalNode) Clone(parent *hierarchicalNode) *hierarchicalNode {
        newNode := &hierarchicalNode{
                parent: parent,
                attr: &drfAttr{
                        share:            node.attr.share,
                        dominantResource: node.attr.dominantResource,
                        allocated:        node.attr.allocated.Clone(),
                },
                request:   node.request.Clone(),
                weight:    node.weight,
                saturated: node.saturated,
                hierarchy: node.hierarchy,
                children:  nil,
        }
        if node.children != nil {
                newNode.children = map[string]*hierarchicalNode{}
                for _, child := range node.children {
                        newNode.children[child.hierarchy] = child.Clone(newNode)
                }
        }
        return newNode
}

// resourceSaturated returns true if any resource of the job is saturated or the job demands fully allocated resource
func resourceSaturated(allocated *api.Resource,
        jobRequest *api.Resource, demandingResources map[v1.ResourceName]bool) bool {
        for _, rn := range allocated.ResourceNames() {
                if allocated.Get(rn) != 0 && jobRequest.Get(rn) != 0 &&
                        allocated.Get(rn) >= jobRequest.Get(rn) {
                        return true
                }
                if !demandingResources[rn] && jobRequest.Get(rn) != 0 {
                        return true
                }
        }
        return false
}

type drfAttr struct {
        share            float64
        dominantResource string
        allocated        *api.Resource
}

func (attr *drfAttr) String() string {
        return fmt.Sprintf("dominant resource <%s>, dominant share %f, allocated %s",
                attr.dominantResource, attr.share, attr.allocated)
}

type drfPlugin struct {
        totalResource  *api.Resource
        totalAllocated *api.Resource

        // Key is Job ID
        jobAttrs map[api.JobID]*drfAttr

        // map[namespaceName]->attr
        namespaceOpts map[string]*drfAttr

        // hierarchical tree root
        hierarchicalRoot *hierarchicalNode

        // Arguments given for the plugin
        pluginArguments framework.Arguments
}

// New return drf plugin
func New(arguments framework.Arguments) framework.Plugin {
        return &drfPlugin{
                totalResource:  api.EmptyResource(),
                totalAllocated: api.EmptyResource(),
                jobAttrs:       map[api.JobID]*drfAttr{},
                namespaceOpts:  map[string]*drfAttr{},
                hierarchicalRoot: &hierarchicalNode{
                        attr:      &drfAttr{allocated: api.EmptyResource()},
                        request:   api.EmptyResource(),
                        hierarchy: "root",
                        weight:    1,
                        children:  map[string]*hierarchicalNode{},
                },
                pluginArguments: arguments,
        }
}

func (drf *drfPlugin) Name() string {
        return PluginName
}

// HierarchyEnabled returns if hierarchy is enabled
func (drf *drfPlugin) HierarchyEnabled(ssn *framework.Session) bool {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if plugin.Name != PluginName {
                                continue
                        }
                        return plugin.EnabledHierarchy != nil && *plugin.EnabledHierarchy
                }
        }
        return false
}

// NamespaceOrderEnabled returns the NamespaceOrder for this plugin is enabled in this session or not
func (drf *drfPlugin) NamespaceOrderEnabled(ssn *framework.Session) bool {
        for _, tier := range ssn.Tiers {
                for _, plugin := range tier.Plugins {
                        if plugin.Name != PluginName {
                                continue
                        }
                        return plugin.EnabledNamespaceOrder != nil && *plugin.EnabledNamespaceOrder
                }
        }
        return false
}

func (drf *drfPlugin) compareQueues(root *hierarchicalNode, lqueue *api.QueueInfo, rqueue *api.QueueInfo) float64 {
        lnode := root
        lpaths := strings.Split(lqueue.Hierarchy, "/")
        rnode := root
        rpaths := strings.Split(rqueue.Hierarchy, "/")
        depth := 0
        if len(lpaths) < len(rpaths) {
                depth = len(lpaths)
        } else {
                depth = len(rpaths)
        }
        for i := 0; i < depth; i++ {
                // Saturated nodes have minumun prioirty,
                // so that demanding nodes will be poped first.
                if !lnode.saturated && rnode.saturated {
                        return -1
                }
                if lnode.saturated && !rnode.saturated {
                        return 1
                }
                if lnode.attr.share/lnode.weight == rnode.attr.share/rnode.weight {
                        if i < depth-1 {
                                lnode = lnode.children[lpaths[i+1]]
                                rnode = rnode.children[rpaths[i+1]]
                        }
                } else {
                        return lnode.attr.share/lnode.weight - rnode.attr.share/rnode.weight
                }
        }
        return 0
}

func (drf *drfPlugin) OnSessionOpen(ssn *framework.Session) {
        // Prepare scheduling data for this session.
        drf.totalResource.Add(ssn.TotalResource)

        klog.V(4).Infof("Total Allocatable %s", drf.totalResource)

        namespaceOrderEnabled := drf.NamespaceOrderEnabled(ssn)
        hierarchyEnabled := drf.HierarchyEnabled(ssn)

        for _, job := range ssn.Jobs {
                attr := &drfAttr{
                        allocated: api.EmptyResource(),
                }

                for status, tasks := range job.TaskStatusIndex {
                        if api.AllocatedStatus(status) {
                                for _, t := range tasks {
                                        attr.allocated.Add(t.Resreq)
                                }
                        }
                }

                // Calculate the init share of Job
                drf.updateJobShare(job.Namespace, job.Name, attr)

                drf.jobAttrs[job.UID] = attr

                if namespaceOrderEnabled {
                        nsOpts, found := drf.namespaceOpts[job.Namespace]
                        if !found {
                                nsOpts = &drfAttr{
                                        allocated: api.EmptyResource(),
                                }
                                drf.namespaceOpts[job.Namespace] = nsOpts
                        }
                        // all task in job should have the same namespace with job
                        nsOpts.allocated.Add(attr.allocated)
                        drf.updateNamespaceShare(job.Namespace, nsOpts)
                }
                if hierarchyEnabled {
                        queue := ssn.Queues[job.Queue]
                        drf.totalAllocated.Add(attr.allocated)
                        drf.UpdateHierarchicalShare(drf.hierarchicalRoot, drf.totalAllocated, job, attr, queue.Hierarchy, queue.Weights)
                }
        }

        preemptableFn := func(preemptor *api.TaskInfo, preemptees []*api.TaskInfo) ([]*api.TaskInfo, int) {
                var victims []*api.TaskInfo

                addVictim := func(candidate *api.TaskInfo) {
                        victims = append(victims, candidate)
                }

                if namespaceOrderEnabled {
                        // apply the namespace share policy on preemptee firstly

                        lWeight := ssn.NamespaceInfo[api.NamespaceName(preemptor.Namespace)].GetWeight()
                        lNsAtt := drf.namespaceOpts[preemptor.Namespace]
                        lNsAlloc := lNsAtt.allocated.Clone().Add(preemptor.Resreq)
                        _, lNsShare := drf.calculateShare(lNsAlloc, drf.totalResource)
                        lNsShareWeighted := lNsShare / float64(lWeight)

                        namespaceAllocation := map[string]*api.Resource{}

                        // undecidedPreemptees means this policy could not judge preemptee is preemptable or not
                        // and left it to next policy
                        undecidedPreemptees := []*api.TaskInfo{}

                        for _, preemptee := range preemptees {
                                if preemptor.Namespace == preemptee.Namespace {
                                        // policy is disabled when they are in the same namespace
                                        undecidedPreemptees = append(undecidedPreemptees, preemptee)
                                        continue
                                }

                                // compute the preemptee namespace weighted share after preemption
                                nsAllocation, found := namespaceAllocation[preemptee.Namespace]
                                if !found {
                                        rNsAtt := drf.namespaceOpts[preemptee.Namespace]
                                        nsAllocation = rNsAtt.allocated.Clone()
                                        namespaceAllocation[preemptee.Namespace] = nsAllocation
                                }
                                rWeight := ssn.NamespaceInfo[api.NamespaceName(preemptee.Namespace)].GetWeight()
                                rNsAlloc := nsAllocation.Sub(preemptee.Resreq)
                                _, rNsShare := drf.calculateShare(rNsAlloc, drf.totalResource)
                                rNsShareWeighted := rNsShare / float64(rWeight)

                                // to avoid ping pong actions, the preemptee namespace should
                                // have the higher weighted share after preemption.
                                if lNsShareWeighted < rNsShareWeighted {
                                        addVictim(preemptee)
                                        continue
                                }
                                if lNsShareWeighted-rNsShareWeighted > shareDelta {
                                        continue
                                }

                                // equal namespace order leads to judgement of jobOrder
                                undecidedPreemptees = append(undecidedPreemptees, preemptee)
                        }

                        preemptees = undecidedPreemptees
                }

                latt := drf.jobAttrs[preemptor.Job]
                lalloc := latt.allocated.Clone().Add(preemptor.Resreq)
                _, ls := drf.calculateShare(lalloc, drf.totalResource)

                allocations := map[api.JobID]*api.Resource{}

                for _, preemptee := range preemptees {
                        if _, found := allocations[preemptee.Job]; !found {
                                ratt := drf.jobAttrs[preemptee.Job]
                                allocations[preemptee.Job] = ratt.allocated.Clone()
                        }
                        ralloc := allocations[preemptee.Job].Sub(preemptee.Resreq)
                        _, rs := drf.calculateShare(ralloc, drf.totalResource)

                        if ls < rs || math.Abs(ls-rs) <= shareDelta {
                                addVictim(preemptee)
                        }
                }

                klog.V(4).Infof("Victims from DRF plugins are %+v", victims)

                return victims, util.Permit
        }

        ssn.AddPreemptableFn(drf.Name(), preemptableFn)

        if hierarchyEnabled {
                queueOrderFn := func(l interface{}, r interface{}) int {
                        lv := l.(*api.QueueInfo)
                        rv := r.(*api.QueueInfo)
                        ret := drf.compareQueues(drf.hierarchicalRoot, lv, rv)
                        if ret < 0 {
                                return -1
                        }
                        if ret > 0 {
                                return 1
                        }
                        return 0
                }
                ssn.AddQueueOrderFn(drf.Name(), queueOrderFn)

                reclaimFn := func(reclaimer *api.TaskInfo, reclaimees []*api.TaskInfo) ([]*api.TaskInfo, int) {
                        var victims []*api.TaskInfo
                        // clone hdrf tree
                        totalAllocated := drf.totalAllocated.Clone()
                        root := drf.hierarchicalRoot.Clone(nil)

                        //  update reclaimer hdrf
                        ljob := ssn.Jobs[reclaimer.Job]
                        lqueue := ssn.Queues[ljob.Queue]
                        ljob = ljob.Clone()
                        attr := drf.jobAttrs[ljob.UID]
                        lattr := &drfAttr{
                                allocated: attr.allocated.Clone(),
                        }
                        lattr.allocated.Add(reclaimer.Resreq)
                        totalAllocated.Add(reclaimer.Resreq)
                        drf.updateShare(lattr)
                        drf.UpdateHierarchicalShare(root, totalAllocated, ljob, lattr, lqueue.Hierarchy, lqueue.Weights)

                        for _, preemptee := range reclaimees {
                                rjob := ssn.Jobs[preemptee.Job]
                                rqueue := ssn.Queues[rjob.Queue]

                                // update hdrf of reclaimee job
                                totalAllocated.Sub(preemptee.Resreq)
                                rjob = rjob.Clone()
                                attr := drf.jobAttrs[rjob.UID]
                                rattr := &drfAttr{
                                        allocated: attr.allocated.Clone(),
                                }
                                rattr.allocated.Sub(preemptee.Resreq)
                                drf.updateShare(rattr)
                                drf.UpdateHierarchicalShare(root, totalAllocated, rjob, rattr, rqueue.Hierarchy, rqueue.Weights)

                                // compare hdrf of queues
                                ret := drf.compareQueues(root, lqueue, rqueue)

                                // resume hdrf of reclaimee job
                                totalAllocated.Add(preemptee.Resreq)
                                rattr.allocated.Add(preemptee.Resreq)
                                drf.updateShare(rattr)
                                drf.UpdateHierarchicalShare(root, totalAllocated, rjob, rattr, rqueue.Hierarchy, rqueue.Weights)

                                if ret < 0 {
                                        victims = append(victims, preemptee)
                                }

                                if ret > shareDelta {
                                        continue
                                }
                        }

                        klog.V(4).Infof("Victims from HDRF plugins are %+v", victims)

                        return victims, util.Permit
                }
                ssn.AddReclaimableFn(drf.Name(), reclaimFn)
        }

        jobOrderFn := func(l interface{}, r interface{}) int {
                lv := l.(*api.JobInfo)
                rv := r.(*api.JobInfo)

                klog.V(4).Infof("DRF JobOrderFn: <%v/%v> share state: %v, <%v/%v> share state: %v",
                        lv.Namespace, lv.Name, drf.jobAttrs[lv.UID].share, rv.Namespace, rv.Name, drf.jobAttrs[rv.UID].share)

                if drf.jobAttrs[lv.UID].share == drf.jobAttrs[rv.UID].share {
                        return 0
                }

                if drf.jobAttrs[lv.UID].share < drf.jobAttrs[rv.UID].share {
                        return -1
                }

                return 1
        }

        ssn.AddJobOrderFn(drf.Name(), jobOrderFn)

        namespaceOrderFn := func(l interface{}, r interface{}) int {
                lv := l.(api.NamespaceName)
                rv := r.(api.NamespaceName)

                lOpt := drf.namespaceOpts[string(lv)]
                rOpt := drf.namespaceOpts[string(rv)]

                lWeight := ssn.NamespaceInfo[lv].GetWeight()
                rWeight := ssn.NamespaceInfo[rv].GetWeight()

                klog.V(4).Infof("DRF NamespaceOrderFn: <%v> share state: %f, weight %v, <%v> share state: %f, weight %v",
                        lv, lOpt.share, lWeight, rv, rOpt.share, rWeight)

                lWeightedShare := lOpt.share / float64(lWeight)
                rWeightedShare := rOpt.share / float64(rWeight)

                metrics.UpdateNamespaceWeight(string(lv), lWeight)
                metrics.UpdateNamespaceWeight(string(rv), rWeight)
                metrics.UpdateNamespaceWeightedShare(string(lv), lWeightedShare)
                metrics.UpdateNamespaceWeightedShare(string(rv), rWeightedShare)

                if lWeightedShare == rWeightedShare {
                        return 0
                }

                if lWeightedShare < rWeightedShare {
                        return -1
                }

                return 1
        }

        if namespaceOrderEnabled {
                ssn.AddNamespaceOrderFn(drf.Name(), namespaceOrderFn)
        }

        // Register event handlers.
        ssn.AddEventHandler(&framework.EventHandler{
                AllocateFunc: func(event *framework.Event) {
                        attr := drf.jobAttrs[event.Task.Job]
                        attr.allocated.Add(event.Task.Resreq)

                        job := ssn.Jobs[event.Task.Job]
                        drf.updateJobShare(job.Namespace, job.Name, attr)

                        nsShare := -1.0
                        if namespaceOrderEnabled {
                                nsOpt := drf.namespaceOpts[event.Task.Namespace]
                                nsOpt.allocated.Add(event.Task.Resreq)

                                drf.updateNamespaceShare(event.Task.Namespace, nsOpt)
                                nsShare = nsOpt.share
                        }
                        if hierarchyEnabled {
                                queue := ssn.Queues[job.Queue]

                                drf.totalAllocated.Add(event.Task.Resreq)
                                drf.UpdateHierarchicalShare(drf.hierarchicalRoot, drf.totalAllocated, job, attr, queue.Hierarchy, queue.Weights)
                        }

                        klog.V(4).Infof("DRF AllocateFunc: task <%v/%v>, resreq <%v>,  share <%v>, namespace share <%v>",
                                event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share, nsShare)
                },
                DeallocateFunc: func(event *framework.Event) {
                        attr := drf.jobAttrs[event.Task.Job]
                        attr.allocated.Sub(event.Task.Resreq)

                        job := ssn.Jobs[event.Task.Job]
                        drf.updateJobShare(job.Namespace, job.Name, attr)

                        nsShare := -1.0
                        if namespaceOrderEnabled {
                                nsOpt := drf.namespaceOpts[event.Task.Namespace]
                                nsOpt.allocated.Sub(event.Task.Resreq)

                                drf.updateNamespaceShare(event.Task.Namespace, nsOpt)
                                nsShare = nsOpt.share
                        }

                        if hierarchyEnabled {
                                queue := ssn.Queues[job.Queue]
                                drf.totalAllocated.Sub(event.Task.Resreq)
                                drf.UpdateHierarchicalShare(drf.hierarchicalRoot, drf.totalAllocated, job, attr, queue.Hierarchy, queue.Weights)
                        }

                        klog.V(4).Infof("DRF EvictFunc: task <%v/%v>, resreq <%v>,  share <%v>, namespace share <%v>",
                                event.Task.Namespace, event.Task.Name, event.Task.Resreq, attr.share, nsShare)
                },
        })
}

func (drf *drfPlugin) updateNamespaceShare(namespaceName string, attr *drfAttr) {
        drf.updateShare(attr)
        metrics.UpdateNamespaceShare(namespaceName, attr.share)
}

// build hierarchy if the node does not exist
func (drf *drfPlugin) buildHierarchy(root *hierarchicalNode, job *api.JobInfo, attr *drfAttr,
        hierarchy, hierarchicalWeights string) {
        inode := root
        paths := strings.Split(hierarchy, "/")
        weights := strings.Split(hierarchicalWeights, "/")

        for i := 1; i < len(paths); i++ {
                if child, ok := inode.children[paths[i]]; ok {
                        inode = child
                } else {
                        fweight, _ := strconv.ParseFloat(weights[i], 64)
                        if fweight < 1 {
                                fweight = 1
                        }
                        child = &hierarchicalNode{
                                weight:    fweight,
                                hierarchy: paths[i],
                                request:   api.EmptyResource(),
                                attr: &drfAttr{
                                        allocated: api.EmptyResource(),
                                },
                                children: make(map[string]*hierarchicalNode),
                        }
                        klog.V(4).Infof("Node %s added to %s, weight %f",
                                child.hierarchy, inode.hierarchy, fweight)
                        inode.children[paths[i]] = child
                        child.parent = inode
                        inode = child
                }
        }

        child := &hierarchicalNode{
                weight:    1,
                attr:      attr,
                hierarchy: string(job.UID),
                request:   job.TotalRequest.Clone(),
                children:  nil,
        }
        inode.children[string(job.UID)] = child
        // update drf attribute bottom up
        klog.V(4).Infof("Job <%s/%s> added to %s, weights %s, attr %v, total request: %s",
                job.Namespace, job.Name, inode.hierarchy, hierarchicalWeights, child.attr, job.TotalRequest)
}

// updateNamespaceShare updates the node attribute recursively
func (drf *drfPlugin) updateHierarchicalShare(node *hierarchicalNode,
        demandingResources map[v1.ResourceName]bool) {
        if node.children == nil {
                node.saturated = resourceSaturated(node.attr.allocated,
                        node.request, demandingResources)
                klog.V(4).Infof("Update hierarchical node %s, share %f, dominant %s, resource %v, saturated: %t",
                        node.hierarchy, node.attr.share, node.attr.dominantResource, node.attr.allocated, node.saturated)
        } else {
                var mdr float64 = 1
                // get minimun dominant resource share
                for _, child := range node.children {
                        drf.updateHierarchicalShare(child, demandingResources)
                        // skip empty child and saturated child
                        if child.attr.share != 0 && !child.saturated {
                                _, resShare := drf.calculateShare(child.attr.allocated, drf.totalResource)
                                if resShare < mdr {
                                        mdr = resShare
                                }
                        }
                }

                node.attr.allocated = api.EmptyResource()
                saturated := true
                for _, child := range node.children {
                        if !child.saturated {
                                saturated = false
                        }
                        // only consider non-empty children
                        if child.attr.share != 0 {
                                // saturated child is not scaled
                                if child.saturated {
                                        t := child.attr.allocated
                                        node.attr.allocated.Add(t)
                                } else {
                                        t := child.attr.allocated.Clone().Multi(mdr / child.attr.share)
                                        node.attr.allocated.Add(t)
                                }
                        }
                }
                node.attr.dominantResource, node.attr.share = drf.calculateShare(
                        node.attr.allocated, drf.totalResource)
                node.saturated = saturated
                klog.V(4).Infof("Update hierarchical node %s, share %f, dominant resource %s, resource %v, saturated: %t",
                        node.hierarchy, node.attr.share, node.attr.dominantResource, node.attr.allocated, node.saturated)
        }
}

func (drf *drfPlugin) UpdateHierarchicalShare(root *hierarchicalNode, totalAllocated *api.Resource, job *api.JobInfo, attr *drfAttr, hierarchy, hierarchicalWeights string) {
        // filter out demanding resources
        demandingResources := map[v1.ResourceName]bool{}
        for _, rn := range drf.totalResource.ResourceNames() {
                if totalAllocated.Get(rn) < drf.totalResource.Get(rn) {
                        demandingResources[rn] = true
                }
        }
        drf.buildHierarchy(root, job, attr, hierarchy, hierarchicalWeights)
        drf.updateHierarchicalShare(root, demandingResources)
}

func (drf *drfPlugin) updateJobShare(jobNs, jobName string, attr *drfAttr) {
        drf.updateShare(attr)
        metrics.UpdateJobShare(jobNs, jobName, attr.share)
}

func (drf *drfPlugin) updateShare(attr *drfAttr) {
        attr.dominantResource, attr.share = drf.calculateShare(attr.allocated, drf.totalResource)
}

func (drf *drfPlugin) calculateShare(allocated, totalResource *api.Resource) (string, float64) {
        res := float64(0)
        dominantResource := ""
        for _, rn := range totalResource.ResourceNames() {
                share := helpers.Share(allocated.Get(rn), totalResource.Get(rn))
                if share > res {
                        res = share
                        dominantResource = string(rn)
                }
        }

        return dominantResource, res
}

func (drf *drfPlugin) OnSessionClose(session *framework.Session) {
        // Clean schedule data.
        drf.totalResource = api.EmptyResource()
        drf.totalAllocated = api.EmptyResource()
        drf.jobAttrs = map[api.JobID]*drfAttr{}
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import (
        v1 "k8s.io/api/core/v1"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
        "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        nodeinfov1alpha1 "volcano.sh/apis/pkg/apis/nodeinfo/v1alpha1"
        "volcano.sh/volcano/pkg/scheduler/api"
)

// TopologyHint is a struct containing the NUMANodeAffinity for a Container
type TopologyHint struct {
        NUMANodeAffinity bitmask.BitMask
        // Preferred is set to true when the NUMANodeAffinity encodes a preferred
        // allocation for the Container. It is set to false otherwise.
        Preferred bool
}

// Policy is an interface for topology manager policy
type Policy interface {
        // Predicate Get the best hit.
        Predicate(providersHints []map[string][]TopologyHint) (TopologyHint, bool)
}

// HintProvider is an interface for components that want to collaborate to
// achieve globally optimal concrete resource alignment with respect to
// NUMA locality.
type HintProvider interface {
        // Name returns provider name used for register and logging.
        Name() string
        // GetTopologyHints returns hints if this hint provider has a preference,
        GetTopologyHints(container *v1.Container, topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets) map[string][]TopologyHint
        Allocate(container *v1.Container, bestHit *TopologyHint, topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets) map[string]cpuset.CPUSet
}

// GetPolicy return the interface matched the input task topology config
func GetPolicy(node *api.NodeInfo, numaNodes []int) Policy {
        switch batch.NumaPolicy(node.NumaSchedulerInfo.Policies[nodeinfov1alpha1.TopologyManagerPolicy]) {
        case batch.None:
                return NewPolicyNone(numaNodes)
        case batch.BestEffort:
                return NewPolicyBestEffort(numaNodes)
        case batch.Restricted:
                return NewPolicyRestricted(numaNodes)
        case batch.SingleNumaNode:
                return NewPolicySingleNumaNode(numaNodes)
        }

        return &policyNone{}
}

// AccumulateProvidersHints return all TopologyHint collection from different providers
func AccumulateProvidersHints(container *v1.Container,
        topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets,
        hintProviders []HintProvider) (providersHints []map[string][]TopologyHint) {
        for _, provider := range hintProviders {
                hints := provider.GetTopologyHints(container, topoInfo, resNumaSets)
                providersHints = append(providersHints, hints)
        }

        return providersHints
}

// Allocate return all resource assignment collection from different providers
func Allocate(container *v1.Container, bestHit *TopologyHint,
        topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets, hintProviders []HintProvider) map[string]cpuset.CPUSet {
        allResAlloc := make(map[string]cpuset.CPUSet)
        for _, provider := range hintProviders {
                resAlloc := provider.Allocate(container, bestHit, topoInfo, resNumaSets)
                for resName, assign := range resAlloc {
                        allResAlloc[resName] = assign
                }
        }

        return allResAlloc
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import (
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
)

func filterProvidersHints(providersHints []map[string][]TopologyHint) [][]TopologyHint {
        var allProviderHints [][]TopologyHint
        for _, hints := range providersHints {
                // If hints is nil, insert a single, preferred any-numa hint into allProviderHints.
                if len(hints) == 0 {
                        klog.Infof("[numatopo] Hint Provider has no preference for NUMA affinity with any resource")
                        allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}})
                        continue
                }

                // Otherwise, accumulate the hints for each resource type into allProviderHints.
                for resource := range hints {
                        if hints[resource] == nil {
                                klog.Infof("[numatopo] Hint Provider has no preference for NUMA affinity with resource '%s'", resource)
                                allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}})
                                continue
                        }

                        if len(hints[resource]) == 0 {
                                klog.Infof("[numatopo] Hint Provider has no possible NUMA affinities for resource '%s'", resource)
                                allProviderHints = append(allProviderHints, []TopologyHint{{nil, false}})
                                continue
                        }

                        allProviderHints = append(allProviderHints, hints[resource])
                }
        }
        return allProviderHints
}

func mergeFilteredHints(numaNodes []int, filteredHints [][]TopologyHint) TopologyHint {
        // Set the default affinity as an any-numa affinity containing the list
        // of NUMA Nodes available on this machine.
        defaultAffinity, _ := bitmask.NewBitMask(numaNodes...)

        // Set the bestHint to return from this function as {nil false}.
        // This will only be returned if no better hint can be found when
        // merging hints from each hint provider.
        bestHint := TopologyHint{defaultAffinity, false}
        iterateAllProviderTopologyHints(filteredHints, func(permutation []TopologyHint) {
                // Get the NUMANodeAffinity from each hint in the permutation and see if any
                // of them encode unpreferred allocations.
                mergedHint := mergePermutation(numaNodes, permutation)
                // Only consider mergedHints that result in a NUMANodeAffinity > 0 to
                // replace the current bestHint.
                if mergedHint.NUMANodeAffinity.Count() == 0 {
                        return
                }

                // If the current bestHint is non-preferred and the new mergedHint is
                // preferred, always choose the preferred hint over the non-preferred one.
                if mergedHint.Preferred && !bestHint.Preferred {
                        bestHint = mergedHint
                        return
                }

                // If the current bestHint is preferred and the new mergedHint is
                // non-preferred, never update bestHint, regardless of mergedHint's
                // narowness.
                if !mergedHint.Preferred && bestHint.Preferred {
                        return
                }

                // If mergedHint and bestHint has the same preference, only consider
                // mergedHints that have a narrower NUMANodeAffinity than the
                // NUMANodeAffinity in the current bestHint.
                if !mergedHint.NUMANodeAffinity.IsNarrowerThan(bestHint.NUMANodeAffinity) {
                        return
                }

                // In all other cases, update bestHint to the current mergedHint
                bestHint = mergedHint
        })

        return bestHint
}

// Iterate over all permutations of hints in 'allProviderHints [][]TopologyHint'.
//
// This procedure is implemented as a recursive function over the set of hints
// in 'allproviderHints[i]'. It applies the function 'callback' to each
// permutation as it is found. It is the equivalent of:
//
// for i := 0; i < len(providerHints[0]); i++
//     for j := 0; j < len(providerHints[1]); j++
//         for k := 0; k < len(providerHints[2]); k++
//             ...
//             for z := 0; z < len(providerHints[-1]); z++
//                 permutation := []TopologyHint{
//                     providerHints[0][i],
//                     providerHints[1][j],
//                     providerHints[2][k],
//                     ...
//                     providerHints[-1][z]
//                 }
//                 callback(permutation)
func iterateAllProviderTopologyHints(allProviderHints [][]TopologyHint, callback func([]TopologyHint)) {
        // Internal helper function to accumulate the permutation before calling the callback.
        var iterate func(i int, accum []TopologyHint)
        iterate = func(i int, accum []TopologyHint) {
                // Base case: we have looped through all providers and have a full permutation.
                if i == len(allProviderHints) {
                        callback(accum)
                        return
                }

                // Loop through all hints for provider 'i', and recurse to build the
                // the permutation of this hint with all hints from providers 'i++'.
                for j := range allProviderHints[i] {
                        iterate(i+1, append(accum, allProviderHints[i][j]))
                }
        }
        iterate(0, []TopologyHint{})
}

// Merge a TopologyHints permutation to a single hint by performing a bitwise-AND
// of their affinity masks. The hint shall be preferred if all hits in the permutation
// are preferred.
func mergePermutation(numaNodes []int, permutation []TopologyHint) TopologyHint {
        // Get the NUMANodeAffinity from each hint in the permutation and see if any
        // of them encode unpreferred allocations.
        preferred := true
        defaultAffinity, _ := bitmask.NewBitMask(numaNodes...)
        var numaAffinities []bitmask.BitMask
        for _, hint := range permutation {
                // Only consider hints that have an actual NUMANodeAffinity set.
                if hint.NUMANodeAffinity == nil {
                        numaAffinities = append(numaAffinities, defaultAffinity)
                } else {
                        numaAffinities = append(numaAffinities, hint.NUMANodeAffinity)
                }

                if !hint.Preferred {
                        preferred = false
                }
        }

        // Merge the affinities using a bitwise-and operation.
        mergedAffinity := bitmask.And(defaultAffinity, numaAffinities...)
        // Build a mergedHint from the merged affinity mask, indicating if an
        // preferred allocation was used to generate the affinity mask or not.
        return TopologyHint{mergedAffinity, preferred}
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import "k8s.io/klog"

type policyBestEffort struct {
        numaNodes []int
}

// NewPolicyBestEffort return a new policy interface
func NewPolicyBestEffort(numaNodes []int) Policy {
        return &policyBestEffort{numaNodes: numaNodes}
}

func (p *policyBestEffort) canAdmitPodResult(hint *TopologyHint) bool {
        return true
}

func (p *policyBestEffort) Predicate(providersHints []map[string][]TopologyHint) (TopologyHint, bool) {
        filteredProvidersHints := filterProvidersHints(providersHints)
        bestHint := mergeFilteredHints(p.numaNodes, filteredProvidersHints)
        admit := p.canAdmitPodResult(&bestHint)

        klog.V(4).Infof("bestHint: %v admit %v\n", bestHint, admit)
        return bestHint, admit
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

type policyNone struct {
        numaNodes []int
}

// NewPolicyNone return a new policy interface
func NewPolicyNone(numaNodes []int) Policy {
        return &policyNone{numaNodes: numaNodes}
}

func (policy *policyNone) canAdmitPodResult(hint *TopologyHint) bool {
        return true
}

func (policy *policyNone) Predicate(providersHints []map[string][]TopologyHint) (TopologyHint, bool) {
        return TopologyHint{}, policy.canAdmitPodResult(nil)
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import "k8s.io/klog"

type policyRestricted struct {
        numaNodes []int
}

// NewPolicyRestricted return a new policy interface
func NewPolicyRestricted(numaNodes []int) Policy {
        return &policyRestricted{numaNodes: numaNodes}
}

func (p *policyRestricted) canAdmitPodResult(hint *TopologyHint) bool {
        return hint.Preferred
}

func (p *policyRestricted) Predicate(providersHints []map[string][]TopologyHint) (TopologyHint, bool) {
        filteredHints := filterProvidersHints(providersHints)
        bestHint := mergeFilteredHints(p.numaNodes, filteredHints)
        admit := p.canAdmitPodResult(&bestHint)

        klog.V(4).Infof("bestHint: %v admit %v\n", bestHint, admit)
        return bestHint, admit
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import "k8s.io/klog"

type policySingleNumaNode struct {
        numaNodes []int
}

// NewPolicySingleNumaNode return a new policy interface
func NewPolicySingleNumaNode(numaNodes []int) Policy {
        return &policySingleNumaNode{numaNodes: numaNodes}
}

func (policy *policySingleNumaNode) canAdmitPodResult(hint *TopologyHint) bool {
        return hint.Preferred
}

// Return hints that have valid bitmasks with exactly one bit set.
func filterSingleNumaHints(allResourcesHints [][]TopologyHint) [][]TopologyHint {
        var filteredResourcesHints [][]TopologyHint
        for _, oneResourceHints := range allResourcesHints {
                var filtered []TopologyHint
                for _, hint := range oneResourceHints {
                        if hint.NUMANodeAffinity == nil && hint.Preferred {
                                filtered = append(filtered, hint)
                        }
                        if hint.NUMANodeAffinity != nil && hint.NUMANodeAffinity.Count() == 1 && hint.Preferred {
                                filtered = append(filtered, hint)
                        }
                }
                filteredResourcesHints = append(filteredResourcesHints, filtered)
        }
        return filteredResourcesHints
}

func (policy *policySingleNumaNode) Predicate(providersHints []map[string][]TopologyHint) (TopologyHint, bool) {
        filteredHints := filterProvidersHints(providersHints)
        singleNumaHints := filterSingleNumaHints(filteredHints)
        bestHint := mergeFilteredHints(policy.numaNodes, singleNumaHints)
        klog.V(4).Infof("bestHint: %v\n", bestHint)
        admit := policy.canAdmitPodResult(&bestHint)
        return bestHint, admit
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpumanager

import (
        "fmt"
        "sort"

        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
)

type cpuAccumulator struct {
        topo          *topology.CPUTopology
        details       topology.CPUDetails
        numCPUsNeeded int
        result        cpuset.CPUSet
}

func newCPUAccumulator(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int) *cpuAccumulator {
        return &cpuAccumulator{
                topo:          topo,
                details:       topo.CPUDetails.KeepOnly(availableCPUs),
                numCPUsNeeded: numCPUs,
                result:        cpuset.NewCPUSet(),
        }
}

func (a *cpuAccumulator) take(cpus cpuset.CPUSet) {
        a.result = a.result.Union(cpus)
        a.details = a.details.KeepOnly(a.details.CPUs().Difference(a.result))
        a.numCPUsNeeded -= cpus.Size()
}

// isSocketFree Returns true if the supplied socket is fully available in `topoDetails`.
func (a *cpuAccumulator) isSocketFree(socketID int) bool {
        return a.details.CPUsInSockets(socketID).Size() == a.topo.CPUsPerSocket()
}

// isCoreFree Returns true if the supplied core is fully available in `topoDetails`.
func (a *cpuAccumulator) isCoreFree(coreID int) bool {
        return a.details.CPUsInCores(coreID).Size() == a.topo.CPUsPerCore()
}

// freeSockets Returns free socket IDs as a slice sorted by:
// - socket ID, ascending.
func (a *cpuAccumulator) freeSockets() []int {
        return a.details.Sockets().Filter(a.isSocketFree).ToSlice()
}

// freeCores Returns core IDs as a slice sorted by:
// - the number of whole available cores on the socket, ascending
// - socket ID, ascending
// - core ID, ascending
func (a *cpuAccumulator) freeCores() []int {
        socketIDs := a.details.Sockets().ToSliceNoSort()
        sort.Slice(socketIDs,
                func(i, j int) bool {
                        iCores := a.details.CoresInSockets(socketIDs[i]).Filter(a.isCoreFree)
                        jCores := a.details.CoresInSockets(socketIDs[j]).Filter(a.isCoreFree)
                        return iCores.Size() < jCores.Size() || socketIDs[i] < socketIDs[j]
                })

        coreIDs := []int{}
        for _, s := range socketIDs {
                coreIDs = append(coreIDs, a.details.CoresInSockets(s).Filter(a.isCoreFree).ToSlice()...)
        }
        return coreIDs
}

// freeCPUs Returns CPU IDs as a slice sorted by:
// - socket affinity with result
// - number of CPUs available on the same socket
// - number of CPUs available on the same core
// - socket ID.
// - core ID.
func (a *cpuAccumulator) freeCPUs() []int {
        result := []int{}
        cores := a.details.Cores().ToSlice()

        sort.Slice(
                cores,
                func(i, j int) bool {
                        iCore := cores[i]
                        jCore := cores[j]

                        iCPUs := a.topo.CPUDetails.CPUsInCores(iCore).ToSlice()
                        jCPUs := a.topo.CPUDetails.CPUsInCores(jCore).ToSlice()

                        iSocket := a.topo.CPUDetails[iCPUs[0]].SocketID
                        jSocket := a.topo.CPUDetails[jCPUs[0]].SocketID

                        // Compute the number of CPUs in the result reside on the same socket
                        // as each core.
                        iSocketColoScore := a.topo.CPUDetails.CPUsInSockets(iSocket).Intersection(a.result).Size()
                        jSocketColoScore := a.topo.CPUDetails.CPUsInSockets(jSocket).Intersection(a.result).Size()

                        // Compute the number of available CPUs available on the same socket
                        // as each core.
                        iSocketFreeScore := a.details.CPUsInSockets(iSocket).Size()
                        jSocketFreeScore := a.details.CPUsInSockets(jSocket).Size()

                        // Compute the number of available CPUs on each core.
                        iCoreFreeScore := a.details.CPUsInCores(iCore).Size()
                        jCoreFreeScore := a.details.CPUsInCores(jCore).Size()

                        return iSocketColoScore > jSocketColoScore ||
                                iSocketFreeScore < jSocketFreeScore ||
                                iCoreFreeScore < jCoreFreeScore ||
                                iSocket < jSocket ||
                                iCore < jCore
                })

        // For each core, append sorted CPU IDs to result.
        for _, core := range cores {
                result = append(result, a.details.CPUsInCores(core).ToSlice()...)
        }
        return result
}

func (a *cpuAccumulator) needs(n int) bool {
        return a.numCPUsNeeded >= n
}

func (a *cpuAccumulator) isSatisfied() bool {
        return a.numCPUsNeeded < 1
}

func (a *cpuAccumulator) isFailed() bool {
        return a.numCPUsNeeded > a.details.CPUs().Size()
}

// takeByTopology return the assigned cpuset
func takeByTopology(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int) (cpuset.CPUSet, error) {
        acc := newCPUAccumulator(topo, availableCPUs, numCPUs)
        if acc.isSatisfied() {
                return acc.result, nil
        }
        if acc.isFailed() {
                return cpuset.NewCPUSet(), fmt.Errorf("not enough cpus available to satisfy request")
        }

        // Algorithm: topology-aware best-fit
        // 1. Acquire whole sockets, if available and the container requires at
        //    least a socket's-worth of CPUs.
        if acc.needs(acc.topo.CPUsPerSocket()) {
                for _, s := range acc.freeSockets() {
                        klog.V(4).Infof("[cpumanager] takeByTopology: claiming socket [%d]", s)
                        acc.take(acc.details.CPUsInSockets(s))
                        if acc.isSatisfied() {
                                return acc.result, nil
                        }
                        if !acc.needs(acc.topo.CPUsPerSocket()) {
                                break
                        }
                }
        }

        // 2. Acquire whole cores, if available and the container requires at least
        //    a core's-worth of CPUs.
        if acc.needs(acc.topo.CPUsPerCore()) {
                for _, c := range acc.freeCores() {
                        klog.V(4).Infof("[cpumanager] takeByTopology: claiming core [%d]", c)
                        acc.take(acc.details.CPUsInCores(c))
                        if acc.isSatisfied() {
                                return acc.result, nil
                        }
                        if !acc.needs(acc.topo.CPUsPerCore()) {
                                break
                        }
                }
        }

        // 3. Acquire single threads, preferring to fill partially-allocated cores
        //    on the same sockets as the whole cores we have already taken in this
        //    allocation.
        for _, c := range acc.freeCPUs() {
                klog.V(4).Infof("[cpumanager] takeByTopology: claiming CPU [%d]", c)
                if acc.needs(1) {
                        acc.take(cpuset.NewCPUSet(c))
                }
                if acc.isSatisfied() {
                        return acc.result, nil
                }
        }

        return cpuset.NewCPUSet(), fmt.Errorf("failed to allocate cpus")
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpumanager

import (
        "math"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
        "k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
        "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/plugins/numaaware/policy"
)

type cpuMng struct {
}

// NewProvider return a new provider
func NewProvider() policy.HintProvider {
        return &cpuMng{}
}

// Name return the cpu manager name
func (mng *cpuMng) Name() string {
        return "cpuMng"
}

// guaranteedCPUs return the intger num of request cpu
func guaranteedCPUs(container *v1.Container) int {
        cpuQuantity := container.Resources.Requests[v1.ResourceCPU]
        if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() {
                return 0
        }

        return int(cpuQuantity.Value())
}

// generateCPUTopologyHints return the numa topology hints based on
// - availableCPUs
func generateCPUTopologyHints(availableCPUs cpuset.CPUSet, CPUDetails topology.CPUDetails, request int) []policy.TopologyHint {
        minAffinitySize := CPUDetails.NUMANodes().Size()
        hints := []policy.TopologyHint{}
        bitmask.IterateBitMasks(CPUDetails.NUMANodes().ToSlice(), func(mask bitmask.BitMask) {
                // First, update minAffinitySize for the current request size.
                cpusInMask := CPUDetails.CPUsInNUMANodes(mask.GetBits()...).Size()
                if cpusInMask >= request && mask.Count() < minAffinitySize {
                        minAffinitySize = mask.Count()
                }

                // Then check to see if we have enough CPUs available on the current
                // numa node bitmask to satisfy the CPU request.
                numMatching := 0
                // Finally, check to see if enough available CPUs remain on the current
                // NUMA node combination to satisfy the CPU request.
                for _, c := range availableCPUs.ToSlice() {
                        if mask.IsSet(CPUDetails[c].NUMANodeID) {
                                numMatching++
                        }
                }

                // If they don't, then move onto the next combination.
                if numMatching < request {
                        return
                }

                // Otherwise, create a new hint from the numa node bitmask and add it to the
                // list of hints.  We set all hint preferences to 'false' on the first
                // pass through.
                hints = append(hints, policy.TopologyHint{
                        NUMANodeAffinity: mask,
                        Preferred:        false,
                })
        })

        // Loop back through all hints and update the 'Preferred' field based on
        // counting the number of bits sets in the affinity mask and comparing it
        // to the minAffinitySize. Only those with an equal number of bits set (and
        // with a minimal set of numa nodes) will be considered preferred.
        for i := range hints {
                if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
                        hints[i].Preferred = true
                }
        }

        return hints
}

func (mng *cpuMng) GetTopologyHints(container *v1.Container,
        topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets) map[string][]policy.TopologyHint {
        if _, ok := container.Resources.Requests[v1.ResourceCPU]; !ok {
                klog.Warningf("container %s has no cpu request", container.Name)
                return nil
        }

        requestNum := guaranteedCPUs(container)
        if requestNum == 0 {
                klog.Warningf(" the cpu request isn't  integer in container %s", container.Name)
                return nil
        }

        cputopo := &topology.CPUTopology{
                NumCPUs:    topoInfo.CPUDetail.CPUs().Size(),
                NumCores:   topoInfo.CPUDetail.Cores().Size() * topoInfo.CPUDetail.Sockets().Size(),
                NumSockets: topoInfo.CPUDetail.Sockets().Size(),
                CPUDetails: topoInfo.CPUDetail,
        }

        reserved := cpuset.NewCPUSet()
        reservedCPUs, ok := topoInfo.ResReserved[v1.ResourceCPU]
        if ok {
                // Take the ceiling of the reservation, since fractional CPUs cannot be
                // exclusively allocated.
                reservedCPUsFloat := float64(reservedCPUs.MilliValue()) / 1000
                numReservedCPUs := int(math.Ceil(reservedCPUsFloat))
                reserved, _ = takeByTopology(cputopo, cputopo.CPUDetails.CPUs(), numReservedCPUs)
                klog.V(4).Infof("[cpumanager] reserve cpuset :%v", reserved)
        }

        availableCPUSet, ok := resNumaSets[string(v1.ResourceCPU)]
        if !ok {
                klog.Warningf("no cpu resource")
                return nil
        }

        availableCPUSet = availableCPUSet.Difference(reserved)
        klog.V(4).Infof("requested: %d, availableCPUSet: %v", requestNum, availableCPUSet)
        return map[string][]policy.TopologyHint{
                string(v1.ResourceCPU): generateCPUTopologyHints(availableCPUSet, topoInfo.CPUDetail, requestNum),
        }
}

func (mng *cpuMng) Allocate(container *v1.Container, bestHit *policy.TopologyHint,
        topoInfo *api.NumatopoInfo, resNumaSets api.ResNumaSets) map[string]cpuset.CPUSet {
        cputopo := &topology.CPUTopology{
                NumCPUs:    topoInfo.CPUDetail.CPUs().Size(),
                NumCores:   topoInfo.CPUDetail.Cores().Size() * topoInfo.CPUDetail.Sockets().Size(),
                NumSockets: topoInfo.CPUDetail.Sockets().Size(),
                CPUDetails: topoInfo.CPUDetail,
        }

        reserved := cpuset.NewCPUSet()
        reservedCPUs, ok := topoInfo.ResReserved[v1.ResourceCPU]
        if ok {
                // Take the ceiling of the reservation, since fractional CPUs cannot be
                // exclusively allocated.
                reservedCPUsFloat := float64(reservedCPUs.MilliValue()) / 1000
                numReservedCPUs := int(math.Ceil(reservedCPUsFloat))
                reserved, _ = takeByTopology(cputopo, cputopo.CPUDetails.CPUs(), numReservedCPUs)
                klog.V(3).Infof("[cpumanager] reserve cpuset :%v", reserved)
        }

        requestNum := guaranteedCPUs(container)
        availableCPUSet := resNumaSets[string(v1.ResourceCPU)]
        availableCPUSet = availableCPUSet.Difference(reserved)

        klog.V(4).Infof("alignedCPUs: %v requestNum: %v bestHit %v", availableCPUSet, requestNum, bestHit)

        result := cpuset.NewCPUSet()
        if bestHit.NUMANodeAffinity != nil {
                alignedCPUs := cpuset.NewCPUSet()
                for _, numaNodeID := range bestHit.NUMANodeAffinity.GetBits() {
                        alignedCPUs = alignedCPUs.Union(availableCPUSet.Intersection(cputopo.CPUDetails.CPUsInNUMANodes(numaNodeID)))
                }

                numAlignedToAlloc := alignedCPUs.Size()
                if requestNum < numAlignedToAlloc {
                        numAlignedToAlloc = requestNum
                }

                alignedCPUs, err := takeByTopology(cputopo, alignedCPUs, numAlignedToAlloc)
                if err != nil {
                        return map[string]cpuset.CPUSet{
                                string(v1.ResourceCPU): cpuset.NewCPUSet(),
                        }
                }

                result = result.Union(alignedCPUs)
        }

        // Get any remaining CPUs from what's leftover after attempting to grab aligned ones.
        remainingCPUs, err := takeByTopology(cputopo, availableCPUSet.Difference(result), requestNum-result.Size())
        if err != nil {
                return map[string]cpuset.CPUSet{
                        string(v1.ResourceCPU): cpuset.NewCPUSet(),
                }
        }

        result = result.Union(remainingCPUs)

        return map[string]cpuset.CPUSet{
                string(v1.ResourceCPU): result,
        }
}

/*
Copyright 2020 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package predicates

import (
        "fmt"
        "sync"

        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        batch "volcano.sh/apis/pkg/apis/batch/v1alpha1"
)

type predicateCache struct {
        sync.RWMutex
        cache map[string]map[string]bool //key_1: nodename key_2:pod uid
}

// predicateCacheNew return cache map
func predicateCacheNew() *predicateCache {
        return &predicateCache{
                cache: make(map[string]map[string]bool),
        }
}

// getPodTemplateUID return pod template key
func getPodTemplateUID(pod *v1.Pod) string {
        uid, found := pod.Annotations[batch.PodTemplateKey]
        if !found {
                return ""
        }

        return uid
}

// PredicateWithCache: check the predicate result existed in cache
func (pc *predicateCache) PredicateWithCache(nodeName string, pod *v1.Pod) (bool, error) {
        podTemplateUID := getPodTemplateUID(pod)
        if podTemplateUID == "" {
                return false, fmt.Errorf("no anonation of volcano.sh/template-uid in pod %s", pod.Name)
        }

        pc.RLock()
        defer pc.RUnlock()
        if nodeCache, exist := pc.cache[nodeName]; exist {
                if result, exist := nodeCache[podTemplateUID]; exist {
                        klog.V(4).Infof("Predicate node %s and pod %s result %v", nodeName, pod.Name, result)
                        return result, nil
                }
        }

        return false, fmt.Errorf("no information of node %s and pod %s in predicate cache", nodeName, pod.Name)
}

// UpdateCache update cache data
func (pc *predicateCache) UpdateCache(nodeName string, pod *v1.Pod, fit bool) {
        podTemplateUID := getPodTemplateUID(pod)
        if podTemplateUID == "" {
                klog.V(3).Infof("Don't find pod %s template uid", pod.Name)
                return
        }

        pc.Lock()
        defer pc.Unlock()

        if _, exist := pc.cache[nodeName]; !exist {
                podCache := make(map[string]bool)
                podCache[podTemplateUID] = fit
                pc.cache[nodeName] = podCache
        } else {
                pc.cache[nodeName][podTemplateUID] = fit
        }
}

/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package predicates

import (
        "fmt"

        v1 "k8s.io/api/core/v1"

        "volcano.sh/volcano/pkg/scheduler/api"
)

// checkNodeGPUSharingPredicate checks if a gpu sharing pod can be scheduled on a node.
func checkNodeGPUSharingPredicate(pod *v1.Pod, nodeInfo *api.NodeInfo) (bool, error) {
        // no gpu sharing request
        if api.GetGPUResourceOfPod(pod) <= 0 {
                return true, nil
        }

        id := predicateGPU(pod, nodeInfo)
        if id < 0 {
                return false, fmt.Errorf("no enough gpu memory on single device of node %s", nodeInfo.Name)
        }
        return true, nil
}

// predicateGPU returns the available GPU ID
func predicateGPU(pod *v1.Pod, node *api.NodeInfo) int {
        gpuRequest := api.GetGPUResourceOfPod(pod)
        allocatableGPUs := node.GetDevicesIdleGPUMemory()

        for devID := 0; devID < len(allocatableGPUs); devID++ {
                availableGPU, ok := allocatableGPUs[devID]
                if ok {
                        if availableGPU >= gpuRequest {
                                return devID
                        }
                }
        }

        return -1
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package predicates

import (
        "context"
        "fmt"
        "strings"

        v1 "k8s.io/api/core/v1"

        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/scheduler/apis/config"
        "k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity"
        "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeaffinity"
        "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeports"
        "k8s.io/kubernetes/pkg/scheduler/framework/plugins/nodeunschedulable"
        "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration"
        k8sframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/plugins/util"
        "volcano.sh/volcano/pkg/scheduler/plugins/util/k8s"
)

const (
        // PluginName indicates name of volcano scheduler plugin.
        PluginName = "predicates"

        // GPUSharingPredicate is the key for enabling GPU Sharing Predicate in YAML
        GPUSharingPredicate = "predicate.GPUSharingEnable"

        // CachePredicate control cache predicate feature
        CachePredicate = "predicate.CacheEnable"

        // ProportionalPredicate is the key for enabling Proportional Predicate in YAML
        ProportionalPredicate = "predicate.ProportionalEnable"
        // ProportionalResource is the key for additional resource key name
        ProportionalResource = "predicate.resources"
        // ProportionalResourcesPrefix is the key prefix for additional resource key name
        ProportionalResourcesPrefix = ProportionalResource + "."
)

type predicatesPlugin struct {
        // Arguments given for the plugin
        pluginArguments framework.Arguments
}

// New return predicate plugin
func New(arguments framework.Arguments) framework.Plugin {
        return &predicatesPlugin{pluginArguments: arguments}
}

func (pp *predicatesPlugin) Name() string {
        return PluginName
}

type baseResource struct {
        CPU    float64
        Memory float64
}

type predicateEnable struct {
        gpuSharingEnable   bool
        cacheEnable        bool
        proportionalEnable bool
        proportional       map[v1.ResourceName]baseResource
}

func enablePredicate(args framework.Arguments) predicateEnable {
        /*
           User Should give predicatesEnable in this format(predicate.GPUSharingEnable).
           Currently supported only GPUSharing predicate checks.

           actions: "reclaim, allocate, backfill, preempt"
           tiers:
           - plugins:
             - name: priority
             - name: gang
             - name: conformance
           - plugins:
             - name: drf
             - name: predicates
               arguments:
                 predicate.GPUSharingEnable: true
                 predicate.CacheEnable: true
                 predicate.ProportionalEnable: true
                 predicate.resources: nvidia.com/gpu
                 predicate.resources.nvidia.com/gpu.cpu: 4
                 predicate.resources.nvidia.com/gpu.memory: 8
             - name: proportion
             - name: nodeorder
        */

        predicate := predicateEnable{
                gpuSharingEnable:   false,
                cacheEnable:        false,
                proportionalEnable: false,
        }

        // Checks whether predicate.GPUSharingEnable is provided or not, if given, modifies the value in predicateEnable struct.
        args.GetBool(&predicate.gpuSharingEnable, GPUSharingPredicate)
        args.GetBool(&predicate.cacheEnable, CachePredicate)
        // Checks whether predicate.ProportionalEnable is provided or not, if given, modifies the value in predicateEnable struct.
        args.GetBool(&predicate.proportionalEnable, ProportionalPredicate)
        resourcesProportional := make(map[v1.ResourceName]baseResource)
        resourcesStr := args[ProportionalResource]
        resources := strings.Split(resourcesStr, ",")
        for _, resource := range resources {
                resource = strings.TrimSpace(resource)
                if resource == "" {
                        continue
                }
                // proportional.resources.[ResourceName]
                cpuResourceKey := ProportionalResourcesPrefix + resource + ".cpu"
                cpuResourceRate := 1.0
                args.GetFloat64(&cpuResourceRate, cpuResourceKey)
                if cpuResourceRate < 0 {
                        cpuResourceRate = 1.0
                }
                memoryResourceKey := ProportionalResourcesPrefix + resource + ".memory"
                memoryResourceRate := 1.0
                args.GetFloat64(&memoryResourceRate, memoryResourceKey)
                if memoryResourceRate < 0 {
                        memoryResourceRate = 1.0
                }
                r := baseResource{
                        CPU:    cpuResourceRate,
                        Memory: memoryResourceRate,
                }
                resourcesProportional[v1.ResourceName(resource)] = r
        }
        predicate.proportional = resourcesProportional

        return predicate
}

func (pp *predicatesPlugin) OnSessionOpen(ssn *framework.Session) {
        pl := util.NewPodListerFromNode(ssn)
        nodeMap := util.GenerateNodeMapAndSlice(ssn.Nodes)

        pCache := predicateCacheNew()
        predicate := enablePredicate(pp.pluginArguments)

        kubeClient := ssn.KubeClient()
        // Register event handlers to update task info in PodLister & nodeMap
        ssn.AddEventHandler(&framework.EventHandler{
                AllocateFunc: func(event *framework.Event) {
                        pod := pl.UpdateTask(event.Task, event.Task.NodeName)

                        nodeName := event.Task.NodeName
                        node, found := nodeMap[nodeName]
                        if !found {
                                klog.Errorf("predicates, update pod %s/%s allocate to NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName)
                                return
                        }

                        if predicate.gpuSharingEnable && api.GetGPUResourceOfPod(pod) > 0 {
                                nodeInfo, ok := ssn.Nodes[nodeName]
                                if !ok {
                                        klog.Errorf("Failed to get node %s info from cache", nodeName)
                                        return
                                }

                                id := predicateGPU(pod, nodeInfo)
                                if id < 0 {
                                        klog.Errorf("The node %s can't place the pod %s in ns %s", pod.Spec.NodeName, pod.Name, pod.Namespace)
                                        return
                                }
                                patch := api.AddGPUIndexPatch(id)
                                pod, err := kubeClient.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
                                if err != nil {
                                        klog.Errorf("Patch pod %s failed with patch %s: %v", pod.Name, patch, err)
                                        return
                                }
                                dev, ok := nodeInfo.GPUDevices[id]
                                if !ok {
                                        klog.Errorf("Failed to get GPU %d from node %s", id, nodeName)
                                        return
                                }
                                dev.PodMap[string(pod.UID)] = pod
                                klog.V(4).Infof("predicates with gpu sharing, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName)
                        }

                        node.AddPod(pod)
                        klog.V(4).Infof("predicates, update pod %s/%s allocate to node [%s]", pod.Namespace, pod.Name, nodeName)
                },
                DeallocateFunc: func(event *framework.Event) {
                        pod := pl.UpdateTask(event.Task, "")
                        nodeName := event.Task.NodeName
                        node, found := nodeMap[nodeName]
                        if !found {
                                klog.Errorf("predicates, update pod %s/%s allocate from NOT EXIST node [%s]", pod.Namespace, pod.Name, nodeName)
                                return
                        }

                        if predicate.gpuSharingEnable && api.GetGPUResourceOfPod(pod) > 0 {
                                // deallocate pod gpu id
                                id := api.GetGPUIndex(pod)
                                patch := api.RemoveGPUIndexPatch()
                                _, err := kubeClient.CoreV1().Pods(pod.Namespace).Patch(context.TODO(), pod.Name, types.JSONPatchType, []byte(patch), metav1.PatchOptions{})
                                if err != nil {
                                        klog.Errorf("Patch pod %s failed with patch %s: %v", pod.Name, patch, err)
                                        return
                                }

                                nodeInfo, ok := ssn.Nodes[nodeName]
                                if !ok {
                                        klog.Errorf("Failed to get node %s info from cache", nodeName)
                                        return
                                }
                                if dev, ok := nodeInfo.GPUDevices[id]; ok {
                                        delete(dev.PodMap, string(pod.UID))
                                }

                                klog.V(4).Infof("predicates with gpu sharing, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName)
                        }

                        err := node.RemovePod(pod)
                        if err != nil {
                                klog.Errorf("predicates, remove pod %s/%s from node [%s] error: %v", pod.Namespace, pod.Name, nodeName, err)
                                return
                        }
                        klog.V(4).Infof("predicates, update pod %s/%s deallocate from node [%s]", pod.Namespace, pod.Name, nodeName)
                },
        })

        // Initialize k8s plugins
        // TODO: Add more predicates, k8s.io/kubernetes/pkg/scheduler/framework/plugins/legacy_registry.go
        handle := k8s.NewFrameworkHandle(nodeMap, ssn.KubeClient(), ssn.InformerFactory())
        // 1. NodeUnschedulable
        plugin, _ := nodeunschedulable.New(nil, handle)
        nodeUnscheduleFilter := plugin.(*nodeunschedulable.NodeUnschedulable)
        // 2. NodeAffinity
        plugin, _ = nodeaffinity.New(nil, handle)
        nodeAffinityFilter := plugin.(*nodeaffinity.NodeAffinity)
        // 3. NodePorts
        plugin, _ = nodeports.New(nil, handle)
        nodePortFilter := plugin.(*nodeports.NodePorts)
        // 4. TaintToleration
        plugin, _ = tainttoleration.New(nil, handle)
        tolerationFilter := plugin.(*tainttoleration.TaintToleration)
        // 5. InterPodAffinity
        plArgs := &config.InterPodAffinityArgs{}
        plugin, _ = interpodaffinity.New(plArgs, handle)
        podAffinityFilter := plugin.(*interpodaffinity.InterPodAffinity)

        ssn.AddPredicateFn(pp.Name(), func(task *api.TaskInfo, node *api.NodeInfo) error {
                nodeInfo, found := nodeMap[node.Name]
                if !found {
                        return fmt.Errorf("failed to predicates, node info for %s not found", node.Name)
                }

                if node.Allocatable.MaxTaskNum <= len(nodeInfo.Pods) {
                        klog.V(4).Infof("NodePodNumber predicates Task <%s/%s> on Node <%s> failed",
                                task.Namespace, task.Name, node.Name)
                        return api.NewFitError(task, node, api.NodePodNumberExceeded)
                }

                state := k8sframework.NewCycleState()
                predicateByStablefilter := func(pod *v1.Pod, nodeInfo *k8sframework.NodeInfo) (bool, error) {
                        // CheckNodeUnschedulable
                        status := nodeUnscheduleFilter.Filter(context.TODO(), state, task.Pod, nodeInfo)
                        if !status.IsSuccess() {
                                return false, fmt.Errorf("plugin %s predicates failed %s", nodeunschedulable.Name, status.Message())
                        }

                        // Check NodeAffinity
                        status = nodeAffinityFilter.Filter(context.TODO(), state, task.Pod, nodeInfo)
                        if !status.IsSuccess() {
                                return false, fmt.Errorf("plugin %s predicates failed %s", nodeaffinity.Name, status.Message())
                        }

                        // PodToleratesNodeTaints: TaintToleration
                        status = tolerationFilter.Filter(context.TODO(), state, task.Pod, nodeInfo)
                        if !status.IsSuccess() {
                                return false, fmt.Errorf("plugin %s predicates failed %s", tainttoleration.Name, status.Message())
                        }

                        return true, nil
                }

                // Check PredicateWithCache
                {
                        var err error
                        var fit bool
                        if predicate.cacheEnable {
                                fit, err = pCache.PredicateWithCache(node.Name, task.Pod)
                                if err != nil {
                                        fit, err = predicateByStablefilter(task.Pod, nodeInfo)
                                        pCache.UpdateCache(node.Name, task.Pod, fit)
                                } else {
                                        if !fit {
                                                err = fmt.Errorf("plugin equivalence cache predicates failed")
                                        }
                                }
                        } else {
                                fit, err = predicateByStablefilter(task.Pod, nodeInfo)
                        }

                        if !fit {
                                return err
                        }
                }

                // Check NodePorts
                nodePortFilter.PreFilter(context.TODO(), state, task.Pod)
                status := nodePortFilter.Filter(context.TODO(), state, nil, nodeInfo)
                if !status.IsSuccess() {
                        return fmt.Errorf("plugin %s predicates failed %s", nodeaffinity.Name, status.Message())
                }

                // InterPodAffinity Predicate
                status = podAffinityFilter.PreFilter(context.TODO(), state, task.Pod)
                if !status.IsSuccess() {
                        return fmt.Errorf("plugin %s pre-predicates failed %s", interpodaffinity.Name, status.Message())
                }

                status = podAffinityFilter.Filter(context.TODO(), state, task.Pod, nodeInfo)
                if !status.IsSuccess() {
                        return fmt.Errorf("plugin %s predicates failed %s", interpodaffinity.Name, status.Message())
                }

                if predicate.gpuSharingEnable {
                        // CheckGPUSharingPredicate
                        fit, err := checkNodeGPUSharingPredicate(task.Pod, node)
                        if err != nil {
                                return err
                        }

                        klog.V(4).Infof("checkNodeGPUSharingPredicate predicates Task <%s/%s> on Node <%s>: fit %v",
                                task.Namespace, task.Name, node.Name, fit)
                }
                if predicate.proportionalEnable {
                        // Check ProportionalPredicate
                        fit, err := checkNodeResourceIsProportional(task, node, predicate.proportional)
                        if err != nil {
                                return err
                        }
                        klog.V(4).Infof("checkNodeResourceIsProportional predicates Task <%s/%s> on Node <%s>: fit %v",
                                task.Namespace, task.Name, node.Name, fit)
                }
                return nil
        })
}

func (pp *predicatesPlugin) OnSessionClose(ssn *framework.Session) {}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package predicates

import (
        "fmt"

        v1 "k8s.io/api/core/v1"

        "volcano.sh/volcano/pkg/scheduler/api"
)

// checkNodeResourceIsProportional checks if a gpu:cpu:memory is Proportional
func checkNodeResourceIsProportional(task *api.TaskInfo, node *api.NodeInfo, proportional map[v1.ResourceName]baseResource) (bool, error) {
        for resourceName := range proportional {
                if value, found := task.Resreq.ScalarResources[resourceName]; found && value > 0 {
                        return true, nil
                }
        }
        for resourceName, resourceRate := range proportional {
                if value, found := node.Idle.ScalarResources[resourceName]; found {
                        cpuReserved := value * resourceRate.CPU
                        memoryReserved := value * resourceRate.Memory * 1000 * 1000
                        r := node.Idle.Clone()
                        r = r.Sub(task.Resreq)
                        if r.MilliCPU < cpuReserved || r.Memory < memoryReserved {
                                return false, fmt.Errorf("proportional of resource %s check failed", resourceName)
                        }
                }
        }
        return true, nil
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tasktopology

import (
        "k8s.io/apimachinery/pkg/types"
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
)

type reqAction int

const (
        reqSub reqAction = iota
        reqAdd
)

// Bucket is struct used to classify tasks by affinity and anti-affinity
type Bucket struct {
        index       int
        tasks       map[types.UID]*api.TaskInfo
        taskNameSet map[string]int

        // reqScore is score of resource
        // now, we regard 1 CPU and 1 GPU and 1Gi memory as the same score.
        reqScore float64
        request  *api.Resource

        boundTask int
        node      map[string]int
}

// NewBucket create a new empty bucket
func NewBucket() *Bucket {
        return &Bucket{
                index:       0,
                tasks:       make(map[types.UID]*api.TaskInfo),
                taskNameSet: make(map[string]int),

                reqScore: 0,
                request:  api.EmptyResource(),

                boundTask: 0,
                node:      make(map[string]int),
        }
}

// CalcResReq calculates task resources request
func (b *Bucket) CalcResReq(req *api.Resource, action reqAction) {
        if req == nil {
                return
        }

        cpu := req.MilliCPU
        // treat 1Mi the same as 1m cpu 1m gpu
        mem := req.Memory / 1024 / 1024
        score := cpu + mem
        for _, request := range req.ScalarResources {
                score += request
        }

        switch action {
        case reqSub:
                b.reqScore -= score
                b.request.Sub(req)
        case reqAdd:
                b.reqScore += score
                b.request.Add(req)
        default:
                klog.V(3).Infof("Invalid action <%v> for resource <%v>", action, req)
        }
}

// AddTask adds task into bucket
func (b *Bucket) AddTask(taskName string, task *api.TaskInfo) {
        b.taskNameSet[taskName]++
        if task.NodeName != "" {
                b.node[task.NodeName]++
                b.boundTask++
                return
        }

        b.tasks[task.Pod.UID] = task
        b.CalcResReq(task.Resreq, reqAdd)
}

// TaskBound binds task to bucket
func (b *Bucket) TaskBound(task *api.TaskInfo) {
        b.node[task.NodeName]++
        b.boundTask++

        delete(b.tasks, task.Pod.UID)
        b.CalcResReq(task.Resreq, reqSub)
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tasktopology

import (
        "fmt"
        "math"
        "sort"
        "strings"

        "k8s.io/apimachinery/pkg/types"
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/scheduler/api"
)

type topologyType int

const (
        selfAntiAffinity topologyType = iota
        interAntiAffinity
        selfAffinity
        interAffinity
)

// map[topologyType]priority, the larger number means the higher priority
var affinityPriority = map[topologyType]int{
        selfAntiAffinity:  4,
        interAffinity:     3,
        selfAffinity:      2,
        interAntiAffinity: 1,
}

// JobManager is struct used to save infos about affinity and buckets of a job
type JobManager struct {
        jobID api.JobID

        buckets     []*Bucket
        podInBucket map[types.UID]int
        podInTask   map[types.UID]string
        taskOverPod map[string]map[types.UID]struct{}

        taskAffinityPriority map[string]int // [taskName] -> priority
        taskExistOrder       map[string]int
        interAffinity        map[string]map[string]struct{} // [taskName]->[taskName]
        selfAffinity         map[string]struct{}
        interAntiAffinity    map[string]map[string]struct{} // [taskName]->[taskName]
        selfAntiAffinity     map[string]struct{}

        bucketMaxSize int
        nodeTaskSet   map[string]map[string]int // [nodeName]->[taskName]
}

// NewJobManager creates a new job manager for job
func NewJobManager(jobID api.JobID) *JobManager {
        return &JobManager{
                jobID: jobID,

                buckets:     make([]*Bucket, 0),
                podInBucket: make(map[types.UID]int),
                podInTask:   make(map[types.UID]string),
                taskOverPod: make(map[string]map[types.UID]struct{}),

                taskAffinityPriority: make(map[string]int),
                taskExistOrder:       make(map[string]int),
                interAffinity:        make(map[string]map[string]struct{}),
                interAntiAffinity:    make(map[string]map[string]struct{}),
                selfAffinity:         make(map[string]struct{}),
                selfAntiAffinity:     make(map[string]struct{}),

                bucketMaxSize: 0,
                nodeTaskSet:   make(map[string]map[string]int),
        }
}

// MarkOutOfBucket indicates task is outside of any bucket
func (jm *JobManager) MarkOutOfBucket(uid types.UID) {
        jm.podInBucket[uid] = OutOfBucket
}

// MarkTaskHasTopology indicates task has topology settings
func (jm *JobManager) MarkTaskHasTopology(taskName string, topoType topologyType) {
        priority := affinityPriority[topoType]
        if priority > jm.taskAffinityPriority[taskName] {
                jm.taskAffinityPriority[taskName] = priority
        }
}

// ApplyTaskTopology transforms taskTopology to matrix
// affinity: [[a, b], [c]]
// interAffinity:
//      a   b   c
//  a   -   x   -
//  b   x   -   -
//  c   -   -   -
//  selfAffinity:
//      a   b   c
//      -   -   x
func (jm *JobManager) ApplyTaskTopology(topo *TaskTopology) {
        for _, aff := range topo.Affinity {
                if len(aff) == 1 {
                        taskName := aff[0]
                        jm.selfAffinity[taskName] = struct{}{}
                        jm.MarkTaskHasTopology(taskName, selfAffinity)
                        continue
                }
                for index, src := range aff {
                        for _, dst := range aff[:index] {
                                addAffinity(jm.interAffinity, src, dst)
                                addAffinity(jm.interAffinity, dst, src)
                        }
                        jm.MarkTaskHasTopology(src, interAffinity)
                }
        }

        for _, aff := range topo.AntiAffinity {
                if len(aff) == 1 {
                        taskName := aff[0]
                        jm.selfAntiAffinity[taskName] = struct{}{}
                        jm.MarkTaskHasTopology(taskName, selfAntiAffinity)
                        continue
                }
                for index, src := range aff {
                        for _, dst := range aff[:index] {
                                addAffinity(jm.interAntiAffinity, src, dst)
                                addAffinity(jm.interAntiAffinity, dst, src)
                        }
                        jm.MarkTaskHasTopology(src, interAntiAffinity)
                }
        }

        length := len(topo.TaskOrder)
        for index, taskName := range topo.TaskOrder {
                jm.taskExistOrder[taskName] = length - index
        }
}

// NewBucket creates a new bucket
func (jm *JobManager) NewBucket() *Bucket {
        bucket := NewBucket()
        bucket.index = len(jm.buckets)
        jm.buckets = append(jm.buckets, bucket)
        return bucket
}

// AddTaskToBucket adds task into bucket
func (jm *JobManager) AddTaskToBucket(bucketIndex int, taskName string, task *api.TaskInfo) {
        bucket := jm.buckets[bucketIndex]
        jm.podInBucket[task.Pod.UID] = bucketIndex
        bucket.AddTask(taskName, task)
        if size := len(bucket.tasks) + bucket.boundTask; size > jm.bucketMaxSize {
                jm.bucketMaxSize = size
        }
}

// L compared with R, -1 for L < R, 0 for L == R, 1 for L > R
func (jm *JobManager) taskAffinityOrder(L, R *api.TaskInfo) int {
        LTaskName := jm.podInTask[L.Pod.UID]
        RTaskName := jm.podInTask[R.Pod.UID]

        // in the same vk task, they are equal
        if LTaskName == RTaskName {
                return 0
        }

        // use user defined order firstly
        LOrder := jm.taskExistOrder[LTaskName]
        ROrder := jm.taskExistOrder[RTaskName]
        if LOrder != ROrder {
                if LOrder > ROrder {
                        return 1
                }
                return -1
        }

        LPriority := jm.taskAffinityPriority[LTaskName]
        RPriority := jm.taskAffinityPriority[RTaskName]
        if LPriority != RPriority {
                if LPriority > RPriority {
                        return 1
                }
                return -1
        }

        // all affinity setting of L and R are the same, they are equal
        return 0
}

func (jm *JobManager) buildTaskInfo(tasks map[api.TaskID]*api.TaskInfo) []*api.TaskInfo {
        taskWithoutBucket := make([]*api.TaskInfo, 0, len(tasks))
        for _, task := range tasks {
                pod := task.Pod

                taskName := getTaskName(task)
                if taskName == "" {
                        jm.MarkOutOfBucket(pod.UID)
                        continue
                }
                if _, hasTopology := jm.taskAffinityPriority[taskName]; !hasTopology {
                        jm.MarkOutOfBucket(pod.UID)
                        continue
                }

                jm.podInTask[pod.UID] = taskName
                taskSet, ok := jm.taskOverPod[taskName]
                if !ok {
                        taskSet = make(map[types.UID]struct{})
                        jm.taskOverPod[taskName] = taskSet
                }
                taskSet[pod.UID] = struct{}{}
                taskWithoutBucket = append(taskWithoutBucket, task)
        }
        return taskWithoutBucket
}

func (jm *JobManager) checkTaskSetAffinity(taskName string, taskNameSet map[string]int, onlyAnti bool) int {
        bucketPodAff := 0

        if taskName == "" {
                return bucketPodAff
        }

        for taskNameInBucket, count := range taskNameSet {
                theSameTask := taskNameInBucket == taskName

                if !onlyAnti {
                        affinity := false
                        if theSameTask {
                                _, affinity = jm.selfAffinity[taskName]
                        } else {
                                _, affinity = jm.interAffinity[taskName][taskNameInBucket]
                        }
                        if affinity {
                                bucketPodAff += count
                        }
                }

                antiAffinity := false
                if theSameTask {
                        _, antiAffinity = jm.selfAntiAffinity[taskName]
                } else {
                        _, antiAffinity = jm.interAntiAffinity[taskName][taskNameInBucket]
                }
                if antiAffinity {
                        bucketPodAff -= count
                }
        }

        return bucketPodAff
}

func (jm *JobManager) buildBucket(taskWithOrder []*api.TaskInfo) {
        nodeBucketMapping := make(map[string]*Bucket)

        for _, task := range taskWithOrder {
                klog.V(5).Infof("jobID %s task with order task %s/%s", jm.jobID, task.Namespace, task.Name)

                var selectedBucket *Bucket
                maxAffinity := math.MinInt32

                taskName := getTaskName(task)

                if task.NodeName != "" {
                        // generate bucket by node
                        maxAffinity = 0
                        selectedBucket = nodeBucketMapping[task.NodeName]
                } else {
                        for _, bucket := range jm.buckets {
                                bucketPodAff := jm.checkTaskSetAffinity(taskName, bucket.taskNameSet, false)

                                // choose the best fit affinity, or balance resource between bucket
                                if bucketPodAff > maxAffinity {
                                        maxAffinity = bucketPodAff
                                        selectedBucket = bucket
                                } else if bucketPodAff == maxAffinity && selectedBucket != nil &&
                                        bucket.reqScore < selectedBucket.reqScore {
                                        selectedBucket = bucket
                                }
                        }
                }

                if maxAffinity < 0 || selectedBucket == nil {
                        selectedBucket = jm.NewBucket()
                        if task.NodeName != "" {
                                nodeBucketMapping[task.NodeName] = selectedBucket
                        }
                }

                jm.AddTaskToBucket(selectedBucket.index, taskName, task)
        }
}

// ConstructBucket builds bucket for tasks
func (jm *JobManager) ConstructBucket(tasks map[api.TaskID]*api.TaskInfo) {
        taskWithoutBucket := jm.buildTaskInfo(tasks)

        o := TaskOrder{
                tasks: taskWithoutBucket,

                manager: jm,
        }
        sort.Sort(sort.Reverse(&o))

        jm.buildBucket(o.tasks)
}

// TaskBound binds task to bucket
func (jm *JobManager) TaskBound(task *api.TaskInfo) {
        if taskName := getTaskName(task); taskName != "" {
                set, ok := jm.nodeTaskSet[task.NodeName]
                if !ok {
                        set = make(map[string]int)
                        jm.nodeTaskSet[task.NodeName] = set
                }
                set[taskName]++
        }

        bucket := jm.GetBucket(task)
        if bucket != nil {
                bucket.TaskBound(task)
        }
}

// GetBucket get bucket inside which task has been
func (jm *JobManager) GetBucket(task *api.TaskInfo) *Bucket {
        index, ok := jm.podInBucket[task.Pod.UID]
        if !ok || index == OutOfBucket {
                return nil
        }

        bucket := jm.buckets[index]
        return bucket
}

func (jm *JobManager) String() string {
        // saa: selfAntiAffinity
        // iaa: interAntiAffinity
        // sa: selfAffinity
        // ia: interAffinity
        msg := []string{
                fmt.Sprintf("%s - job %s max %d || saa: %v - iaa: %v - sa: %v - ia: %v || priority: %v - order: %v || ",
                        PluginName, jm.jobID, jm.bucketMaxSize,
                        jm.selfAntiAffinity, jm.interAntiAffinity,
                        jm.selfAffinity, jm.interAffinity,
                        jm.taskAffinityPriority, jm.taskExistOrder,
                ),
        }

        for _, bucket := range jm.buckets {
                bucketMsg := fmt.Sprintf("b:%d -- ", bucket.index)
                var info []string
                for _, task := range bucket.tasks {
                        info = append(info, task.Pod.Name)
                }
                bucketMsg += strings.Join(info, ", ")
                bucketMsg += "|"

                info = nil
                for nodeName, count := range bucket.node {
                        info = append(info, fmt.Sprintf("n%s-%d", nodeName, count))
                }
                bucketMsg += strings.Join(info, ", ")

                msg = append(msg, "["+bucketMsg+"]")
        }
        return strings.Join(msg, " ")
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tasktopology

import (
        "fmt"
        "strings"
        "time"

        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
)

type taskTopologyPlugin struct {
        arguments framework.Arguments

        weight   int
        managers map[api.JobID]*JobManager
}

// New function returns taskTopologyPlugin object
func New(arguments framework.Arguments) framework.Plugin {
        return &taskTopologyPlugin{
                arguments: arguments,

                weight:   calculateWeight(arguments),
                managers: make(map[api.JobID]*JobManager),
        }
}

func (p *taskTopologyPlugin) Name() string {
        return PluginName
}

// TaskOrderFn returns -1 to make l prior to r.
//
// for example:
// A:
//  | bucket1   | bucket2   | out of bucket
//  | a1 a3     | a2        | a4
// B:
//  | bucket1   | out of bucket
//  | b1 b2     | b3
// the right task order should be:
//   a1 a3 a2 b1 b2 a4 b3
func (p *taskTopologyPlugin) TaskOrderFn(l interface{}, r interface{}) int {
        lv, ok := l.(*api.TaskInfo)
        if !ok {
                klog.Errorf("Object is not a taskinfo")
        }
        rv, ok := r.(*api.TaskInfo)
        if !ok {
                klog.Errorf("Object is not a taskinfo")
        }

        lvJobManager := p.managers[lv.Job]
        rvJobManager := p.managers[rv.Job]

        var lvBucket, rvBucket *Bucket
        if lvJobManager != nil {
                lvBucket = lvJobManager.GetBucket(lv)
        } else {
                klog.V(4).Infof("No job manager for job <ID: %s>, do not return task order.", lv.Job)
                return 0
        }
        if rvJobManager != nil {
                rvBucket = rvJobManager.GetBucket(rv)
        } else {
                klog.V(4).Infof("No job manager for job <ID: %s>, do not return task order.", rv.Job)
                return 0
        }

        // the one have bucket would always prior to another
        lvInBucket := lvBucket != nil
        rvInBucket := rvBucket != nil
        if lvInBucket != rvInBucket {
                if lvInBucket {
                        return -1
                }
                return 1
        }

        // comparison between job is not the duty of this plugin
        if lv.Job != rv.Job {
                return 0
        }

        // task out of bucket have no order
        if !lvInBucket && !rvInBucket {
                return 0
        }

        // the big bucket should prior to small one
        lvHasTask := len(lvBucket.tasks)
        rvHasTask := len(rvBucket.tasks)
        if lvHasTask != rvHasTask {
                if lvHasTask > rvHasTask {
                        return -1
                }
                return 1
        }

        lvBucketIndex := lvBucket.index
        rvBucketIndex := rvBucket.index
        // in the same bucket, the affinityOrder is ok
        if lvBucketIndex == rvBucketIndex {
                affinityOrder := lvJobManager.taskAffinityOrder(lv, rv)
                return -affinityOrder
        }

        // the old bucket should prior to young one
        if lvBucketIndex < rvBucketIndex {
                return -1
        }
        return 1
}

func (p *taskTopologyPlugin) calcBucketScore(task *api.TaskInfo, node *api.NodeInfo) (int, *JobManager, error) {
        // task could never fits the node
        maxResource := node.Idle.Clone().Add(node.Releasing)
        if req := task.Resreq; req != nil && maxResource.LessPartly(req, api.Zero) {
                return 0, nil, nil
        }

        jobManager, hasManager := p.managers[task.Job]
        if !hasManager {
                return 0, nil, nil
        }

        bucket := jobManager.GetBucket(task)
        // task out of bucket
        if bucket == nil {
                return 0, jobManager, nil
        }

        // 1. bound task in bucket is the base score of this node
        score := bucket.node[node.Name]

        // 2. task inter/self anti-affinity should be calculated
        if nodeTaskSet := jobManager.nodeTaskSet[node.Name]; nodeTaskSet != nil {
                taskName := getTaskName(task)
                affinityScore := jobManager.checkTaskSetAffinity(taskName, nodeTaskSet, true)
                if affinityScore < 0 {
                        score += affinityScore
                }
        }
        klog.V(4).Infof("task %s/%s, node %s, additional score %d, task %d",
                task.Namespace, task.Name, node.Name, score, len(bucket.tasks))

        // 3. the other tasks in bucket take into considering
        score += len(bucket.tasks)
        if bucket.request == nil || bucket.request.LessEqual(maxResource, api.Zero) {
                return score, jobManager, nil
        }

        remains := bucket.request.Clone()
        // randomly (by map) take out task to make the bucket fits the node
        for bucketTaskID, bucketTask := range bucket.tasks {
                // current task should kept in bucket
                if bucketTaskID == task.Pod.UID || bucketTask.Resreq == nil {
                        continue
                }
                remains.Sub(bucketTask.Resreq)
                score--
                if remains.LessEqual(maxResource, api.Zero) {
                        break
                }
        }
        // here, the bucket remained request will always fit the maxResource
        return score, jobManager, nil
}

func (p *taskTopologyPlugin) NodeOrderFn(task *api.TaskInfo, node *api.NodeInfo) (float64, error) {
        score, jobManager, err := p.calcBucketScore(task, node)
        if err != nil {
                return 0, err
        }
        fScore := float64(score * p.weight)
        if jobManager != nil && jobManager.bucketMaxSize != 0 {
                fScore = fScore * float64(v1alpha1.MaxNodeScore) / float64(jobManager.bucketMaxSize)
        }
        klog.V(4).Infof("task %s/%s at node %s has bucket score %d, score %f",
                task.Namespace, task.Name, node.Name, score, fScore)
        return fScore, nil
}

func (p *taskTopologyPlugin) AllocateFunc(event *framework.Event) {
        task := event.Task

        jobManager, hasManager := p.managers[task.Job]
        if !hasManager {
                return
        }
        jobManager.TaskBound(task)
}

func (p *taskTopologyPlugin) initBucket(ssn *framework.Session) {
        for jobID, job := range ssn.Jobs {
                if noPendingTasks(job) {
                        klog.V(4).Infof("No pending tasks in job <%s/%s> by plugin %s.",
                                job.Namespace, job.Name, PluginName)
                        continue
                }

                jobTopology, err := readTopologyFromPgAnnotations(job)
                if err != nil {
                        klog.V(4).Infof("Failed to read task topology from job <%s/%s> annotations, error: %s.",
                                job.Namespace, job.Name, err.Error())
                        continue
                }
                if jobTopology == nil {
                        continue
                }

                manager := NewJobManager(jobID)
                manager.ApplyTaskTopology(jobTopology)
                manager.ConstructBucket(job.Tasks)

                p.managers[job.UID] = manager
        }
}

func affinityCheck(job *api.JobInfo, affinity [][]string) error {
        if job == nil || affinity == nil {
                return fmt.Errorf("empty input, job: %v, affinity: %v", job, affinity)
        }

        var taskNumber = len(job.Tasks)
        var taskRef = make(map[string]bool, taskNumber)
        for _, task := range job.Tasks {
                tmpStrings := strings.Split(task.Name, "-")
                if _, exist := taskRef[tmpStrings[len(tmpStrings)-2]]; !exist {
                        taskRef[tmpStrings[len(tmpStrings)-2]] = true
                }
        }

        for _, aff := range affinity {
                affTasks := make(map[string]bool, len(aff))
                for _, task := range aff {
                        if len(task) == 0 {
                                continue
                        }
                        if _, exist := taskRef[task]; !exist {
                                return fmt.Errorf("task %s do not exist in job <%s/%s>", task, job.Namespace, job.Name)
                        }
                        if _, exist := affTasks[task]; exist {
                                return fmt.Errorf("task %s is duplicated in job <%s/%s>", task, job.Namespace, job.Name)
                        }
                        affTasks[task] = true
                }
        }

        return nil
}

func splitAnnotations(job *api.JobInfo, annotation string) ([][]string, error) {
        affinityStr := strings.Split(annotation, ";")
        if len(affinityStr) == 0 {
                return nil, nil
        }
        var affinity = make([][]string, len(affinityStr))
        for i, str := range affinityStr {
                affinity[i] = strings.Split(str, ",")
        }
        if err := affinityCheck(job, affinity); err != nil {
                klog.V(4).Infof("Job <%s/%s> affinity key invalid: %s.",
                        job.Namespace, job.Name, err.Error())
                return nil, err
        }
        return affinity, nil
}

func readTopologyFromPgAnnotations(job *api.JobInfo) (*TaskTopology, error) {
        jobAffinityStr, affinityExist := job.PodGroup.Annotations[JobAffinityAnnotations]
        jobAntiAffinityStr, antiAffinityExist := job.PodGroup.Annotations[JobAntiAffinityAnnotations]
        taskOrderStr, taskOrderExist := job.PodGroup.Annotations[TaskOrderAnnotations]

        if !(affinityExist || antiAffinityExist || taskOrderExist) {
                return nil, nil
        }

        var jobTopology = TaskTopology{
                Affinity:     nil,
                AntiAffinity: nil,
                TaskOrder:    nil,
        }

        if affinityExist {
                affinities, err := splitAnnotations(job, jobAffinityStr)
                if err != nil {
                        klog.V(4).Infof("Job <%s/%s> affinity key invalid: %s.",
                                job.Namespace, job.Name, err.Error())
                        return nil, err
                }
                jobTopology.Affinity = affinities
        }

        if antiAffinityExist {
                affinities, err := splitAnnotations(job, jobAntiAffinityStr)
                if err != nil {
                        klog.V(4).Infof("Job <%s/%s> anti affinity key invalid: %s.",
                                job.Namespace, job.Name, err.Error())
                        return nil, err
                }
                jobTopology.AntiAffinity = affinities
        }

        if taskOrderExist {
                jobTopology.TaskOrder = strings.Split(taskOrderStr, ",")
                if err := affinityCheck(job, [][]string{jobTopology.TaskOrder}); err != nil {
                        klog.V(4).Infof("Job <%s/%s> task order key invalid: %s.",
                                job.Namespace, job.Name, err.Error())
                        return nil, err
                }
        }

        return &jobTopology, nil
}

func (p *taskTopologyPlugin) OnSessionOpen(ssn *framework.Session) {
        start := time.Now()
        klog.V(3).Infof("start to init task topology plugin, weight[%d], defined order %v", p.weight, affinityPriority)

        p.initBucket(ssn)

        ssn.AddTaskOrderFn(p.Name(), p.TaskOrderFn)

        ssn.AddNodeOrderFn(p.Name(), p.NodeOrderFn)

        ssn.AddEventHandler(&framework.EventHandler{
                AllocateFunc: p.AllocateFunc,
        })

        klog.V(3).Infof("finished to init task topology plugin, using time %v", time.Since(start))
}

func (p *taskTopologyPlugin) OnSessionClose(ssn *framework.Session) {
        p.managers = nil
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tasktopology

import (
        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
)

const (
        // PluginName indicates name of volcano scheduler plugin
        PluginName = "task-topology"
        // PluginWeight is task-topology plugin weight in nodeOrderFn
        PluginWeight = "task-topology.weight"
        // JobAffinityKey is the key to read in task-topology arguments from job annotations
        JobAffinityKey = "volcano.sh/task-topology"
        // OutOfBucket indicates task is outside of any bucket
        OutOfBucket = -1

        // JobAffinityAnnotations is the key to read in task-topology affinity arguments from podgroup annotations
        JobAffinityAnnotations = "volcano.sh/task-topology-affinity"
        // JobAntiAffinityAnnotations is the key to read in task-topology anti-affinity arguments from podgroup annotations
        JobAntiAffinityAnnotations = "volcano.sh/task-topology-anti-affinity"
        // TaskOrderAnnotations is the key to read in task-topology task order arguments from podgroup annotations
        TaskOrderAnnotations = "volcano.sh/task-topology-task-order"
)

// TaskTopology is struct used to save affinity infos of a job read from job plugin or annotations
type TaskTopology struct {
        Affinity     [][]string `json:"affinity,omitempty"`
        AntiAffinity [][]string `json:"antiAffinity,omitempty"`
        TaskOrder    []string   `json:"taskOrder,omitempty"`
}

func calculateWeight(args framework.Arguments) int {
        /*
           User Should give taskTopologyWeight in this format(task-topology.weight).

           actions: "enqueue, reclaim, allocate, backfill, preempt"
           tiers:
           - plugins:
             - name: task-topology
               arguments:
                 task-topology.weight: 10
        */
        // Values are initialized to 1.
        weight := 1

        args.GetInt(&weight, PluginWeight)

        return weight
}

func getTaskName(task *api.TaskInfo) string {
        return task.Pod.Annotations[v1alpha1.TaskSpecKey]
}

func addAffinity(m map[string]map[string]struct{}, src, dst string) {
        srcMap, ok := m[src]
        if !ok {
                srcMap = make(map[string]struct{})
                m[src] = srcMap
        }
        srcMap[dst] = struct{}{}
}

func noPendingTasks(job *api.JobInfo) bool {
        return len(job.TaskStatusIndex[api.Pending]) == 0
}

// TaskOrder is struct used to save task order
type TaskOrder struct {
        tasks   []*api.TaskInfo
        manager *JobManager
}

func (p *TaskOrder) Len() int { return len(p.tasks) }

func (p *TaskOrder) Swap(l, r int) {
        p.tasks[l], p.tasks[r] = p.tasks[r], p.tasks[l]
}

func (p *TaskOrder) Less(l, r int) bool {
        L := p.tasks[l]
        R := p.tasks[r]

        LHasNode := L.NodeName != ""
        RHasNode := R.NodeName != ""
        if LHasNode || RHasNode {
                // the task bounded would have high priority
                if LHasNode != RHasNode {
                        return !LHasNode
                }
                // all bound, any order is alright
                return L.NodeName > R.NodeName
        }

        result := p.manager.taskAffinityOrder(L, R)
        // they have the same taskAffinity order, any order is alright
        if result == 0 {
                return L.Name > R.Name
        }
        return result < 0
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tdm

import (
        "fmt"
        "strings"
        "time"

        "k8s.io/apimachinery/pkg/util/intstr"
        "k8s.io/klog"
        "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/volcano/pkg/scheduler/api"
        "volcano.sh/volcano/pkg/scheduler/framework"
        tutil "volcano.sh/volcano/pkg/scheduler/plugins/util"
        "volcano.sh/volcano/pkg/scheduler/util"
)

const (
        // PluginName indicates name of volcano scheduler plugin.
        PluginName = "tdm"
        // revocableZoneLayout revocable zone layout
        revocableZoneLayout      = "15:04"
        revocableZoneLabelPrefix = "tdm.revocable-zone."
        evictPeriodLabel         = "tdm.evict.period"
        defaultPodEvictNum       = 1
)

var lastEvictAt time.Time

/*
   actions: "enqueue, reclaim, allocate, preempt"
   tiers:
   - plugins:
     - name: tdm
       arguments:
         tdm.revocable-zone.rz1: 10:00-21:00
         tdm.revocable-zone.rz2: 12:00-14:00
         tdm.evict.period: 1m
*/

type tdmPlugin struct {
        revocableZone map[string]string
        // evictPeriod
        // default 1m
        evictPeriod time.Duration
}

// New function returns prioritizePlugin object
func New(args framework.Arguments) framework.Plugin {
        revocableZone := make(map[string]string)
        evictPeriod := time.Minute

        for k, v := range args {
                if strings.Contains(k, revocableZoneLabelPrefix) {
                        revocableZone[strings.Replace(k, revocableZoneLabelPrefix, "", 1)] = v
                }
        }

        if period, ok := args[evictPeriodLabel]; ok {
                if d, err := time.ParseDuration(period); err == nil {
                        evictPeriod = d
                }
        }

        return &tdmPlugin{revocableZone, evictPeriod}
}

func (tp *tdmPlugin) Name() string {
        return PluginName
}

func parseRevocableZone(rzRaw string) (start, end time.Time, err error) {
        rzValues := strings.Split(strings.TrimSpace(rzRaw), "-")

        if len(rzValues) != 2 {
                err = fmt.Errorf("revocable zone %v format error", rzRaw)
                return
        }

        t1, err := time.Parse(revocableZoneLayout, rzValues[0])
        if err != nil {
                return
        }

        t2, err := time.Parse(revocableZoneLayout, rzValues[1])
        if err != nil {
                return
        }

        now := time.Now()

        start = time.Date(now.Year(), now.Month(), now.Day(), t1.Hour(), t1.Minute(), 0, 0, now.Location())
        if t1.After(t2) || t1.Equal(t2) {
                end = time.Date(now.Year(), now.Month(), now.Day()+1, t2.Hour(), t2.Minute(), 0, 0, now.Location())
        } else {
                end = time.Date(now.Year(), now.Month(), now.Day(), t2.Hour(), t2.Minute(), 0, 0, now.Location())
        }

        return
}

func (tp *tdmPlugin) availableRevocableZone(rz string) error {
        // rzRaw format 00:00-23:59
        rzRaw, ok := tp.revocableZone[rz]
        if !ok {
                return fmt.Errorf("revocable zone %v not support", rz)
        }

        now := time.Now()

        start, end, err := parseRevocableZone(rzRaw)
        if err != nil {
                return err
        }

        if now.Unix() < start.Unix() || now.Unix() > end.Unix() {
                return fmt.Errorf("current time beyond revocable zone %v:%v", rz, rzRaw)
        }

        return nil
}

func (tp *tdmPlugin) OnSessionOpen(ssn *framework.Session) {
        klog.V(4).Infof("Enter tdm plugin ...")
        if klog.V(4) {
                defer func() {
                        klog.V(4).Infof("Leaving tdm plugin.")
                }()
        }

        // tdm plugin just handle revocable node
        predicateFn := func(task *api.TaskInfo, node *api.NodeInfo) error {
                if node.RevocableZone == "" {
                        return nil
                }

                if err := tp.availableRevocableZone(node.RevocableZone); err != nil {
                        return fmt.Errorf("plugin %s predicates %w", tp.Name(), err)
                }

                klog.V(4).Infof("TDM node %v revocable zone %v:%v is active", node.Name, node.RevocableZone, tp.revocableZone[node.RevocableZone])

                if len(task.RevocableZone) == 0 {
                        msg := fmt.Sprintf("task %s/%s is not allow to dispatch to revocable node %s", task.Namespace, task.Name, node.Name)
                        return fmt.Errorf("plugin %s predicates %s", tp.Name(), msg)
                }

                klog.V(4).Infof("TDM filter for Task %s/%s on node %s pass.", task.Namespace, task.Name, node.Name)
                return nil
        }

        // tdm plugin just handle revocable node
        nodeOrderFn := func(task *api.TaskInfo, node *api.NodeInfo) (float64, error) {
                score := 0.0

                if node.RevocableZone == "" {
                        return score, nil
                }

                if err := tp.availableRevocableZone(node.RevocableZone); err != nil {
                        klog.V(4).Infof("TDM not available %s", err)
                        return score, err
                }

                if len(task.RevocableZone) == 0 {
                        klog.V(4).Infof("TDM task %s/%s is not allow to dispatch to revocable node %s", task.Namespace, task.Name, node.Name)
                        return score, nil
                }

                score = float64(v1alpha1.MaxNodeScore)

                klog.V(4).Infof("TDM score for Task %s/%s on node %s is: %v", task.Namespace, task.Name, node.Name, score)
                return score, nil
        }

        preemptableFn := func(preemptor *api.TaskInfo, preemptees []*api.TaskInfo) ([]*api.TaskInfo, int) {
                // for the preemptable or can use revocablezone workload, they can not preempt other tasks.
                if preemptor.Preemptable || len(preemptor.RevocableZone) > 0 {
                        klog.V(4).Infof("TDM task %s/%s is preemptable, do nothing skip", preemptor.Namespace, preemptor.Name)
                        return nil, tutil.Reject
                }

                var victims []*api.TaskInfo
                tasksMap := make(map[api.JobID][]*api.TaskInfo)

                // find preemptable tasks which appear on none revocable node
                for _, task := range preemptees {
                        if !task.Preemptable || task.Status != api.Running {
                                continue
                        }

                        node, ok := ssn.Nodes[task.NodeName]
                        if !ok {
                                continue
                        }

                        if node.RevocableZone != "" {
                                continue
                        }

                        tasksMap[task.Job] = append(tasksMap[task.Job], task)
                }

                for jobID, preemptableTasks := range tasksMap {
                        if job, ok := ssn.Jobs[jobID]; ok {
                                victims = append(victims, tp.maxVictims(job, preemptableTasks)...)
                        }
                }

                klog.V(4).Infof("TDM victims are %+v", victims)

                return victims, tutil.Permit
        }

        victimsFn := func() []*api.TaskInfo {
                if lastEvictAt.Add(tp.evictPeriod).After(time.Now()) {
                        klog.V(4).Infof("TDM next evict time at %v", lastEvictAt)
                        return nil
                }

                klog.V(4).Infof("TDM start to find victims")

                // find preemptable task on timeout revocable zone node
                victims := make([]*api.TaskInfo, 0)
                for rz := range tp.revocableZone {
                        if err := tp.availableRevocableZone(rz); err != nil {
                                klog.V(4).Infof("TDM revocable zone %v disactive, %v", rz, err)
                                // rz disactive, then evict preemptable tasks by job from the revocable node
                                for jobID, preemtableTasks := range tp.revocableNodePreemptableTask(rz, ssn) {
                                        if job, ok := ssn.Jobs[jobID]; ok {
                                                victims = append(victims, tp.maxVictims(job, preemtableTasks)...)
                                        }
                                }
                        }
                }

                // need to consider concurrency?
                lastEvictAt = time.Now()

                klog.V(4).Infof("TDM got %v victims", len(victims))

                return victims
        }

        jobOrderFn := func(l, r interface{}) int {
                lv := l.(*api.JobInfo)
                rv := r.(*api.JobInfo)

                if lv.Preemptable == rv.Preemptable {
                        return 0
                }

                if !lv.Preemptable {
                        return -1
                }

                return 1
        }

        jobPipelinedFn := func(obj interface{}) int {
                jobInfo := obj.(*api.JobInfo)
                occupied := jobInfo.WaitingTaskNum() + jobInfo.ReadyTaskNum()
                if occupied >= jobInfo.MinAvailable {
                        return tutil.Permit
                }
                return tutil.Reject
        }

        jobStarvingFn := func(obj interface{}) bool {
                jobInfo := obj.(*api.JobInfo)
                // allow none preemptable elastic job (deployment) preempt task
                if jobInfo.Preemptable {
                        return false
                }
                return len(jobInfo.TaskStatusIndex[api.Pending]) > 0
        }

        ssn.AddPredicateFn(tp.Name(), predicateFn)
        ssn.AddNodeOrderFn(tp.Name(), nodeOrderFn)
        ssn.AddPreemptableFn(tp.Name(), preemptableFn)
        ssn.AddVictimTasksFns(tp.Name(), victimsFn)
        ssn.AddJobOrderFn(tp.Name(), jobOrderFn)
        ssn.AddJobPipelinedFn(tp.Name(), jobPipelinedFn)
        ssn.AddJobStarvingFns(tp.Name(), jobStarvingFn)
}

func (tp *tdmPlugin) maxVictims(job *api.JobInfo, victims []*api.TaskInfo) []*api.TaskInfo {
        maxPodEvictNum := tp.getMaxPodEvictNum(job)
        targetNum := util.GetMinInt(maxPodEvictNum, len(victims))
        klog.V(3).Infof("Job <%s/%s> max evict:%v, potential victims number:%v, max victims number:%v",
                job.Namespace, job.Name, maxPodEvictNum, len(victims), targetNum)

        return victims[:targetNum]
}

// get max pod evict number from job budget configure
func (tp *tdmPlugin) getMaxPodEvictNum(job *api.JobInfo) int {
        jobRunningTaskNum := len(job.TaskStatusIndex[api.Running])
        if job.Budget.MaxUnavilable != "" {
                maxUnavilable := tp.parseIntStr(job.Budget.MaxUnavilable, len(job.Tasks))
                finalTaskNum := len(job.TaskStatusIndex[api.Succeeded]) + len(job.TaskStatusIndex[api.Failed])
                realUnavilable := len(job.Tasks) - finalTaskNum - jobRunningTaskNum
                if realUnavilable >= maxUnavilable {
                        return 0
                }
                return maxUnavilable - realUnavilable
        }

        if job.Budget.MinAvailable != "" {
                minAvailable := tp.parseIntStr(job.Budget.MinAvailable, len(job.Tasks))
                if jobRunningTaskNum >= minAvailable {
                        return jobRunningTaskNum - minAvailable
                }
        }

        return defaultPodEvictNum
}

func (tp *tdmPlugin) parseIntStr(input string, taskNum int) int {
        resultValue := 0
        tmp := intstr.Parse(input)
        switch tmp.Type {
        case intstr.Int:
                resultValue = tmp.IntValue()
        case intstr.String:
                if v, err := intstr.GetValueFromIntOrPercent(&tmp, taskNum, true); err == nil {
                        resultValue = v
                } else {
                        klog.Warningf("TDM get percent value err: %v", err)
                }
        }

        return resultValue
}

func (tp *tdmPlugin) revocableNodePreemptableTask(rz string, ssn *framework.Session) map[api.JobID][]*api.TaskInfo {
        tasksMap := make(map[api.JobID][]*api.TaskInfo)
        for _, node := range ssn.RevocableNodes {
                if node.RevocableZone != rz {
                        continue
                }

                for _, task := range node.Tasks {
                        if task.Preemptable {
                                if task.Status == api.Running {
                                        tasksMap[task.Job] = append(tasksMap[task.Job], task)
                                }
                        }
                }
        }

        return tasksMap
}

func (tp *tdmPlugin) OnSessionClose(ssn *framework.Session) {}

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
        "fmt"
        "path/filepath"
        "sync"
        "time"

        "github.com/fsnotify/fsnotify"
        "k8s.io/apimachinery/pkg/util/wait"
        "k8s.io/client-go/rest"
        "k8s.io/klog"

        "volcano.sh/volcano/pkg/filewatcher"
        schedcache "volcano.sh/volcano/pkg/scheduler/cache"
        "volcano.sh/volcano/pkg/scheduler/conf"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/metrics"
)

// Scheduler watches for new unscheduled pods for volcano. It attempts to find
// nodes that they fit on and writes bindings back to the api server.
type Scheduler struct {
        cache          schedcache.Cache
        schedulerConf  string
        fileWatcher    filewatcher.FileWatcher
        schedulePeriod time.Duration
        once           sync.Once

        mutex          sync.Mutex
        actions        []framework.Action
        plugins        []conf.Tier
        configurations []conf.Configuration
}

// NewScheduler returns a scheduler
func NewScheduler(
        config *rest.Config,
        schedulerName string,
        schedulerConf string,
        period time.Duration,
        defaultQueue string,
) (*Scheduler, error) {
        var watcher filewatcher.FileWatcher
        if schedulerConf != "" {
                var err error
                path := filepath.Dir(schedulerConf)
                watcher, err = filewatcher.NewFileWatcher(path)
                if err != nil {
                        return nil, fmt.Errorf("failed creating filewatcher for %s: %v", schedulerConf, err)
                }
        }

        scheduler := &Scheduler{
                schedulerConf:  schedulerConf,
                fileWatcher:    watcher,
                cache:          schedcache.New(config, schedulerName, defaultQueue),
                schedulePeriod: period,
        }

        return scheduler, nil
}

// Run runs the Scheduler
func (pc *Scheduler) Run(stopCh <-chan struct{}) {
        pc.loadSchedulerConf()
        go pc.watchSchedulerConf(stopCh)
        // Start cache for policy.
        go pc.cache.Run(stopCh)
        pc.cache.WaitForCacheSync(stopCh)
        go wait.Until(pc.runOnce, pc.schedulePeriod, stopCh)
}

func (pc *Scheduler) runOnce() {
        klog.V(4).Infof("Start scheduling ...")
        scheduleStartTime := time.Now()
        defer klog.V(4).Infof("End scheduling ...")

        pc.mutex.Lock()
        actions := pc.actions
        plugins := pc.plugins
        configurations := pc.configurations
        pc.mutex.Unlock()

        ssn := framework.OpenSession(pc.cache, plugins, configurations)
        defer framework.CloseSession(ssn)

        for _, action := range actions {
                actionStartTime := time.Now()
                action.Execute(ssn)
                metrics.UpdateActionDuration(action.Name(), metrics.Duration(actionStartTime))
        }
        metrics.UpdateE2eDuration(metrics.Duration(scheduleStartTime))
}

func (pc *Scheduler) loadSchedulerConf() {
        var err error
        pc.once.Do(func() {
                pc.actions, pc.plugins, pc.configurations, err = unmarshalSchedulerConf(defaultSchedulerConf)
                if err != nil {
                        klog.Errorf("unmarshal scheduler config %s failed: %v", defaultSchedulerConf, err)
                        panic("invalid default configuration")
                }
        })

        var config string
        if len(pc.schedulerConf) != 0 {
                if config, err = readSchedulerConf(pc.schedulerConf); err != nil {
                        klog.Errorf("Failed to read scheduler configuration '%s', using previous configuration: %v",
                                pc.schedulerConf, err)
                        return
                }
        }

        actions, plugins, configurations, err := unmarshalSchedulerConf(config)
        if err != nil {
                klog.Errorf("scheduler config %s is invalid: %v", config, err)
                return
        }

        pc.mutex.Lock()
        // If it is valid, use the new configuration
        pc.actions = actions
        pc.plugins = plugins
        pc.configurations = configurations
        pc.mutex.Unlock()
}

func (pc *Scheduler) watchSchedulerConf(stopCh <-chan struct{}) {
        if pc.fileWatcher == nil {
                return
        }
        eventCh := pc.fileWatcher.Events()
        errCh := pc.fileWatcher.Errors()
        for {
                select {
                case event, ok := <-eventCh:
                        if !ok {
                                return
                        }
                        klog.V(4).Infof("watch %s event: %v", pc.schedulerConf, event)
                        if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create {
                                pc.loadSchedulerConf()
                        }
                case err, ok := <-errCh:
                        if !ok {
                                return
                        }
                        klog.Infof("watch %s error: %v", pc.schedulerConf, err)
                case <-stopCh:
                        return
                }
        }
}

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
        "fmt"
        "io/ioutil"
        "strings"

        "gopkg.in/yaml.v2"

        "volcano.sh/volcano/pkg/scheduler/conf"
        "volcano.sh/volcano/pkg/scheduler/framework"
        "volcano.sh/volcano/pkg/scheduler/plugins"
)

var defaultSchedulerConf = `
actions: "enqueue, allocate, backfill"
tiers:
- plugins:
  - name: priority
  - name: gang
  - name: conformance
- plugins:
  - name: overcommit
  - name: drf
  - name: predicates
  - name: proportion
  - name: nodeorder
`

func unmarshalSchedulerConf(confStr string) ([]framework.Action, []conf.Tier, []conf.Configuration, error) {
        var actions []framework.Action

        schedulerConf := &conf.SchedulerConfiguration{}

        if err := yaml.Unmarshal([]byte(confStr), schedulerConf); err != nil {
                return nil, nil, nil, err
        }
        // Set default settings for each plugin if not set
        for i, tier := range schedulerConf.Tiers {
                // drf with hierarchy enabled
                hdrf := false
                // proportion enabled
                proportion := false
                for j := range tier.Plugins {
                        if tier.Plugins[j].Name == "drf" &&
                                tier.Plugins[j].EnabledHierarchy != nil &&
                                *tier.Plugins[j].EnabledHierarchy {
                                hdrf = true
                        }
                        if tier.Plugins[j].Name == "proportion" {
                                proportion = true
                        }
                        plugins.ApplyPluginConfDefaults(&schedulerConf.Tiers[i].Plugins[j])
                }
                if hdrf && proportion {
                        return nil, nil, nil, fmt.Errorf("proportion and drf with hierarchy enabled conflicts")
                }
        }

        actionNames := strings.Split(schedulerConf.Actions, ",")
        for _, actionName := range actionNames {
                if action, found := framework.GetAction(strings.TrimSpace(actionName)); found {
                        actions = append(actions, action)
                } else {
                        return nil, nil, nil, fmt.Errorf("failed to found Action %s, ignore it", actionName)
                }
        }

        return actions, schedulerConf.Tiers, schedulerConf.Configurations, nil
}

func readSchedulerConf(confPath string) (string, error) {
        dat, err := ioutil.ReadFile(confPath)
        if err != nil {
                return "", err
        }
        return string(dat), nil
}

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
        "container/heap"

        "volcano.sh/volcano/pkg/scheduler/api"
)

//PriorityQueue implements a scheduling queue.
type PriorityQueue struct {
        queue priorityQueue
}

type priorityQueue struct {
        items  []interface{}
        lessFn api.LessFn
}

// NewPriorityQueue returns a PriorityQueue
func NewPriorityQueue(lessFn api.LessFn) *PriorityQueue {
        return &PriorityQueue{
                queue: priorityQueue{
                        items:  make([]interface{}, 0),
                        lessFn: lessFn,
                },
        }
}

// Push pushes element in the priority Queue
func (q *PriorityQueue) Push(it interface{}) {
        heap.Push(&q.queue, it)
}

// Pop pops element in the priority Queue
func (q *PriorityQueue) Pop() interface{} {
        if q.Len() == 0 {
                return nil
        }

        return heap.Pop(&q.queue)
}

// Empty check if queue is empty
func (q *PriorityQueue) Empty() bool {
        return q.queue.Len() == 0
}

// Len returns Len of the priority queue
func (q *PriorityQueue) Len() int {
        return q.queue.Len()
}

func (pq *priorityQueue) Len() int { return len(pq.items) }

func (pq *priorityQueue) Less(i, j int) bool {
        if pq.lessFn == nil {
                return i < j
        }

        // We want Pop to give us the highest, not lowest, priority so we use greater than here.
        return pq.lessFn(pq.items[i], pq.items[j])
}

func (pq priorityQueue) Swap(i, j int) {
        pq.items[i], pq.items[j] = pq.items[j], pq.items[i]
}

func (pq *priorityQueue) Push(x interface{}) {
        (*pq).items = append((*pq).items, x)
}

func (pq *priorityQueue) Pop() interface{} {
        old := (*pq).items
        n := len(old)
        item := old[n-1]
        (*pq).items = old[0 : n-1]
        return item
}

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
        "context"
        "fmt"
        "math"
        "math/rand"
        "sort"
        "sync"
        "sync/atomic"

        "k8s.io/client-go/util/workqueue"
        "k8s.io/klog"
        k8sframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"

        "volcano.sh/volcano/cmd/scheduler/app/options"
        "volcano.sh/volcano/pkg/scheduler/api"
)

const baselinePercentageOfNodesToFind = 50

var lastProcessedNodeIndex int

// Reservation is used to record target job and locked nodes
var Reservation *ResourceReservation

func init() {
        Reservation = NewResourceReservation()
}

// CalculateNumOfFeasibleNodesToFind returns the number of feasible nodes that once found,
// the scheduler stops its search for more feasible nodes.
func CalculateNumOfFeasibleNodesToFind(numAllNodes int32) (numNodes int32) {
        opts := options.ServerOpts
        if numAllNodes <= opts.MinNodesToFind || opts.PercentageOfNodesToFind >= 100 {
                return numAllNodes
        }

        adaptivePercentage := opts.PercentageOfNodesToFind
        if adaptivePercentage <= 0 {
                adaptivePercentage = baselinePercentageOfNodesToFind - numAllNodes/125
                if adaptivePercentage < opts.MinPercentageOfNodesToFind {
                        adaptivePercentage = opts.MinPercentageOfNodesToFind
                }
        }

        numNodes = numAllNodes * adaptivePercentage / 100
        if numNodes < opts.MinNodesToFind {
                numNodes = opts.MinNodesToFind
        }
        return numNodes
}

// PredicateNodes returns the specified number of nodes that fit a task
func PredicateNodes(task *api.TaskInfo, nodes []*api.NodeInfo, fn api.PredicateFn) ([]*api.NodeInfo, *api.FitErrors) {
        //var workerLock sync.Mutex

        var errorLock sync.Mutex
        fe := api.NewFitErrors()

        allNodes := len(nodes)
        if allNodes == 0 {
                return make([]*api.NodeInfo, 0), fe
        }
        numNodesToFind := CalculateNumOfFeasibleNodesToFind(int32(allNodes))

        //allocate enough space to avoid growing it
        predicateNodes := make([]*api.NodeInfo, numNodesToFind)

        numFoundNodes := int32(0)
        processedNodes := int32(0)

        //create a context with cancellation
        ctx, cancel := context.WithCancel(context.Background())

        checkNode := func(index int) {
                // Check the nodes starting from where is left off in the previous scheduling cycle,
                // to make sure all nodes have the same chance of being examined across pods.
                node := nodes[(lastProcessedNodeIndex+index)%allNodes]
                atomic.AddInt32(&processedNodes, 1)
                klog.V(4).Infof("Considering Task <%v/%v> on node <%v>: <%v> vs. <%v>",
                        task.Namespace, task.Name, node.Name, task.Resreq, node.Idle)

                // TODO (k82cn): Enable eCache for performance improvement.
                if err := fn(task, node); err != nil {
                        klog.V(3).Infof("Predicates failed for task <%s/%s> on node <%s>: %v",
                                task.Namespace, task.Name, node.Name, err)
                        errorLock.Lock()
                        fe.SetNodeError(node.Name, err)
                        errorLock.Unlock()
                        return
                }

                //check if the number of found nodes is more than the numNodesTofind
                length := atomic.AddInt32(&numFoundNodes, 1)
                if length > numNodesToFind {
                        cancel()
                        atomic.AddInt32(&numFoundNodes, -1)
                } else {
                        predicateNodes[length-1] = node
                }
        }

        //workqueue.ParallelizeUntil(context.TODO(), 16, len(nodes), checkNode)
        workqueue.ParallelizeUntil(ctx, 16, allNodes, checkNode)

        //processedNodes := int(numFoundNodes) + len(filteredNodesStatuses) + len(failedPredicateMap)
        lastProcessedNodeIndex = (lastProcessedNodeIndex + int(processedNodes)) % allNodes
        predicateNodes = predicateNodes[:numFoundNodes]
        return predicateNodes, fe
}

// PrioritizeNodes returns a map whose key is node's score and value are corresponding nodes
func PrioritizeNodes(task *api.TaskInfo, nodes []*api.NodeInfo, batchFn api.BatchNodeOrderFn, mapFn api.NodeOrderMapFn, reduceFn api.NodeOrderReduceFn) map[float64][]*api.NodeInfo {
        pluginNodeScoreMap := map[string]k8sframework.NodeScoreList{}
        nodeOrderScoreMap := map[string]float64{}
        nodeScores := map[float64][]*api.NodeInfo{}
        var workerLock sync.Mutex
        scoreNode := func(index int) {
                node := nodes[index]
                mapScores, orderScore, err := mapFn(task, node)
                if err != nil {
                        klog.Errorf("Error in Calculating Priority for the node:%v", err)
                        return
                }

                workerLock.Lock()
                for plugin, score := range mapScores {
                        nodeScoreMap, ok := pluginNodeScoreMap[plugin]
                        if !ok {
                                nodeScoreMap = k8sframework.NodeScoreList{}
                        }
                        hp := k8sframework.NodeScore{}
                        hp.Name = node.Name
                        hp.Score = int64(math.Floor(score))
                        pluginNodeScoreMap[plugin] = append(nodeScoreMap, hp)
                }
                nodeOrderScoreMap[node.Name] = orderScore
                workerLock.Unlock()
        }
        workqueue.ParallelizeUntil(context.TODO(), 16, len(nodes), scoreNode)
        reduceScores, err := reduceFn(task, pluginNodeScoreMap)
        if err != nil {
                klog.Errorf("Error in Calculating Priority for the node:%v", err)
                return nodeScores
        }

        batchNodeScore, err := batchFn(task, nodes)
        if err != nil {
                klog.Errorf("Error in Calculating batch Priority for the node, err %v", err)
                return nodeScores
        }

        for _, node := range nodes {
                if score, found := reduceScores[node.Name]; found {
                        if orderScore, ok := nodeOrderScoreMap[node.Name]; ok {
                                score += orderScore
                        }
                        if batchScore, ok := batchNodeScore[node.Name]; ok {
                                score += batchScore
                        }
                        nodeScores[score] = append(nodeScores[score], node)
                } else {
                        // If no plugin is applied to this node, the default is 0.0
                        score = 0.0
                        if orderScore, ok := nodeOrderScoreMap[node.Name]; ok {
                                score += orderScore
                        }
                        if batchScore, ok := batchNodeScore[node.Name]; ok {
                                score += batchScore
                        }
                        nodeScores[score] = append(nodeScores[score], node)
                }
        }
        return nodeScores
}

// SortNodes returns nodes by order of score
func SortNodes(nodeScores map[float64][]*api.NodeInfo) []*api.NodeInfo {
        var nodesInorder []*api.NodeInfo
        var keys []float64
        for key := range nodeScores {
                keys = append(keys, key)
        }
        sort.Sort(sort.Reverse(sort.Float64Slice(keys)))
        for _, key := range keys {
                nodes := nodeScores[key]
                nodesInorder = append(nodesInorder, nodes...)
        }
        return nodesInorder
}

// SelectBestNode returns best node whose score is highest, pick one randomly if there are many nodes with same score.
func SelectBestNode(nodeScores map[float64][]*api.NodeInfo) *api.NodeInfo {
        var bestNodes []*api.NodeInfo
        maxScore := -1.0
        for score, nodes := range nodeScores {
                if score > maxScore {
                        maxScore = score
                        bestNodes = nodes
                }
        }

        if len(bestNodes) == 0 {
                return nil
        }

        return bestNodes[rand.Intn(len(bestNodes))]
}

// GetNodeList returns values of the map 'nodes'
func GetNodeList(nodes map[string]*api.NodeInfo, nodeList []string) []*api.NodeInfo {
        result := make([]*api.NodeInfo, 0, len(nodeList))
        for _, nodename := range nodeList {
                if ni, ok := nodes[nodename]; ok {
                        result = append(result, ni)
                }
        }
        return result
}

// ValidateVictims returns an error if the resources of the victims can't satisfy the preemptor
func ValidateVictims(preemptor *api.TaskInfo, node *api.NodeInfo, victims []*api.TaskInfo) error {
        if len(victims) == 0 {
                return fmt.Errorf("no victims")
        }
        futureIdle := node.FutureIdle()
        for _, victim := range victims {
                futureIdle.Add(victim.Resreq)
        }
        // Every resource of the preemptor needs to be less or equal than corresponding
        // idle resource after preemption.
        if !preemptor.InitResreq.LessEqual(futureIdle, api.Zero) {
                return fmt.Errorf("not enough resources: requested <%v>, but future idle <%v>",
                        preemptor.InitResreq, futureIdle)
        }
        return nil
}

// ResourceReservation is struct used for resource reservation
type ResourceReservation struct {
        TargetJob   *api.JobInfo
        LockedNodes map[string]*api.NodeInfo
}

// NewResourceReservation is used to create global instance
func NewResourceReservation() *ResourceReservation {
        return &ResourceReservation{
                TargetJob:   nil,
                LockedNodes: map[string]*api.NodeInfo{},
        }
}

// GetMinInt return minimum int from vals
func GetMinInt(vals ...int) int {
        if len(vals) == 0 {
                return 0
        }

        min := vals[0]
        for _, val := range vals {
                if val <= min {
                        min = val
                }
        }
        return min
}

/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
        "fmt"
        "sync"

        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/api/resource"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/types"
        volumescheduling "k8s.io/kubernetes/pkg/controller/volume/scheduling"

        schedulingv2 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/scheduler/api"
)

// BuildResourceList builts resource list object
func BuildResourceList(cpu string, memory string) v1.ResourceList {
        return v1.ResourceList{
                v1.ResourceCPU:      resource.MustParse(cpu),
                v1.ResourceMemory:   resource.MustParse(memory),
                api.GPUResourceName: resource.MustParse("0"),
        }
}

// BuildResourceListWithGPU builts resource list with GPU
func BuildResourceListWithGPU(cpu string, memory string, GPU string) v1.ResourceList {
        return v1.ResourceList{
                v1.ResourceCPU:      resource.MustParse(cpu),
                v1.ResourceMemory:   resource.MustParse(memory),
                api.GPUResourceName: resource.MustParse(GPU),
        }
}

// BuildNode builts node object
func BuildNode(name string, alloc v1.ResourceList, labels map[string]string) *v1.Node {
        return &v1.Node{
                ObjectMeta: metav1.ObjectMeta{
                        Name:        name,
                        Labels:      labels,
                        Annotations: map[string]string{},
                },
                Status: v1.NodeStatus{
                        Capacity:    alloc,
                        Allocatable: alloc,
                },
        }
}

// BuildPod builts Pod object
func BuildPod(namespace, name, nodename string, p v1.PodPhase, req v1.ResourceList, groupName string, labels map[string]string, selector map[string]string) *v1.Pod {
        return &v1.Pod{
                ObjectMeta: metav1.ObjectMeta{
                        UID:       types.UID(fmt.Sprintf("%v-%v", namespace, name)),
                        Name:      name,
                        Namespace: namespace,
                        Labels:    labels,
                        Annotations: map[string]string{
                                schedulingv2.KubeGroupNameAnnotationKey: groupName,
                        },
                },
                Status: v1.PodStatus{
                        Phase: p,
                },
                Spec: v1.PodSpec{
                        NodeName:     nodename,
                        NodeSelector: selector,
                        Containers: []v1.Container{
                                {
                                        Resources: v1.ResourceRequirements{
                                                Requests: req,
                                        },
                                },
                        },
                },
        }
}

// FakeBinder is used as fake binder
type FakeBinder struct {
        sync.Mutex
        Binds   map[string]string
        Channel chan string
}

// Bind used by fake binder struct to bind pods
func (fb *FakeBinder) Bind(p *v1.Pod, hostname string) error {
        fb.Lock()
        defer fb.Unlock()

        key := fmt.Sprintf("%v/%v", p.Namespace, p.Name)
        fb.Binds[key] = hostname

        fb.Channel <- key

        return nil
}

// FakeEvictor is used as fake evictor
type FakeEvictor struct {
        sync.Mutex
        evicts  []string
        Channel chan string
}

// Evicts returns copy of evicted pods.
func (fe *FakeEvictor) Evicts() []string {
        fe.Lock()
        defer fe.Unlock()
        return append([]string{}, fe.evicts...)
}

// Evict is used by fake evictor to evict pods
func (fe *FakeEvictor) Evict(p *v1.Pod, reason string) error {
        fe.Lock()
        defer fe.Unlock()

        fmt.Println("PodName: ", p.Name)
        key := fmt.Sprintf("%v/%v", p.Namespace, p.Name)
        fe.evicts = append(fe.evicts, key)

        fe.Channel <- key

        return nil
}

// FakeStatusUpdater is used for fake status update
type FakeStatusUpdater struct {
}

// UpdatePodCondition is a empty function
func (ftsu *FakeStatusUpdater) UpdatePodCondition(pod *v1.Pod, podCondition *v1.PodCondition) (*v1.Pod, error) {
        // do nothing here
        return nil, nil
}

// UpdatePodGroup is a empty function
func (ftsu *FakeStatusUpdater) UpdatePodGroup(pg *api.PodGroup) (*api.PodGroup, error) {
        // do nothing here
        return nil, nil
}

// FakeVolumeBinder is used as fake volume binder
type FakeVolumeBinder struct {
}

// AllocateVolumes is a empty function
func (fvb *FakeVolumeBinder) AllocateVolumes(task *api.TaskInfo, hostname string, podVolumes *volumescheduling.PodVolumes) error {
        return nil
}

// BindVolumes is a empty function
func (fvb *FakeVolumeBinder) BindVolumes(task *api.TaskInfo, podVolumes *volumescheduling.PodVolumes) error {
        return nil
}

// GetPodVolumes is a empty function
func (fvb *FakeVolumeBinder) GetPodVolumes(task *api.TaskInfo, node *v1.Node) (*volumescheduling.PodVolumes, error) {
        return nil, nil
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        "encoding/json"
        "fmt"
        "strconv"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

const (
        // DefaultQueue constant stores the name of the queue as "default"
        DefaultQueue = "default"
        // DefaultMaxRetry is the default number of retries.
        DefaultMaxRetry = 3

        defaultSchedulerName = "volcano"

        defaultMaxRetry int32 = 3
)

func init() {
        router.RegisterAdmission(service)
}

var service = &router.AdmissionService{
        Path: "/jobs/mutate",
        Func: Jobs,

        MutatingConfig: &whv1beta1.MutatingWebhookConfiguration{
                Webhooks: []whv1beta1.MutatingWebhook{{
                        Name: "mutatejob.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{"batch.volcano.sh"},
                                                APIVersions: []string{"v1alpha1"},
                                                Resources:   []string{"jobs"},
                                        },
                                },
                        },
                }},
        },
}

type patchOperation struct {
        Op    string      `json:"op"`
        Path  string      `json:"path"`
        Value interface{} `json:"value,omitempty"`
}

// Jobs mutate jobs.
func Jobs(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
        klog.V(3).Infof("mutating jobs")

        job, err := schema.DecodeJob(ar.Request.Object, ar.Request.Resource)
        if err != nil {
                return util.ToAdmissionResponse(err)
        }

        var patchBytes []byte
        switch ar.Request.Operation {
        case v1beta1.Create:
                patchBytes, _ = createPatch(job)
        default:
                err = fmt.Errorf("expect operation to be 'CREATE' ")
                return util.ToAdmissionResponse(err)
        }

        klog.V(3).Infof("AdmissionResponse: patch=%v", string(patchBytes))
        reviewResponse := v1beta1.AdmissionResponse{
                Allowed: true,
                Patch:   patchBytes,
        }
        pt := v1beta1.PatchTypeJSONPatch
        reviewResponse.PatchType = &pt

        return &reviewResponse
}

func createPatch(job *v1alpha1.Job) ([]byte, error) {
        var patch []patchOperation
        pathQueue := patchDefaultQueue(job)
        if pathQueue != nil {
                patch = append(patch, *pathQueue)
        }
        pathScheduler := patchDefaultScheduler(job)
        if pathScheduler != nil {
                patch = append(patch, *pathScheduler)
        }
        pathMaxRetry := patchDefaultMaxRetry(job)
        if pathMaxRetry != nil {
                patch = append(patch, *pathMaxRetry)
        }
        pathSpec := mutateSpec(job.Spec.Tasks, "/spec/tasks")
        if pathSpec != nil {
                patch = append(patch, *pathSpec)
        }
        pathMinAvailable := patchDefaultMinAvailable(job)
        if pathMinAvailable != nil {
                patch = append(patch, *pathMinAvailable)
        }
        return json.Marshal(patch)
}

func patchDefaultQueue(job *v1alpha1.Job) *patchOperation {
        //Add default queue if not specified.
        if job.Spec.Queue == "" {
                return &patchOperation{Op: "add", Path: "/spec/queue", Value: DefaultQueue}
        }
        return nil
}

func patchDefaultScheduler(job *v1alpha1.Job) *patchOperation {
        // Add default scheduler name if not specified.
        if job.Spec.SchedulerName == "" {
                return &patchOperation{Op: "add", Path: "/spec/schedulerName", Value: defaultSchedulerName}
        }
        return nil
}

func patchDefaultMaxRetry(job *v1alpha1.Job) *patchOperation {
        // Add default maxRetry if maxRetry is zero.
        if job.Spec.MaxRetry == 0 {
                return &patchOperation{Op: "add", Path: "/spec/maxRetry", Value: DefaultMaxRetry}
        }
        return nil
}

func patchDefaultMinAvailable(job *v1alpha1.Job) *patchOperation {
        // Add default minAvailable if minAvailable is zero.
        if job.Spec.MinAvailable == 0 {
                var jobMinAvailable int32
                for _, task := range job.Spec.Tasks {
                        if task.MinAvailable != nil {
                                jobMinAvailable += *task.MinAvailable
                        } else {
                                jobMinAvailable += task.Replicas
                        }
                }

                return &patchOperation{Op: "add", Path: "/spec/minAvailable", Value: jobMinAvailable}
        }
        return nil
}

func mutateSpec(tasks []v1alpha1.TaskSpec, basePath string) *patchOperation {
        patched := false
        for index := range tasks {
                // add default task name
                taskName := tasks[index].Name
                if len(taskName) == 0 {
                        patched = true
                        tasks[index].Name = v1alpha1.DefaultTaskSpec + strconv.Itoa(index)
                }

                if tasks[index].Template.Spec.HostNetwork && tasks[index].Template.Spec.DNSPolicy == "" {
                        patched = true
                        tasks[index].Template.Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
                }

                if tasks[index].MinAvailable == nil {
                        patched = true
                        minAvailable := tasks[index].Replicas
                        tasks[index].MinAvailable = &minAvailable
                }

                if tasks[index].MaxRetry == 0 {
                        patched = true
                        tasks[index].MaxRetry = defaultMaxRetry
                }
        }
        if !patched {
                return nil
        }
        return &patchOperation{
                Op:    "replace",
                Path:  basePath,
                Value: tasks,
        }
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
        "context"
        "fmt"
        "strings"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        v1 "k8s.io/api/core/v1"
        apiequality "k8s.io/apimachinery/pkg/api/equality"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/util/validation"
        "k8s.io/apimachinery/pkg/util/validation/field"
        "k8s.io/klog"
        k8score "k8s.io/kubernetes/pkg/apis/core"
        k8scorev1 "k8s.io/kubernetes/pkg/apis/core/v1"
        v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
        k8scorevalid "k8s.io/kubernetes/pkg/apis/core/validation"

        "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        jobhelpers "volcano.sh/volcano/pkg/controllers/job/helpers"
        "volcano.sh/volcano/pkg/controllers/job/plugins"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

func init() {
        router.RegisterAdmission(service)
}

var service = &router.AdmissionService{
        Path: "/jobs/validate",
        Func: AdmitJobs,

        Config: config,

        ValidatingConfig: &whv1beta1.ValidatingWebhookConfiguration{
                Webhooks: []whv1beta1.ValidatingWebhook{{
                        Name: "validatejob.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create, whv1beta1.Update},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{"batch.volcano.sh"},
                                                APIVersions: []string{"v1alpha1"},
                                                Resources:   []string{"jobs"},
                                        },
                                },
                        },
                }},
        },
}

var config = &router.AdmissionServiceConfig{}

// AdmitJobs is to admit jobs and return response.
func AdmitJobs(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
        klog.V(3).Infof("admitting jobs -- %s", ar.Request.Operation)

        job, err := schema.DecodeJob(ar.Request.Object, ar.Request.Resource)
        if err != nil {
                return util.ToAdmissionResponse(err)
        }
        var msg string
        reviewResponse := v1beta1.AdmissionResponse{}
        reviewResponse.Allowed = true

        switch ar.Request.Operation {
        case v1beta1.Create:
                msg = validateJobCreate(job, &reviewResponse)
        case v1beta1.Update:
                oldJob, err := schema.DecodeJob(ar.Request.OldObject, ar.Request.Resource)
                if err != nil {
                        return util.ToAdmissionResponse(err)
                }
                err = validateJobUpdate(oldJob, job)
                if err != nil {
                        return util.ToAdmissionResponse(err)
                }
        default:
                err := fmt.Errorf("expect operation to be 'CREATE' or 'UPDATE'")
                return util.ToAdmissionResponse(err)
        }

        if !reviewResponse.Allowed {
                reviewResponse.Result = &metav1.Status{Message: strings.TrimSpace(msg)}
        }
        return &reviewResponse
}

func validateJobCreate(job *v1alpha1.Job, reviewResponse *v1beta1.AdmissionResponse) string {
        var msg string
        taskNames := map[string]string{}
        var totalReplicas int32

        if job.Spec.MinAvailable < 0 {
                reviewResponse.Allowed = false
                return "job 'minAvailable' must be >= 0."
        }

        if job.Spec.MaxRetry < 0 {
                reviewResponse.Allowed = false
                return "'maxRetry' cannot be less than zero."
        }

        if job.Spec.TTLSecondsAfterFinished != nil && *job.Spec.TTLSecondsAfterFinished < 0 {
                reviewResponse.Allowed = false
                return "'ttlSecondsAfterFinished' cannot be less than zero."
        }

        if len(job.Spec.Tasks) == 0 {
                reviewResponse.Allowed = false
                return "No task specified in job spec"
        }

        for index, task := range job.Spec.Tasks {
                if task.Replicas < 0 {
                        msg += fmt.Sprintf(" 'replicas' < 0 in task: %s;", task.Name)
                }

                if task.MinAvailable != nil && *task.MinAvailable > task.Replicas {
                        msg += fmt.Sprintf(" 'minAvailable' is greater than 'replicas' in task: %s, job: %s", task.Name, job.Name)
                }

                // count replicas
                totalReplicas += task.Replicas

                // validate task name
                if errMsgs := validation.IsDNS1123Label(task.Name); len(errMsgs) > 0 {
                        msg += fmt.Sprintf(" %v;", errMsgs)
                }

                // duplicate task name
                if _, found := taskNames[task.Name]; found {
                        msg += fmt.Sprintf(" duplicated task name %s;", task.Name)
                        break
                } else {
                        taskNames[task.Name] = task.Name
                }

                if err := validatePolicies(task.Policies, field.NewPath("spec.tasks.policies")); err != nil {
                        msg += err.Error() + fmt.Sprintf(" valid events are %v, valid actions are %v",
                                getValidEvents(), getValidActions())
                }
                podName := jobhelpers.MakePodName(job.Name, task.Name, index)
                msg += validateK8sPodNameLength(podName)
                msg += validateTaskTemplate(task, job, index)
        }

        msg += validateJobName(job)

        if totalReplicas < job.Spec.MinAvailable {
                msg += "job 'minAvailable' should not be greater than total replicas in tasks;"
        }

        if err := validatePolicies(job.Spec.Policies, field.NewPath("spec.policies")); err != nil {
                msg = msg + err.Error() + fmt.Sprintf(" valid events are %v, valid actions are %v;",
                        getValidEvents(), getValidActions())
        }

        // invalid job plugins
        if len(job.Spec.Plugins) != 0 {
                for name := range job.Spec.Plugins {
                        if _, found := plugins.GetPluginBuilder(name); !found {
                                msg += fmt.Sprintf(" unable to find job plugin: %s", name)
                        }
                }
        }

        if err := validateIO(job.Spec.Volumes); err != nil {
                msg += err.Error()
        }

        queue, err := config.VolcanoClient.SchedulingV1beta1().Queues().Get(context.TODO(), job.Spec.Queue, metav1.GetOptions{})
        if err != nil {
                msg += fmt.Sprintf(" unable to find job queue: %v", err)
        } else if queue.Status.State != schedulingv1beta1.QueueStateOpen {
                msg += fmt.Sprintf("can only submit job to queue with state `Open`, "+
                        "queue `%s` status is `%s`", queue.Name, queue.Status.State)
        }

        if msg != "" {
                reviewResponse.Allowed = false
        }

        return msg
}

func validateJobUpdate(old, new *v1alpha1.Job) error {
        var totalReplicas int32
        for _, task := range new.Spec.Tasks {
                if task.Replicas < 0 {
                        return fmt.Errorf("'replicas' must be >= 0 in task: %s", task.Name)
                }

                if task.MinAvailable != nil && *task.MinAvailable > task.Replicas {
                        return fmt.Errorf("'minAvailable' must be <= 'replicas' in task: %s;", task.Name)
                }
                // count replicas
                totalReplicas += task.Replicas
        }
        if new.Spec.MinAvailable > totalReplicas {
                return fmt.Errorf("job 'minAvailable' must not be greater than total replicas")
        }
        if new.Spec.MinAvailable < 0 {
                return fmt.Errorf("job 'minAvailable' must be >= 0")
        }

        if len(old.Spec.Tasks) != len(new.Spec.Tasks) {
                return fmt.Errorf("job updates may not add or remove tasks")
        }
        // other fields under spec are not allowed to mutate
        new.Spec.MinAvailable = old.Spec.MinAvailable
        new.Spec.PriorityClassName = old.Spec.PriorityClassName
        for i := range new.Spec.Tasks {
                new.Spec.Tasks[i].Replicas = old.Spec.Tasks[i].Replicas
                new.Spec.Tasks[i].MinAvailable = old.Spec.Tasks[i].MinAvailable
        }

        // job controller will update the pvc name if not provided
        for i := range new.Spec.Volumes {
                if new.Spec.Volumes[i].VolumeClaim != nil {
                        new.Spec.Volumes[i].VolumeClaimName = ""
                }
        }
        for i := range old.Spec.Volumes {
                if old.Spec.Volumes[i].VolumeClaim != nil {
                        old.Spec.Volumes[i].VolumeClaimName = ""
                }
        }

        if !apiequality.Semantic.DeepEqual(new.Spec, old.Spec) {
                return fmt.Errorf("job updates may not change fields other than `minAvailable`, `tasks[*].replicas under spec`")
        }

        return nil
}

func validateTaskTemplate(task v1alpha1.TaskSpec, job *v1alpha1.Job, index int) string {
        var v1PodTemplate v1.PodTemplate
        v1PodTemplate.Template = *task.Template.DeepCopy()
        k8scorev1.SetObjectDefaults_PodTemplate(&v1PodTemplate)

        var coreTemplateSpec k8score.PodTemplateSpec
        k8scorev1.Convert_v1_PodTemplateSpec_To_core_PodTemplateSpec(&v1PodTemplate.Template, &coreTemplateSpec, nil)

        // Skip verify container SecurityContex.Privileged as it depends on
        // the kube-apiserver `allow-privileged` flag.
        for i, container := range coreTemplateSpec.Spec.Containers {
                if container.SecurityContext != nil && container.SecurityContext.Privileged != nil {
                        coreTemplateSpec.Spec.Containers[i].SecurityContext.Privileged = nil
                }
        }

        corePodTemplate := k8score.PodTemplate{
                ObjectMeta: metav1.ObjectMeta{
                        Name:      task.Name,
                        Namespace: job.Namespace,
                },
                Template: coreTemplateSpec,
        }

        if allErrs := k8scorevalid.ValidatePodTemplate(&corePodTemplate); len(allErrs) > 0 {
                msg := fmt.Sprintf("spec.task[%d].", index)
                for index := range allErrs {
                        msg += allErrs[index].Error() + ". "
                }
                return msg
        }

        msg := validateTaskTopoPolicy(task, index)
        if msg != "" {
                return msg
        }

        return ""
}

func validateK8sPodNameLength(podName string) string {
        if errMsgs := validation.IsQualifiedName(podName); len(errMsgs) > 0 {
                return fmt.Sprintf("create pod with name %s validate failed %v;", podName, errMsgs)
        }
        return ""
}

func validateJobName(job *v1alpha1.Job) string {
        if errMsgs := validation.IsQualifiedName(job.Name); len(errMsgs) > 0 {
                return fmt.Sprintf("create job with name %s validate failed %v", job.Name, errMsgs)
        }
        return ""
}

func validateTaskTopoPolicy(task v1alpha1.TaskSpec, index int) string {
        if task.TopologyPolicy == "" || task.TopologyPolicy == v1alpha1.None {
                return ""
        }

        template := task.Template.DeepCopy()

        for id, container := range template.Spec.Containers {
                if len(container.Resources.Requests) == 0 {
                        template.Spec.Containers[id].Resources.Requests = container.Resources.Limits.DeepCopy()
                }
        }

        for id, container := range template.Spec.InitContainers {
                if len(container.Resources.Requests) == 0 {
                        template.Spec.InitContainers[id].Resources.Requests = container.Resources.Limits.DeepCopy()
                }
        }

        pod := &v1.Pod{
                Spec: template.Spec,
        }

        if v1qos.GetPodQOS(pod) != v1.PodQOSGuaranteed {
                return fmt.Sprintf("spec.task[%d] isn't Guaranteed pod, kind=%v", index, v1qos.GetPodQOS(pod))
        }

        for id, container := range append(template.Spec.Containers, template.Spec.InitContainers...) {
                requestNum := guaranteedCPUs(container)
                if requestNum == 0 {
                        return fmt.Sprintf("the cpu request isn't  an integer in spec.task[%d] container[%d].",
                                index, id)
                }
        }

        return ""
}

func guaranteedCPUs(container v1.Container) int {
        cpuQuantity := container.Resources.Requests[v1.ResourceCPU]
        if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() {
                return 0
        }

        return int(cpuQuantity.Value())
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
        "fmt"

        "github.com/hashicorp/go-multierror"

        "k8s.io/apimachinery/pkg/util/validation/field"
        "k8s.io/kubernetes/pkg/apis/core/validation"

        batchv1alpha1 "volcano.sh/apis/pkg/apis/batch/v1alpha1"
        busv1alpha1 "volcano.sh/apis/pkg/apis/bus/v1alpha1"
)

// policyEventMap defines all policy events and whether to allow external use.
var policyEventMap = map[busv1alpha1.Event]bool{
        busv1alpha1.AnyEvent:           true,
        busv1alpha1.PodFailedEvent:     true,
        busv1alpha1.PodEvictedEvent:    true,
        busv1alpha1.JobUnknownEvent:    true,
        busv1alpha1.TaskCompletedEvent: true,
        busv1alpha1.TaskFailedEvent:    true,
        busv1alpha1.OutOfSyncEvent:     false,
        busv1alpha1.CommandIssuedEvent: false,
        busv1alpha1.JobUpdatedEvent:    true,
}

// policyActionMap defines all policy actions and whether to allow external use.
var policyActionMap = map[busv1alpha1.Action]bool{
        busv1alpha1.AbortJobAction:     true,
        busv1alpha1.RestartJobAction:   true,
        busv1alpha1.RestartTaskAction:  true,
        busv1alpha1.TerminateJobAction: true,
        busv1alpha1.CompleteJobAction:  true,
        busv1alpha1.ResumeJobAction:    true,
        busv1alpha1.SyncJobAction:      false,
        busv1alpha1.EnqueueAction:      false,
        busv1alpha1.SyncQueueAction:    false,
        busv1alpha1.OpenQueueAction:    false,
        busv1alpha1.CloseQueueAction:   false,
}

func validatePolicies(policies []batchv1alpha1.LifecyclePolicy, fldPath *field.Path) error {
        var err error
        policyEvents := map[busv1alpha1.Event]struct{}{}
        exitCodes := map[int32]struct{}{}

        for _, policy := range policies {
                if (policy.Event != "" || len(policy.Events) != 0) && policy.ExitCode != nil {
                        err = multierror.Append(err, fmt.Errorf("must not specify event and exitCode simultaneously"))
                        break
                }

                if policy.Event == "" && len(policy.Events) == 0 && policy.ExitCode == nil {
                        err = multierror.Append(err, fmt.Errorf("either event and exitCode should be specified"))
                        break
                }

                if len(policy.Event) != 0 || len(policy.Events) != 0 {
                        bFlag := false
                        policyEventsList := getEventList(policy)
                        for _, event := range policyEventsList {
                                if allow, ok := policyEventMap[event]; !ok || !allow {
                                        err = multierror.Append(err, field.Invalid(fldPath, event, "invalid policy event"))
                                        bFlag = true
                                        break
                                }

                                if allow, ok := policyActionMap[policy.Action]; !ok || !allow {
                                        err = multierror.Append(err, field.Invalid(fldPath, policy.Action, "invalid policy action"))
                                        bFlag = true
                                        break
                                }
                                if _, found := policyEvents[event]; found {
                                        err = multierror.Append(err, fmt.Errorf("duplicate event %v  across different policy", event))
                                        bFlag = true
                                        break
                                } else {
                                        policyEvents[event] = struct{}{}
                                }
                        }
                        if bFlag {
                                break
                        }
                } else {
                        if *policy.ExitCode == 0 {
                                err = multierror.Append(err, fmt.Errorf("0 is not a valid error code"))
                                break
                        }
                        if _, found := exitCodes[*policy.ExitCode]; found {
                                err = multierror.Append(err, fmt.Errorf("duplicate exitCode %v", *policy.ExitCode))
                                break
                        } else {
                                exitCodes[*policy.ExitCode] = struct{}{}
                        }
                }
        }

        if _, found := policyEvents[busv1alpha1.AnyEvent]; found && len(policyEvents) > 1 {
                err = multierror.Append(err, fmt.Errorf("if there's * here, no other policy should be here"))
        }

        return err
}

func getEventList(policy batchv1alpha1.LifecyclePolicy) []busv1alpha1.Event {
        policyEventsList := policy.Events
        if len(policy.Event) > 0 {
                policyEventsList = append(policyEventsList, policy.Event)
        }
        uniquePolicyEventlist := removeDuplicates(policyEventsList)
        return uniquePolicyEventlist
}

func removeDuplicates(eventList []busv1alpha1.Event) []busv1alpha1.Event {
        keys := make(map[busv1alpha1.Event]bool)
        list := []busv1alpha1.Event{}
        for _, val := range eventList {
                if _, value := keys[val]; !value {
                        keys[val] = true
                        list = append(list, val)
                }
        }
        return list
}

func getValidEvents() []busv1alpha1.Event {
        var events []busv1alpha1.Event
        for e, allow := range policyEventMap {
                if allow {
                        events = append(events, e)
                }
        }

        return events
}

func getValidActions() []busv1alpha1.Action {
        var actions []busv1alpha1.Action
        for a, allow := range policyActionMap {
                if allow {
                        actions = append(actions, a)
                }
        }

        return actions
}

// validateIO validates IO configuration.
func validateIO(volumes []batchv1alpha1.VolumeSpec) error {
        volumeMap := map[string]bool{}
        for _, volume := range volumes {
                if len(volume.MountPath) == 0 {
                        return fmt.Errorf(" mountPath is required;")
                }
                if _, found := volumeMap[volume.MountPath]; found {
                        return fmt.Errorf(" duplicated mountPath: %s;", volume.MountPath)
                }
                if volume.VolumeClaim == nil && volume.VolumeClaimName == "" {
                        return fmt.Errorf(" either VolumeClaim or VolumeClaimName must be specified;")
                }
                if len(volume.VolumeClaimName) != 0 {
                        if volume.VolumeClaim != nil {
                                return fmt.Errorf("conflict: If you want to use an existing PVC, just specify VolumeClaimName." +
                                        "If you want to create a new PVC, you do not need to specify VolumeClaimName")
                        }
                        if errMsgs := validation.ValidatePersistentVolumeName(volume.VolumeClaimName, false); len(errMsgs) > 0 {
                                return fmt.Errorf("invalid VolumeClaimName %s : %v", volume.VolumeClaimName, errMsgs)
                        }
                }

                volumeMap[volume.MountPath] = true
        }
        return nil
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        "github.com/imdario/mergo"
        "gopkg.in/yaml.v2"
        "k8s.io/klog"

        v1 "k8s.io/api/core/v1"

        wkconfig "volcano.sh/volcano/pkg/webhooks/config"
)

type annotationResGroup struct{}

const (
        // defaultAnnotationKey: default annotation key
        defaultAnnotationKey = "volcano.sh/resource-group"
)

// NewAnnotationResGroup create a new structure
func NewAnnotationResGroup() ResGroup {
        return &annotationResGroup{}
}

// getAnnotation get annotations from the resource group
func getAnnotation(resGroupConfig wkconfig.ResGroupConfig) map[string]string {
        annotations := make(map[string]string)
        for _, val := range resGroupConfig.Object.Value {
                tmp := make(map[string]string)
                err := yaml.Unmarshal([]byte(val), &tmp)
                if err != nil {
                        continue
                }

                if err := mergo.Merge(&annotations, &tmp); err != nil {
                        klog.Errorf("annotations merge failed, err=%v", err)
                        continue
                }
        }

        return annotations
}

// IsBelongResGroup adjust whether pod is belong to the resource group
func (resGroup *annotationResGroup) IsBelongResGroup(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) bool {
        if resGroupConfig.Object.Key != "" && resGroupConfig.Object.Key != "annotation" {
                return false
        }

        annotations := getAnnotation(resGroupConfig)
        klog.V(3).Infof("annotations : %v", annotations)
        for key, annotation := range annotations {
                if pod.Annotations[key] == annotation {
                        return true
                }
        }

        if resGroupConfig.Object.Key == "" && pod.Annotations[defaultAnnotationKey] == resGroupConfig.ResourceGroup {
                return true
        }

        return false
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        v1 "k8s.io/api/core/v1"

        wkconfig "volcano.sh/volcano/pkg/webhooks/config"
)

// ResGroup interface for resource group
type ResGroup interface {
        IsBelongResGroup(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) bool
}

// GetResGroup return the interface besed on resourceGroup.Object.Key
func GetResGroup(resourceGroup wkconfig.ResGroupConfig) ResGroup {
        switch resourceGroup.Object.Key {
        case "namespace":
                return NewNamespaceResGroup()
        case "annotation":
                return NewAnnotationResGroup()
        }
        return NewAnnotationResGroup()
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        "encoding/json"
        "fmt"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        v1 "k8s.io/api/core/v1"
        "k8s.io/klog"

        wkconfig "volcano.sh/volcano/pkg/webhooks/config"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

// patchOperation define the patch operation structure
type patchOperation struct {
        Op    string      `json:"op"`
        Path  string      `json:"path"`
        Value interface{} `json:"value,omitempty"`
}

// init register mutate pod
func init() {
        router.RegisterAdmission(service)
}

var service = &router.AdmissionService{
        Path:   "/pods/mutate",
        Func:   Pods,
        Config: config,
        MutatingConfig: &whv1beta1.MutatingWebhookConfiguration{
                Webhooks: []whv1beta1.MutatingWebhook{{
                        Name: "mutatepod.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{""},
                                                APIVersions: []string{"v1"},
                                                Resources:   []string{"pods"},
                                        },
                                },
                        },
                }},
        },
}

var config = &router.AdmissionServiceConfig{}

// Pods mutate pods.
func Pods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
        klog.V(3).Infof("mutating pods -- %s", ar.Request.Operation)
        pod, err := schema.DecodePod(ar.Request.Object, ar.Request.Resource)
        if err != nil {
                return util.ToAdmissionResponse(err)
        }

        if pod.Namespace == "" {
                pod.Namespace = ar.Request.Namespace
        }

        var patchBytes []byte
        switch ar.Request.Operation {
        case v1beta1.Create:
                patchBytes, _ = createPatch(pod)
        default:
                err = fmt.Errorf("expect operation to be 'CREATE' ")
                return util.ToAdmissionResponse(err)
        }

        reviewResponse := v1beta1.AdmissionResponse{
                Allowed: true,
                Patch:   patchBytes,
        }
        pt := v1beta1.PatchTypeJSONPatch
        reviewResponse.PatchType = &pt

        return &reviewResponse
}

// createPatch patch pod
func createPatch(pod *v1.Pod) ([]byte, error) {
        if config.ConfigData == nil {
                klog.V(5).Infof("admission configuration is empty.")
                return nil, nil
        }

        var patch []patchOperation
        config.ConfigData.Lock()
        defer config.ConfigData.Unlock()

        for _, resourceGroup := range config.ConfigData.ResGroupsConfig {
                klog.V(3).Infof("resourceGroup %s", resourceGroup.ResourceGroup)
                group := GetResGroup(resourceGroup)
                if !group.IsBelongResGroup(pod, resourceGroup) {
                        continue
                }

                patchLabel := patchLabels(pod, resourceGroup)
                if patchLabel != nil {
                        patch = append(patch, *patchLabel)
                }

                patchToleration := patchTaintToleration(pod, resourceGroup)
                if patchToleration != nil {
                        patch = append(patch, *patchToleration)
                }
                patchScheduler := patchSchedulerName(resourceGroup)
                if patchScheduler != nil {
                        patch = append(patch, *patchScheduler)
                }

                klog.V(5).Infof("pod patch %v", patch)
                return json.Marshal(patch)
        }

        return json.Marshal(patch)
}

// patchLabels patch label
func patchLabels(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) *patchOperation {
        if len(resGroupConfig.Labels) == 0 {
                return nil
        }

        nodeSelector := make(map[string]string)
        for key, label := range pod.Spec.NodeSelector {
                nodeSelector[key] = label
        }

        for key, label := range resGroupConfig.Labels {
                nodeSelector[key] = label
        }

        return &patchOperation{Op: "add", Path: "/spec/nodeSelector", Value: nodeSelector}
}

// patchTaintToleration patch taint toleration
func patchTaintToleration(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) *patchOperation {
        if len(resGroupConfig.Tolerations) == 0 {
                return nil
        }

        var dst []v1.Toleration
        dst = append(dst, pod.Spec.Tolerations...)
        dst = append(dst, resGroupConfig.Tolerations...)

        return &patchOperation{Op: "add", Path: "/spec/tolerations", Value: dst}
}

// patchSchedulerName patch scheduler
func patchSchedulerName(resGroupConfig wkconfig.ResGroupConfig) *patchOperation {
        if resGroupConfig.SchedulerName == "" {
                return nil
        }

        return &patchOperation{Op: "add", Path: "/spec/schedulerName", Value: resGroupConfig.SchedulerName}
}

/*
Copyright 2021 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        v1 "k8s.io/api/core/v1"

        wkconfig "volcano.sh/volcano/pkg/webhooks/config"
)

type namespaceResGroup struct{}

// NewNamespaceResGroup create a new structure
func NewNamespaceResGroup() ResGroup {
        return &namespaceResGroup{}
}

// IsBelongResGroup adjust whether pod is belong to the resource group
func (resGroup *namespaceResGroup) IsBelongResGroup(pod *v1.Pod, resGroupConfig wkconfig.ResGroupConfig) bool {
        if resGroupConfig.Object.Key != "namespace" {
                return false
        }

        for _, val := range resGroupConfig.Object.Value {
                if pod.Namespace == val {
                        return true
                }
        }

        return false
}

/*
Copyright 2019 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
        "context"
        "fmt"
        "strconv"

        "strings"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        v1 "k8s.io/api/core/v1"
        apierrors "k8s.io/apimachinery/pkg/api/errors"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/util/intstr"
        "k8s.io/klog"

        "volcano.sh/apis/pkg/apis/helpers"
        vcv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

func init() {
        router.RegisterAdmission(service)
}

var service = &router.AdmissionService{
        Path: "/pods/validate",
        Func: AdmitPods,

        Config: config,

        ValidatingConfig: &whv1beta1.ValidatingWebhookConfiguration{
                Webhooks: []whv1beta1.ValidatingWebhook{{
                        Name: "validatepod.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{""},
                                                APIVersions: []string{"v1"},
                                                Resources:   []string{"pods"},
                                        },
                                },
                        },
                }},
        },
}

var config = &router.AdmissionServiceConfig{}

// AdmitPods is to admit pods and return response.
func AdmitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
        klog.V(3).Infof("admitting pods -- %s", ar.Request.Operation)

        pod, err := schema.DecodePod(ar.Request.Object, ar.Request.Resource)
        if err != nil {
                return util.ToAdmissionResponse(err)
        }

        var msg string
        reviewResponse := v1beta1.AdmissionResponse{}
        reviewResponse.Allowed = true

        switch ar.Request.Operation {
        case v1beta1.Create:
                msg = validatePod(pod, &reviewResponse)
        default:
                err := fmt.Errorf("expect operation to be 'CREATE'")
                return util.ToAdmissionResponse(err)
        }

        if !reviewResponse.Allowed {
                reviewResponse.Result = &metav1.Status{Message: strings.TrimSpace(msg)}
        }
        return &reviewResponse
}

/*
allow pods to create when
1. schedulerName of pod isn't volcano
2. pod has Podgroup whose phase isn't Pending
3. normal pods whose schedulerName is volcano don't have podgroup.
4. check pod budget annotations configure
*/
func validatePod(pod *v1.Pod, reviewResponse *v1beta1.AdmissionResponse) string {
        if pod.Spec.SchedulerName != config.SchedulerName {
                return ""
        }

        pgName := ""
        msg := ""

        // vc-job, SN == volcano
        if pod.Annotations != nil {
                pgName = pod.Annotations[vcv1beta1.KubeGroupNameAnnotationKey]
        }
        if pgName != "" {
                if err := checkPGPhase(pod, pgName, true); err != nil {
                        msg = err.Error()
                        reviewResponse.Allowed = false
                }
                return msg
        }

        // normal pod, SN == volcano
        pgName = helpers.GeneratePodgroupName(pod)
        if err := checkPGPhase(pod, pgName, false); err != nil {
                msg = err.Error()
                reviewResponse.Allowed = false
        }

        // check pod annotatations
        if err := validateAnnotation(pod); err != nil {
                msg = err.Error()
                reviewResponse.Allowed = false
        }

        return msg
}

func checkPGPhase(pod *v1.Pod, pgName string, isVCJob bool) error {
        pg, err := config.VolcanoClient.SchedulingV1beta1().PodGroups(pod.Namespace).Get(context.TODO(), pgName, metav1.GetOptions{})
        if err != nil {
                if isVCJob || (!isVCJob && !apierrors.IsNotFound(err)) {
                        return fmt.Errorf("failed to get PodGroup for pod <%s/%s>: %v", pod.Namespace, pod.Name, err)
                }
                return nil
        }
        if pg.Status.Phase != vcv1beta1.PodGroupPending {
                return nil
        }
        return fmt.Errorf("failed to create pod <%s/%s> as the podgroup phase is Pending",
                pod.Namespace, pod.Name)
}

func validateAnnotation(pod *v1.Pod) error {
        num := 0
        if len(pod.Annotations) > 0 {
                keys := []string{
                        vcv1beta1.JDBMinAvailable,
                        vcv1beta1.JDBMaxUnavailable,
                }
                for _, key := range keys {
                        if value, found := pod.Annotations[key]; found {
                                num++
                                if err := validateIntPercentageStr(key, value); err != nil {
                                        recordEvent(err)
                                        return err
                                }
                        }
                }
                if num > 1 {
                        return fmt.Errorf("not allow configure multiple annotations <%v> at same time", keys)
                }
        }
        return nil
}

func recordEvent(err error) {
        config.Recorder.Eventf(nil, v1.EventTypeWarning, "Admit", "Create pod failed due to %v", err)
}

func validateIntPercentageStr(key, value string) error {
        tmp := intstr.Parse(value)
        switch tmp.Type {
        case intstr.Int:
                if tmp.IntValue() <= 0 {
                        return fmt.Errorf("invalid value <%q> for %v, it must be a positive integer", value, key)
                }
                return nil
        case intstr.String:
                s := strings.Replace(tmp.StrVal, "%", "", -1)
                v, err := strconv.Atoi(s)
                if err != nil {
                        return fmt.Errorf("invalid value %v for %v", err, key)
                }
                if v <= 0 || v >= 100 {
                        return fmt.Errorf("invalid value <%q> for %v, it must be a valid percentage which between 1%% ~ 99%%", tmp.StrVal, key)
                }
                return nil
        }
        return fmt.Errorf("invalid type: neither int nor percentage for %v", key)
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mutate

import (
        "encoding/json"
        "fmt"
        "strings"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/klog"

        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

func init() {
        router.RegisterAdmission(service)
}

var service = &router.AdmissionService{
        Path: "/queues/mutate",
        Func: Queues,

        MutatingConfig: &whv1beta1.MutatingWebhookConfiguration{
                Webhooks: []whv1beta1.MutatingWebhook{{
                        Name: "mutatequeue.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{schedulingv1beta1.SchemeGroupVersion.Group},
                                                APIVersions: []string{schedulingv1beta1.SchemeGroupVersion.Version},
                                                Resources:   []string{"queues"},
                                        },
                                },
                        },
                }},
        },
}

type patchOperation struct {
        Op    string      `json:"op"`
        Path  string      `json:"path"`
        Value interface{} `json:"value,omitempty"`
}

// Queues mutate queues.
func Queues(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
        klog.V(3).Infof("Mutating %s queue %s.", ar.Request.Operation, ar.Request.Name)

        queue, err := schema.DecodeQueue(ar.Request.Object, ar.Request.Resource)
        if err != nil {
                return util.ToAdmissionResponse(err)
        }

        var patchBytes []byte
        switch ar.Request.Operation {
        case v1beta1.Create:
                patchBytes, err = createQueuePatch(queue)
        default:
                return util.ToAdmissionResponse(fmt.Errorf("invalid operation `%s`, "+
                        "expect operation to be `CREATE`", ar.Request.Operation))
        }

        if err != nil {
                return &v1beta1.AdmissionResponse{
                        Allowed: false,
                        Result:  &metav1.Status{Message: err.Error()},
                }
        }

        pt := v1beta1.PatchTypeJSONPatch
        return &v1beta1.AdmissionResponse{
                Allowed:   true,
                Patch:     patchBytes,
                PatchType: &pt,
        }
}

func createQueuePatch(queue *schedulingv1beta1.Queue) ([]byte, error) {
        var patch []patchOperation

        // add root node if the root node not specified
        hierarchy := queue.Annotations[schedulingv1beta1.KubeHierarchyAnnotationKey]
        hierarchicalWeights := queue.Annotations[schedulingv1beta1.KubeHierarchyWeightAnnotationKey]

        if hierarchy != "" && hierarchicalWeights != "" && !strings.HasPrefix(hierarchy, "root") {
                // based on https://tools.ietf.org/html/rfc6901#section-3
                // escape "/" with "~1"
                patch = append(patch, patchOperation{
                        Op:    "add",
                        Path:  fmt.Sprintf("/metadata/annotations/%s", strings.ReplaceAll(schedulingv1beta1.KubeHierarchyAnnotationKey, "/", "~1")),
                        Value: fmt.Sprintf("root/%s", hierarchy),
                })
                patch = append(patch, patchOperation{
                        Op:    "add",
                        Path:  fmt.Sprintf("/metadata/annotations/%s", strings.ReplaceAll(schedulingv1beta1.KubeHierarchyWeightAnnotationKey, "/", "~1")),
                        Value: fmt.Sprintf("1/%s", hierarchicalWeights),
                })
        }

        trueValue := true
        if queue.Spec.Reclaimable == nil {
                patch = append(patch, patchOperation{
                        Op:    "add",
                        Path:  "/spec/reclaimable",
                        Value: &trueValue,
                })
        }

        defaultWeight := 1
        if queue.Spec.Weight == 0 {
                patch = append(patch, patchOperation{
                        Op:    "add",
                        Path:  "/spec/weight",
                        Value: &defaultWeight,
                })
        }

        return json.Marshal(patch)
}

/*
Copyright 2018 The Volcano Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
        "context"
        "fmt"
        "strconv"
        "strings"

        "k8s.io/api/admission/v1beta1"
        whv1beta1 "k8s.io/api/admissionregistration/v1beta1"
        metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
        "k8s.io/apimachinery/pkg/util/validation/field"
        "k8s.io/klog"

        schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
        "volcano.sh/volcano/pkg/webhooks/router"
        "volcano.sh/volcano/pkg/webhooks/schema"
        "volcano.sh/volcano/pkg/webhooks/util"
)

func init() {
        router.RegisterAdmission(service)
}

var service = &router.AdmissionService{
        Path: "/queues/validate",
        Func: AdmitQueues,

        Config: config,

        ValidatingConfig: &whv1beta1.ValidatingWebhookConfiguration{
                Webhooks: []whv1beta1.ValidatingWebhook{{
                        Name: "validatequeue.volcano.sh",
                        Rules: []whv1beta1.RuleWithOperations{
                                {
                                        Operations: []whv1beta1.OperationType{whv1beta1.Create, whv1beta1.Update, whv1beta1.Delete},
                                        Rule: whv1beta1.Rule{
                                                APIGroups:   []string{schedulingv1beta1.SchemeGroupVersion.Group},
                                                APIVersions: []string{schedulingv1beta1.SchemeGroupVersion.Version},
                                                Resources:   []string{"queues"},
                                        },
                                },
                        },
                }},
        },
}

var config = &router.AdmissionServiceConfig{}

// AdmitQueues is to admit queues and return response.
func AdmitQueues(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse {
        klog.V(3).Infof("Admitting %s queue %s.", ar.Request.Operation, ar.Request.Name)

        queue, err := schema.DecodeQueue(ar.Request.Object, ar.Request.Resource)
        if err != nil {
                return util.ToAdmissionResponse(err)
        }

        switch ar.Request.Operation {
        case v1beta1.Create, v1beta1.Update:
                err = validateQueue(queue)
        case v1beta1.Delete:
                err = validateQueueDeleting(ar.Request.Name)
        default:
                return util.ToAdmissionResponse(fmt.Errorf("invalid operation `%s`, "+
                        "expect operation to be `CREATE`, `UPDATE` or `DELETE`", ar.Request.Operation))
        }

        if err != nil {
                return &v1beta1.AdmissionResponse{
                        Allowed: false,
                        Result:  &metav1.Status{Message: err.Error()},
                }
        }

        return &v1beta1.AdmissionResponse{
                Allowed: true,
        }
}

func validateQueue(queue *schedulingv1beta1.Queue) error {
        errs := field.ErrorList{}
        resourcePath := field.NewPath("requestBody")

        errs = append(errs, validateStateOfQueue(queue.Status.State, resourcePath.Child("spec").Child("state"))...)
        errs = append(errs, validateWeightOfQueue(queue.Spec.Weight, resourcePath.Child("spec").Child("weight"))...)
        errs = append(errs, validateHierarchicalAttributes(queue, resourcePath.Child("metadata").Child("annotations"))...)

        if len(errs) > 0 {
                return errs.ToAggregate()
        }

        return nil
}
func validateHierarchicalAttributes(queue *schedulingv1beta1.Queue, fldPath *field.Path) field.ErrorList {
        errs := field.ErrorList{}
        hierarchy := queue.Annotations[schedulingv1beta1.KubeHierarchyAnnotationKey]
        hierarchicalWeights := queue.Annotations[schedulingv1beta1.KubeHierarchyWeightAnnotationKey]
        if hierarchy != "" || hierarchicalWeights != "" {
                paths := strings.Split(hierarchy, "/")
                weights := strings.Split(hierarchicalWeights, "/")
                // path length must be the same with weights length
                if len(paths) != len(weights) {
                        return append(errs, field.Invalid(fldPath, hierarchy,
                                fmt.Sprintf("%s must have the same length with %s",
                                        schedulingv1beta1.KubeHierarchyAnnotationKey,
                                        schedulingv1beta1.KubeHierarchyWeightAnnotationKey,
                                )))
                }

                // check weights format
                for _, weight := range weights {
                        weightFloat, err := strconv.ParseFloat(weight, 64)
                        if err != nil {
                                return append(errs, field.Invalid(fldPath, hierarchicalWeights,
                                        fmt.Sprintf("%s in the %s is invalid number: %v",
                                                weight, hierarchicalWeights, err,
                                        )))
                        }
                        if weightFloat <= 0 {
                                return append(errs, field.Invalid(fldPath, hierarchicalWeights,
                                        fmt.Sprintf("%s in the %s must be larger than 0",
                                                weight, hierarchicalWeights,
                                        )))
                        }
                }

                // The node is not allowed to be in the sub path of a node.
                // For example, a queue with "root/sci" conflicts with a queue with "root/sci/dev"
                queueList, err := config.VolcanoClient.SchedulingV1beta1().Queues().List(context.TODO(), metav1.ListOptions{})
                if err != nil {
                        return append(errs, field.Invalid(fldPath, hierarchy,
                                fmt.Sprintf("checking %s, list queues failed: %v",
                                        schedulingv1beta1.KubeHierarchyAnnotationKey,
                                        err,
                                )))
                }
                for _, queueInTree := range queueList.Items {
                        hierarchyInTree := queueInTree.Annotations[schedulingv1beta1.KubeHierarchyAnnotationKey]
                        if hierarchyInTree != "" && queue.Name != queueInTree.Name &&
                                strings.HasPrefix(hierarchyInTree, hierarchy) {
                                return append(errs, field.Invalid(fldPath, hierarchy,
                                        fmt.Sprintf("%s is not allowed to be in the sub path of %s of queue %s",
                                                hierarchy, hierarchyInTree, queueInTree.Name)))
                        }
                }
        }
        return errs
}

func validateStateOfQueue(value schedulingv1beta1.QueueState, fldPath *field.Path) field.ErrorList {
        errs := field.ErrorList{}

        if len(value) == 0 {
                return errs
        }

        validQueueStates := []schedulingv1beta1.QueueState{
                schedulingv1beta1.QueueStateOpen,
                schedulingv1beta1.QueueStateClosed,
        }

        for _, validQueue := range validQueueStates {
                if value == validQueue {
                        return errs
                }
        }

        return append(errs, field.Invalid(fldPath, value, fmt.Sprintf("queue state must be in %v", validQueueStates)))
}

func validateWeightOfQueue(value int32, fldPath *field.Path) field.ErrorList {
        errs := field.ErrorList{}
        if value > 0 {
                return errs
        }
        return append(errs, field.Invalid(fldPath, value, "queue weight must be a positive integer"))
}

func validateQueueDeleting(queue string) error {
        if queue == "default" {
                return fmt.Errorf("`%s` queue can not be deleted", "default")
        }

        q, err := config.VolcanoClient.SchedulingV1beta1().Queues().Get(context.TODO(), queue, metav1.GetOptions{})
        if err != nil {
                return err
        }

        if q.Status.State != schedulingv1beta1.QueueStateClosed {
                return fmt.Errorf("only queue with state `%s` can be deleted, queue `%s` state is `%s`",
                        schedulingv1beta1.QueueStateClosed, q.Name, q.Status.State)
        }

        return nil
}