June 20-22 Announcing HashiConf Europe full schedule: keynotes, sessions, labs & more Register Now
  • Overview
    • Batch Processing Workloads
    • Edge Workload Management
    • Non-Containerized Application Orchestration
    • Simple Container Orchestration
  • Enterprise
  • Tutorials
  • Docs
  • API
  • Plugins
  • Tools
  • Community
GitHub—Stars on GitHub
Download
    • v1.3.x (latest)
    • v1.2.x
    • v1.1.x
    • v1.0.x
    • v0.12.x
    • v0.11.x
    • Overview
    • Quickstart
      • Overview
      • Requirements
      • Reference Architecture
      • Deployment Guide
    • Windows Service
    • Overview
    • Specific Version Details
    • Overview
    • Consul
    • Consul Service Mesh
    • Vault Integration

    • Overview
    • Architecture
      • Overview
      • Base
      • Task Drivers
      • Devices
      • Storage
      • Overview
      • Internals
      • Preemption
    • Consensus Protocol
    • Filesystem
    • Gossip Protocol
    • Security Model
    • Overview
    • acl
    • audit
    • autopilot
    • client
    • consul
    • plugin
    • sentinel
    • search
    • server
    • server_join
    • telemetry
    • tls
    • ui
    • vault
    • Overview
      • Overview
      • bootstrap
      • policy apply
      • policy delete
      • policy info
      • policy list
      • token create
      • token delete
      • token info
      • token list
      • token self
      • token update
    • agent
    • agent-info
      • Overview
      • exec
      • fs
      • logs
      • restart
      • signal
      • status
      • stop
      • Overview
      • validate
      • Overview
      • fail
      • list
      • pause
      • promote
      • resume
      • status
      • unblock
      • Overview
      • list
      • status
      • Overview
      • allocs
      • deployments
      • dispatch
      • eval
      • history
      • init
      • inspect
      • plan
      • periodic force
      • promote
      • revert
      • run
      • scale
      • scaling-events
      • status
      • stop
      • validate
      • Overview
      • get
    • monitor
      • Overview
      • apply
      • delete
      • inspect
      • list
      • status
      • Overview
      • config
      • drain
      • eligibility
      • status
      • Overview
      • api
      • autopilot get-config
      • autopilot set-config
      • debug
      • keygen
      • keyring
      • metrics
      • raft info
      • raft list-peers
      • raft logs
      • raft remove-peer
      • raft state
      • snapshot agent
      • snapshot inspect
      • snapshot restore
      • snapshot save
      • snapshot state
      • Overview
      • status
      • Overview
      • apply
      • delete
      • init
      • inspect
      • list
      • status
      • Overview
      • apply
      • dismiss
      • info
      • list
      • Overview
      • policy info
      • policy list
      • Overview
      • apply
      • delete
      • list
      • read
      • Overview
      • force-leave
      • join
      • members
      • Overview
      • service delete
      • service info
      • service list
    • status
      • Overview
      • gc
      • reconcile summaries
    • ui
    • version
      • Overview
      • create
      • delete
      • deregister
      • detach
      • init
      • register
      • snapshot create
      • snapshot delete
      • snapshot list
      • status

    • Overview
      • Overview
      • Expressions
        • Overview
          • chunklist
          • coalesce
          • coalescelist
          • compact
          • concat
          • contains
          • distinct
          • element
          • flatten
          • index
          • keys
          • length
          • lookup
          • merge
          • range
          • reverse
          • setintersection
          • setproduct
          • setunion
          • slice
          • sort
          • values
          • zipmap
          • can
          • convert
          • try
          • bcrypt
          • md5
          • rsadecrypt
          • sha1
          • sha256
          • sha512
          • formatdate
          • timeadd
          • base64decode
          • base64encode
          • csvdecode
          • jsondecode
          • jsonencode
          • urlencode
          • yamldecode
          • yamlencode
          • abspath
          • basename
          • dirname
          • file
          • fileexists
          • fileset
          • pathexpand
          • cidrhost
          • cidrnetmask
          • cidrsubnet
          • cidrsubnets
          • abs
          • ceil
          • floor
          • log
          • max
          • min
          • parseint
          • pow
          • signum
          • chomp
          • format
          • formatlist
          • indent
          • join
          • lower
          • regex_replace
          • replace
          • split
          • strrev
          • substr
          • title
          • trim
          • trimprefix
          • trimspace
          • trimsuffix
          • upper
          • uuidv4
          • uuidv5
      • Locals
      • Syntax
      • Variables
    • artifact
    • affinity
    • check_restart
    • connect
    • constraint
    • csi_plugin
    • device
    • dispatch_payload
    • env
    • ephemeral_disk
    • expose
    • gateway
    • group
    • job
    • lifecycle
    • logs
    • meta
    • migrate
    • multiregion
    • network
    • parameterized
    • periodic
    • proxy
    • reschedule
    • resources
    • restart
    • scaling
    • service
    • sidecar_service
    • sidecar_task
    • spread
    • task
    • template
    • update
    • upstreams
    • vault
    • volume
    • volume_mount
    • Overview
    • Docker
    • Isolated Fork/Exec
    • Java
    • Podman
    • QEMU
    • Raw Fork/Exec
      • Overview
      • containerd
      • Firecracker driver
      • Jailtask driver
      • Lightrun
      • LXC
      • Pot
      • Rkt Deprecated
      • Rookout
      • Singularity
      • systemd-nspawn
      • Windows IIS
      • Overview
      • ECS
    • Overview
      • Overview
      • Nvidia
      • USB Beta
  • Schedulers
    • Overview
    • Runtime Environment
    • Variable Interpolation
    • Overview
      • Overview
      • apm
      • dynamic_application_sizing
      • http
      • nomad
      • policy
      • policy_eval
      • source
      • strategy
      • target
      • telemetry
    • API
    • CLI
    • Policy
    • Telemetry
      • Overview
        • Overview
        • Datadog
        • Nomad API
        • Prometheus
        • Overview
        • Dynamic Application Sizing Average
        • Dynamic Application Sizing Max
        • Dynamic Application Sizing Percentile
        • Fixed Value
        • Pass-Through
        • Target Value
        • Threshold
        • Overview
        • Amazon Web Services Autoscaling Group
        • Azure Virtual Machine Scale Set
        • Dynamic Application Sizing
        • Google Cloud Engine Managed Instance Group
        • Nomad Task Group
      • Community
      • Overview
      • Checks
      • Node Selector Strategy
        • Overview
        • Base
        • APM
        • Strategy
        • Target
    • Overview
    • Operating Nomad Agents
    • Monitoring Nomad
    • Metrics Reference
    • Cluster Management
    • Transport Security
    • Access Control

    • Overview
    • Alternative to Kubernetes
    • Supplement to Kubernetes
  • Nomad Ecosystem
  • Nomad Partnerships
  • Who Uses Nomad
    • Overview
      • Overview
      • FAQ
  • FAQ
Type '/' to Search

»csi_plugin Stanza

Placementjob -> group -> task -> volume

The "csi_plugin" stanza allows the task to specify it provides a Container Storage Interface plugin to the cluster. Nomad will automatically register the plugin so that it can be used by other jobs to claim volumes.

csi_plugin {
  id        = "csi-hostpath"
  type      = "monolith"
  mount_dir = "/csi"
}
csi_plugin {
  id        = "csi-hostpath"
  type      = "monolith"
  mount_dir = "/csi"
}

»csi_plugin Parameters

  • id (string: <required>) - This is the ID for the plugin. Some plugins will require both controller and node plugin types (see below); you need to use the same ID for both so that Nomad knows they belong to the same plugin.

  • type (string: <required>) - One of node, controller, or monolith. Each plugin supports one or more types. Each Nomad client node where you want to mount a volume will need a node plugin instance. Some plugins will also require one or more controller plugin instances to communicate with the storage provider's APIs. Some plugins can serve as both controller and node at the same time, and these are called monolith plugins. Refer to your CSI plugin's documentation.

  • mount_dir (string: <required>) - The directory path inside the container where the plugin will expect a Unix domain socket for bidirectional communication with Nomad.

Note: Plugins running as node or monolith require root privileges (or CAP_SYS_ADMIN on Linux) to mount volumes on the host. With the Docker task driver, you can use the privileged = true configuration, but no other default task drivers currently have this option.

»Recommendations for Deploying CSI Plugins

CSI plugins run as Nomad tasks, but after mounting the volume are not in the data path for the volume. Tasks that mount volumes write and read directly to the volume via a bind-mount and there is no communication between the job and the CSI plugin. But when an allocation that mounts a volume stops, Nomad will need to communicate with the plugin on that allocation's node to unmount the volume. This has implications on how to deploy CSI plugins:

  • If you are stopping jobs on a node, you must stop tasks that claim volumes before stopping the node or monolith plugin for those volumes. If you use the node drain feature, plugin tasks will automatically be drained last.

  • Only the most recently-placed allocation for a given plugin ID and type (controller or node) will be used by any given client node. Run node plugins as system jobs and distribute controller plugins across client nodes using a constraint as shown below.

  • Some plugins will create volumes only in the same location as the plugin. For example, the AWS EBS plugin will create and mount volumes only within the same Availability Zone. You should configure your plugin task as recommended by the plugin's documentation to use the topology_request field in your volume specification.

»csi_plugin Examples

job "plugin-efs" {
  datacenters = ["dc1"]

  # you can run node plugins as service jobs as well, but running
  # as a system job ensures all nodes in the DC have a copy.
  type = "system"

  # only one plugin of a given type and ID should be deployed on
  # any given client node
  constraint {
    operator = "distinct_hosts"
    value = true
  }

  group "nodes" {
    task "plugin" {
      driver = "docker"

      config {
        image = "amazon/aws-efs-csi-driver:v.1.3.2"

        args = [
          "--endpoint=unix://csi/csi.sock",
          "--logtostderr",
          "--v=5",
        ]

        # all CSI node plugins will need to run as privileged tasks
        # so they can mount volumes to the host. controller plugins
        # do not need to be privileged.
        privileged = true
      }

      csi_plugin {
        id        = "aws-efs0"
        type      = "node"
        mount_dir = "/csi"  # this path /csi matches the --endpoint
                            # argument for the container
      }
    }
  }
}
job "plugin-efs" {
  datacenters = ["dc1"]

  # you can run node plugins as service jobs as well, but running
  # as a system job ensures all nodes in the DC have a copy.
  type = "system"

  # only one plugin of a given type and ID should be deployed on
  # any given client node
  constraint {
    operator = "distinct_hosts"
    value = true
  }

  group "nodes" {
    task "plugin" {
      driver = "docker"

      config {
        image = "amazon/aws-efs-csi-driver:v.1.3.2"

        args = [
          "--endpoint=unix://csi/csi.sock",
          "--logtostderr",
          "--v=5",
        ]

        # all CSI node plugins will need to run as privileged tasks
        # so they can mount volumes to the host. controller plugins
        # do not need to be privileged.
        privileged = true
      }

      csi_plugin {
        id        = "aws-efs0"
        type      = "node"
        mount_dir = "/csi"  # this path /csi matches the --endpoint
                            # argument for the container
      }
    }
  }
}
github logoEdit this page
DocsAPIResourcesPrivacySecurityPress KitConsent Manager