Skip to main content

Restate Server Configuration Reference

Default configuration​

💡Tip

Note that configuration defaults might change across server releases, if you want to make sure you use stable values, use an explicit configuration file an pass the path via --config-path=<PATH> as described above.

Not an exhaustive list

The following is the default configuration. It does not include all possible configuration options, since some can be conflicting. Take a look at the configuration reference below for a full list of options.

restate.toml
roles = [
    "worker",
    "admin",
    "metadata-server",
    "log-server",
    "http-ingress",
]
cluster-name = "localcluster"
auto-provision = true
advertised-address = "http://127.0.0.1:5122/"
default-num-partitions = 24
default-replication = 1
shutdown-timeout = "1m"
tracing-filter = "info"
log-filter = "warn,restate=info"
log-format = "pretty"
log-disable-ansi-codes = false
tokio-console-bind-address = "0.0.0.0:6669"
histogram-inactivity-timeout = "3m"
no-proxy = []
connect-timeout = "10s"
disable-prometheus = false
rocksdb-total-memory-size = "6.0 GiB"
rocksdb-total-memtables-ratio = 0.5
rocksdb-high-priority-bg-threads = 2
rocksdb-write-stall-threshold = "3s"
rocksdb-enable-stall-on-memory-limit = false
rocksdb-perf-level = "enable-count"
metadata-update-interval = "10s"
metadata-fetch-from-peer-timeout = "3s"
initialization-timeout = "5m"
disable-telemetry = false
gossip-tick-interval = "100ms"
gossip-failure-threshold = 10
gossip-num-peers = 2
gossip-fd-stability-threshold = 3
gossip-suspect-interval = "5s"
gossip-loneliness-threshold = 30
gossip-extras-exchange-frequency = 10
gossip-time-skew-threshold = "1s"

[metadata-client]
type = "replicated"
addresses = ["http://127.0.0.1:5122/"]
connect-timeout = "3s"
keep-alive-interval = "5s"
keep-alive-timeout = "5s"

[metadata-client.backoff-policy]
type = "exponential"
initial-interval = "100ms"
factor = 1.399999976158142
max-attempts = 10
max-interval = "1s"

[http-keep-alive-options]
interval = "40s"
timeout = "20s"

[network-error-retry-policy]
type = "exponential"
initial-interval = "10ms"
factor = 2.0
max-attempts = 15
max-interval = "5s"

[worker]
internal-queue-length = 1000
cleanup-interval = "1h"
max-command-batch-size = 32

[worker.storage]
rocksdb-disable-wal = true
rocksdb-memory-ratio = 0.49000000953674316

[worker.invoker]
inactivity-timeout = "1m"
abort-timeout = "1m"
message-size-warning = "10.0 MiB"
in-memory-queue-length-limit = 66049
concurrent-invocations-limit = 1000

[worker.invoker.retry-policy]
type = "exponential"
initial-interval = "50ms"
factor = 2.0
max-interval = "10s"

[worker.snapshots.object-store-retry-policy]
type = "exponential"
initial-interval = "100ms"
factor = 2.0
max-attempts = 10
max-interval = "10s"

[admin]
bind-address = "0.0.0.0:9070"
heartbeat-interval = "1s 500ms"
log-trim-check-interval = "1h"
disable-web-ui = false
disable-cluster-controller = false

[admin.query-engine]
memory-size = "4.0 GiB"

[ingress]
bind-address = "0.0.0.0:8080"
kafka-clusters = []

[bifrost]
default-provider = "replicated"
seal-retry-interval = "2s"
auto-recovery-interval = "15s"
append-retry-min-interval = "10ms"
append-retry-max-interval = "1s"
record-cache-memory-size = "250.0 MiB"
disable-auto-improvement = false

[bifrost.local]
rocksdb-disable-wal = false
rocksdb-memory-ratio = 0.5
rocksdb-disable-wal-fsync = false
writer-batch-commit-count = 5000
writer-batch-commit-duration = "0s"

[bifrost.replicated-loglet]
maximum-inflight-records = 1000
sequencer-inactivity-timeout = "15s"
log-server-rpc-timeout = "2s"
readahead-records = 100
readahead-trigger-ratio = 0.5

[bifrost.replicated-loglet.sequencer-retry-policy]
type = "exponential"
initial-interval = "250ms"
factor = 2.0
max-interval = "5s"

[bifrost.replicated-loglet.log-server-retry-policy]
type = "exponential"
initial-interval = "250ms"
factor = 2.0
max-attempts = 3
max-interval = "2s"

[bifrost.read-retry-policy]
type = "exponential"
initial-interval = "50ms"
factor = 2.0
max-attempts = 50
max-interval = "1s"

[metadata-server]
request-queue-length = 32
rocksdb-memory-ratio = 0.009999999776482582
rocksdb-disable-wal = false
raft-election-tick = 10
raft-heartbeat-tick = 2
raft-tick-interval = "100ms"
status-update-interval = "5s"
log-trim-threshold = 1000

[networking]
connect-timeout = "3s"
handshake-timeout = "3s"
http2-keep-alive-interval = "1s"
http2-keep-alive-timeout = "3s"
http2-adaptive-window = true
data-stream-window-size = "2.0 MiB"

[networking.connect-retry-policy]
type = "exponential"
initial-interval = "250ms"
factor = 2.0
max-attempts = 10
max-interval = "3s"

[log-server]
rocksdb-disable-wal = false
rocksdb-memory-ratio = 0.5
rocksdb-disable-wal-fsync = false
rocksdb-max-sub-compactions = 0
writer-batch-commit-count = 5000
incoming-network-queue-length = 1000

Configuration reference​

Restate configuration file
type : object

Configuration for Restate server.

  • worker Worker options
    type : object
    • internal-queue-length Internal queue for partition processor communication
      type : integer
      Possible values : >= 1
      Default value : 1000
    • num-timers-in-memory-limit Num timers in memory limit
      type : integer
      ❓ Nullable
      Possible values : >= 1
      Default value : 
      null

      The number of timers in memory limit is used to bound the amount of timers loaded in memory. If this limit is set, when exceeding it, the timers farther in the future will be spilled to disk.

    • cleanup-interval Cleanup interval
      type : string
      Default value : 1h

      In order to clean up completed invocations, that is invocations invoked with an idempotency id, or workflows, Restate periodically scans among the completed invocations to check whether they need to be removed or not. This interval sets the scan interval of the cleanup procedure. Default: 1 hour.

      Can be configured using the humantime format.

    • storage Storage options
      type : object
      • num-partitions-to-share-memory-budget integer OR null
        type : integer
        ❓ Nullable
        Possible values : >= 1

        How many partitions to divide memory across?

        By default this uses the value defined in default-num-partitions in the common section of the config.

      • rocksdb-memory-budget Non-zero human-readable bytes OR null
        anyOf
        type : string (non-zero human-bytes)
        Length : >= 1 character(s)
        Possible values : >= 1
        Pattern : ^\d+(\.\d+)? ?[KMG]B$

        Non-zero human-readable bytes

      • rocksdb-memory-ratio number
        type : number
        Default value : 0.49000000953674316

        The memory budget for rocksdb memtables as ratio

        This defines the total memory for rocksdb as a ratio of all memory available to memtables (See rocksdb-total-memtables-ratio in common). The budget is then divided evenly across partitions. The divisor is defined in num-partitions-to-share-memory-budget

      • persist-lsn-interval Persist LSN interval (deprecated) deprecated writeOnly
        type : string
        ❓ Nullable
        🚨 Deprecated
        🚨 write only

        This configuration option is deprecated and ignored in Restate >= 1.3.3.

      • persist-lsn-threshold Persist LSN threshold (deprecated) deprecated writeOnly
        type : integer
        ❓ Nullable
        🚨 Deprecated
        🚨 write only
        Possible values : >= 0

        This configuration option is deprecated and ignored in Restate >= 1.3.3.

      • rocksdb-disable-direct-io-for-reads Disable Direct IO for reads
        type : boolean
        ❓ Nullable

        Files will be opened in "direct I/O" mode which means that data r/w from the disk will not be cached or buffered. The hardware buffer of the devices may however still be used. Memory mapped files are not impacted by these parameters.

      • rocksdb-disable-direct-io-for-flush-and-compactions Disable Direct IO for flush and compactions
        type : boolean
        ❓ Nullable

        Use O_DIRECT for writes in background flush and compactions.

      • rocksdb-disable-wal Disable WAL
        type : boolean
        ❓ Nullable

        The default depends on the different rocksdb use-cases at Restate.

        Supports hot-reloading (Partial / Bifrost only)

      • rocksdb-disable-statistics boolean OR null
        type : boolean
        ❓ Nullable

        Disable rocksdb statistics collection

        Default: False (statistics enabled)

      • rocksdb-max-background-jobs RocksDB max background jobs (flushes and compactions)
        type : integer
        ❓ Nullable
        Possible values : >= 1

        Default: the number of CPU cores on this node.

      • rocksdb-compaction-readahead-size RocksDB compaction readahead size in bytes
        anyOf
        type : string (non-zero human-bytes)
        Length : >= 1 character(s)
        Possible values : >= 1
        Pattern : ^\d+(\.\d+)? ?[KMG]B$

        Non-zero human-readable bytes

      • rocksdb-statistics-level RocksDB statistics level
        anyOf
        oneOf
        type : string
        Possible values : 
        • disable-all

        Disable all metrics

      • rocksdb-log-level RocksDB log level
        anyOf
        type : string
        Possible values : 
        • debug
        • info
        • warn
        • error
        • fatal
        • header

        Verbosity of the LOG.

      • rocksdb-log-keep-file-num RocksDB log keep file num
        type : integer
        ❓ Nullable
        Possible values : >= 0
        Default value : 
        null

        Number of info LOG files to keep

        Default: 1

      • rocksdb-log-max-file-size RocksDB log max file size
        anyOf
        type : string (non-zero human-bytes)
        Length : >= 1 character(s)
        Possible values : >= 1
        Pattern : ^\d+(\.\d+)? ?[KMG]B$

        Non-zero human-readable bytes

      • rocksdb-block-size RocksDB block size
        anyOf
        type : string (non-zero human-bytes)
        Length : >= 1 character(s)
        Possible values : >= 1
        Pattern : ^\d+(\.\d+)? ?[KMG]B$

        Non-zero human-readable bytes

    • invoker Invoker options
      type : object
      • retry-policy Retry policy
        oneOf
        type : object

        No retry strategy.

        • type "none" required constant
          type : string
          Possible values : 
          • none
      • inactivity-timeout Inactivity timeout
        type : string
        Default value : 1m

        This timer guards against stalled service/handler invocations. Once it expires, Restate triggers a graceful termination by asking the service invocation to suspend (which preserves intermediate progress).

        The 'abort timeout' is used to abort the invocation, in case it doesn't react to the request to suspend.

        Can be configured using the humantime format.

      • abort-timeout Abort timeout
        type : string
        Default value : 1m

        This timer guards against stalled service/handler invocations that are supposed to terminate. The abort timeout is started after the 'inactivity timeout' has expired and the service/handler invocation has been asked to gracefully terminate. Once the timer expires, it will abort the service/handler invocation.

        This timer potentially interrupts user code. If the user code needs longer to gracefully terminate, then this value needs to be set accordingly.

        Can be configured using the humantime format.

      • message-size-warning Non-zero human-readable bytes
        type : string (non-zero human-bytes)
        Length : >= 1 character(s)
        Possible values : >= 1
        Pattern : ^\d+(\.\d+)? ?[KMG]B$

        Non-zero human-readable bytes

      • message-size-limit Message size limit
        anyOf
        type : string (non-zero human-bytes)
        Length : >= 1 character(s)
        Possible values : >= 1
        Pattern : ^\d+(\.\d+)? ?[KMG]B$

        Non-zero human-readable bytes

      • tmp-dir Temporary directory
        type : string
        ❓ Nullable
        Default value : 
        null

        Temporary directory to use for the invoker temporary files. If empty, the system temporary directory will be used instead.

      • in-memory-queue-length-limit Spill invocations to disk
        type : integer
        Possible values : >= 1
        Default value : 66049

        Defines the threshold after which queues invocations will spill to disk at the path defined in tmp-dir. In other words, this is the number of invocations that can be kept in memory before spilling to disk. This is a per-partition limit.

      • concurrent-invocations-limit Limit number of concurrent invocations from this node
        type : integer
        ❓ Nullable
        Possible values : >= 1
        Default value : 1000

        Number of concurrent invocations that can be processed by the invoker.

    • max-command-batch-size Maximum command batch size for partition processors
      type : integer
      Possible values : >= 1
      Default value : 32

      The maximum number of commands a partition processor will apply in a batch. The larger this value is, the higher the throughput and latency are.

    • snapshots Snapshot options.
      type : object

      Partition store snapshotting settings. At a minimum, set destination and snapshot-interval-num-records to enable snapshotting. For a complete example, see Snapshots.

      • destination Snapshot destination URL
        type : string
        ❓ Nullable
        Default value : 
        null

        Base URL for cluster snapshots. Supports s3:// and file:// protocol scheme. S3-compatible object stores must support ETag-based conditional writes.

        Default: None

      • snapshot-interval-num-records Automatic snapshot creation frequency
        type : integer
        ❓ Nullable
        Possible values : >= 1
        Default value : 
        null

        Number of log records that trigger a snapshot to be created.

        As snapshots are created asynchronously, the actual number of new records that will trigger a snapshot will vary. The counter for the subsequent snapshot begins from the LSN at which the previous snapshot export was initiated. Only leader Partition Processors will take snapshots for a given partition.

        This setting does not influence explicitly requested snapshots triggered using restatectl.

        Default: None - automatic snapshots are disabled

      • object-store-retry-policy Retry policy
        oneOf
        type : object

        No retry strategy.

        • type "none" required constant
          type : string
          Possible values : 
          • none
      • aws-profile AWS profile
        type : string
        ❓ Nullable

        The AWS configuration profile to use for S3 object store destinations. If you use named profiles in your AWS configuration, you can replace all the other settings with a single profile reference. See the [AWS documentation on profiles] (https://docs.aws.amazon.com/sdkref/latest/guide/file-format.html) for more.

      • aws-region AWS region
        type : string
        ❓ Nullable

        AWS region to use with S3 object store destinations. This may be inferred from the environment, for example the current region when running in EC2. Because of the request signing algorithm this must have a value. For Minio, you can generally set this to any string, such as us-east-1.

      • aws-access-key-id AWS access key
        type : string
        ❓ Nullable

        Username for Minio, or consult the service documentation for other S3-compatible stores.

      • aws-secret-access-key AWS secret key
        type : string
        ❓ Nullable

        Password for Minio, or consult the service documentation for other S3-compatible stores.

      • aws-session-token AWS session token
        type : string
        ❓ Nullable

        This is only needed with short-term STS session credentials.

      • aws-endpoint-url Object store API endpoint URL override
        type : string
        ❓ Nullable

        When you use Amazon S3, this is typically inferred from the region and there is no need to set it. With other object stores, you will have to provide an appropriate HTTP(S) endpoint. If not using HTTPS, also set aws-allow-http to true.

      • aws-allow-http Allow insecure HTTP
        type : boolean
        ❓ Nullable

        Allow plain HTTP to be used with the object store endpoint. Required when the endpoint URL that isn't using HTTPS.

  • admin Admin server options
    type : object
    • bind-address Endpoint address
      type : string
      Default value : 0.0.0.0:9070

      Address to bind for the Admin APIs.

    • advertised-admin-endpoint Advertised Admin endpoint
      type : string (uri)
      Default value : 
      null

      Optional advertised Admin API endpoint.

    • concurrent-api-requests-limit Concurrency limit
      type : integer
      ❓ Nullable
      Possible values : >= 1
      Default value : 
      null

      Concurrency limit for the Admin APIs. Default is unlimited.

    • query-engine Storage query engine options
      type : object
      • memory-size Non-zero human-readable bytes
        type : string (non-zero human-bytes)
        Length : >= 1 character(s)
        Possible values : >= 1
        Pattern : ^\d+(\.\d+)? ?[KMG]B$

        Non-zero human-readable bytes

      • tmp-dir Temp folder to use for spill
        type : string
        ❓ Nullable
        Default value : 
        null

        The path to spill to

      • query-parallelism Default query parallelism
        type : integer
        ❓ Nullable
        Possible values : >= 1
        Default value : 
        null

        The number of parallel partitions to use for a query execution

      • pgsql-bind-address Pgsql Bind address deprecated writeOnly
        type : string
        ❓ Nullable
        🚨 Deprecated
        🚨 write only

        The address to bind for the psql service.

    • heartbeat-interval Controller heartbeats
      type : string
      Default value : 1s 500ms

      Controls the interval at which cluster controller polls nodes of the cluster.

    • log-trim-check-interval Log trim check interval
      type : string
      Default value : 1h

      Controls the interval at which cluster controller tries to trim the logs. Log trimming can be disabled by setting it to "0s".

      Note that this is only the interval at which logs are checked, and does not guarantee that trim will be performed. The conditions for safely trim the log vary depending on the deployment. For single nodes, the log records must be durably persisted to disk. In distributed deployments, automatic trimming requires an external snapshot destination - see worker.snapshots for more.

    • disable-web-ui boolean
      type : boolean
      Default value : false

      Disable serving the Restate Web UI on the admin port. Default is false.

    • disable-cluster-controller boolean
      type : boolean
      Default value : false
  • ingress Ingress options
    type : object
    • bind-address Bind address
      type : string
      Default value : 0.0.0.0:8080

      The address to bind for the ingress.

    • concurrent-api-requests-limit Concurrency limit
      type : integer
      ❓ Nullable
      Possible values : >= 1
      Default value : 
      null

      Local concurrency limit to use to limit the amount of concurrent requests. If exceeded, the ingress will reply immediately with an appropriate status code. Default is unlimited.

    • kafka-clusters (Kafka cluster options,...)[]
      type : array
      • items[x] Kafka cluster options
        type : object

        Configuration options to connect to a Kafka cluster.

        • name string required
          type : string

          Cluster name (Used to identify subscriptions).

        • brokers Servers required
          type : array

          Initial list of brokers (host or host:port).

          • items[x] string
            type : string
      Default value : 
      []
    • advertised-ingress-endpoint Ingress endpoint
      type : string (uri)
      Default value : 
      null

      Ingress endpoint that the Web UI should use to interact with.

  • bifrost Bifrost options
    type : object
    • default-provider "local" XOR "replicated"
      oneOf
      type : string
      Possible values : 
      • local

      A local rocksdb-backed loglet.

    • local string
      type : string
      Default value : 
      {"rocksdb-disable-wal":false,"rocksdb-disable-wal-fsync":false,"rocksdb-log-keep-file-num":null,"rocksdb-log-level":null,"rocksdb-log-max-file-size":null,"rocksdb-memory-ratio":0.5,"writer-batch-commit-count":5000,"writer-batch-commit-duration":"0s"}

      Configuration of local loglet provider

    • replicated-loglet object
      type : object
      • maximum-inflight-records integer
        type : integer
        Possible values : >= 1
        Default value : 1000

        Maximum number of inflight records sequencer can accept

        Once this maximum is hit, sequencer will induce back pressure on clients. This controls the total number of records regardless of how many batches.

        Note that this will be increased to fit the biggest batch of records being enqueued.

      • sequencer-retry-policy Retry policy
        oneOf
        type : object

        No retry strategy.

        • type "none" required constant
          type : string
          Possible values : 
          • none
      • sequencer-inactivity-timeout string
        type : string
        Default value : 15s

        Sequencer inactivity timeout

        The sequencer is allowed to consider itself quiescent if it did not commit records for this period of time. It may use this to sends pre-emptive release/seal check requests to log-servers.

        The sequencer is also allowed to use this value as interval to send seal/release checks even if it's not quiescent.

      • log-server-rpc-timeout string
        type : string
        Default value : 2s

        Log Server RPC timeout

        Timeout waiting on log server response

      • log-server-retry-policy Retry policy
        oneOf
        type : object

        No retry strategy.

        • type "none" required constant
          type : string
          Possible values : 
          • none
      • readahead-records integer
        type : integer
        Possible values : >= 1
        Default value : 100

        Maximum number of records to prefetch from log servers

        The number of records bifrost will attempt to prefetch from replicated loglet's log-servers for every loglet reader (e.g. partition processor). Note that this mainly impacts readers that are not co-located with the loglet sequencer (i.e. partition processor followers).

      • readahead-trigger-ratio number
        type : number
        Default value : 0.5

        Trigger to prefetch more records

        When read-ahead is used (readahead-records), this value (percentage in float) will determine when readers should trigger a prefetch for another batch to fill up the buffer. For instance, if this value is 0.3, then bifrost will trigger a prefetch when 30% or more of the read-ahead slots become available (e.g. partition processor consumed records and freed up enough slots).

        The higher the value is, the longer bifrost will wait before it triggers the next fetch, potentially fetching more records as a result.

        To illustrate, if readahead-records is set to 100 and readahead-trigger-ratio is 1.0. Then bifrost will prefetch up to 100 records from log-servers and will not trigger the next prefetch unless the consumer consumes 100% of this buffer. This means that bifrost will read in batches but will not do while the consumer is still reading the previous batch.

        Value must be between 0 and 1. It will be clamped at 1.0.

    • read-retry-policy Retry policy
      oneOf
      type : object

      No retry strategy.

      • type "none" required constant
        type : string
        Possible values : 
        • none
    • seal-retry-interval Seal retry interval
      type : string
      Default value : 2s

      Interval to wait between retries of loglet seal failures

    • auto-recovery-interval Auto recovery threshold
      type : string
      Default value : 15s

      Time interval after which bifrost's auto-recovery mechanism will kick in. This is triggered in scenarios where the control plane took too long to complete loglet reconfigurations.

    • append-retry-min-interval Append retry minimum interval
      type : string
      Default value : 10ms

      Minimum retry duration used by the exponential backoff mechanism for bifrost appends.

    • append-retry-max-interval Append retry maximum interval
      type : string
      Default value : 1s

      Maximum retry duration used by the exponential backoff mechanism for bifrost appends.

    • record-cache-memory-size Human-readable bytes
      type : string (human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Human-readable bytes

    • disable-auto-improvement Disable Automatic Improvement
      type : string
      Default value : false

      When enabled, automatic improvement periodically checks with the loglet provider if the loglet configuration can be improved by performing a reconfiguration.

      This allows the log to pick up replication property changes, apply better placement of replicas, or for other reasons.

  • metadata-server Metadata store options
    type : object
    • request-queue-length integer
      type : integer
      Possible values : >= 1
      Default value : 32

      Limit number of in-flight requests

      Number of in-flight metadata store requests.

    • rocksdb-memory-budget Non-zero human-readable bytes OR null
      anyOf
      type : string (non-zero human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Non-zero human-readable bytes

    • rocksdb-memory-ratio number
      type : number
      Default value : 0.009999999776482582

      The memory budget for rocksdb memtables as ratio

      This defines the total memory for rocksdb as a ratio of all memory available to memtables (See rocksdb-total-memtables-ratio in common).

    • rocksdb-disable-direct-io-for-reads Disable Direct IO for reads
      type : boolean
      ❓ Nullable

      Files will be opened in "direct I/O" mode which means that data r/w from the disk will not be cached or buffered. The hardware buffer of the devices may however still be used. Memory mapped files are not impacted by these parameters.

    • rocksdb-disable-direct-io-for-flush-and-compactions Disable Direct IO for flush and compactions
      type : boolean
      ❓ Nullable

      Use O_DIRECT for writes in background flush and compactions.

    • rocksdb-disable-wal Disable WAL
      type : boolean
      ❓ Nullable

      The default depends on the different rocksdb use-cases at Restate.

      Supports hot-reloading (Partial / Bifrost only)

    • rocksdb-disable-statistics boolean OR null
      type : boolean
      ❓ Nullable

      Disable rocksdb statistics collection

      Default: False (statistics enabled)

    • rocksdb-max-background-jobs RocksDB max background jobs (flushes and compactions)
      type : integer
      ❓ Nullable
      Possible values : >= 1

      Default: the number of CPU cores on this node.

    • rocksdb-compaction-readahead-size RocksDB compaction readahead size in bytes
      anyOf
      type : string (non-zero human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Non-zero human-readable bytes

    • rocksdb-statistics-level RocksDB statistics level
      anyOf
      oneOf
      type : string
      Possible values : 
      • disable-all

      Disable all metrics

    • rocksdb-log-level RocksDB log level
      anyOf
      type : string
      Possible values : 
      • debug
      • info
      • warn
      • error
      • fatal
      • header

      Verbosity of the LOG.

    • rocksdb-log-keep-file-num RocksDB log keep file num
      type : integer
      ❓ Nullable
      Possible values : >= 0
      Default value : 
      null

      Number of info LOG files to keep

      Default: 1

    • rocksdb-log-max-file-size RocksDB log max file size
      anyOf
      type : string (non-zero human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Non-zero human-readable bytes

    • rocksdb-block-size RocksDB block size
      anyOf
      type : string (non-zero human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Non-zero human-readable bytes

    • raft-election-tick integer
      type : integer
      Possible values : >= 1
      Default value : 10

      The number of ticks before triggering an election

      The number of ticks before triggering an election. The value must be larger than raft_heartbeat_tick. It's recommended to set raft_election_tick = 10 * raft_heartbeat_tick. Decrease this value if you want to react faster to failed leaders. Note, decreasing this value too much can lead to cluster instabilities due to falsely detecting dead leaders.

    • raft-heartbeat-tick integer
      type : integer
      Possible values : >= 1
      Default value : 2

      The number of ticks before sending a heartbeat

      A leader sends heartbeat messages to maintain its leadership every heartbeat ticks. Decrease this value to send heartbeats more often.

    • raft-tick-interval string
      type : string
      Default value : 100ms

      The raft tick interval

      The interval at which the raft node will tick. Decrease this value in order to let the Raft node react more quickly to changes. Note, that every tick comes with an overhead. Moreover, the tick interval directly affects the election timeout. If the election timeout becomes too small, then this can cause cluster instabilities due to frequent leader changes.

    • status-update-interval string
      type : string
      Default value : 5s

      The status update interval

      The interval at which the raft node will update its status. Decrease this value in order to see more recent status updates.

    • log-trim-threshold The raft log trim threshold
      type : integer
      ❓ Nullable
      Possible values : >= 0
      Default value : 1000

      The threshold for trimming the raft log. The log will be trimmed if the number of apply entries exceeds this threshold. The default value is 1000.

  • networking Networking options
    type : object

    Common network configuration options for communicating with Restate cluster nodes. Note that similar keys are present in other config sections, such as in Service Client options.

    • connect-timeout Connect timeout
      type : string
      Default value : 3s

      TCP connection timeout for Restate cluster node-to-node network connections.

    • connect-retry-policy Retry policy
      oneOf
      type : object

      No retry strategy.

      • type "none" required constant
        type : string
        Possible values : 
        • none
    • handshake-timeout Handshake timeout
      type : string
      Default value : 3s

      Timeout for receiving a handshake response from Restate cluster peers.

    • http2-keep-alive-interval HTTP/2 Keep Alive Interval
      type : string
      Default value : 1s
    • http2-keep-alive-timeout HTTP/2 Keep Alive Timeout
      type : string
      Default value : 3s
    • http2-adaptive-window HTTP/2 Adaptive Window
      type : boolean
      Default value : true
    • data-stream-window-size Non-zero human-readable bytes
      type : string (non-zero human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Non-zero human-readable bytes

  • log-server Log server options
    type : object

    Configuration is only used on nodes running with log-server role.

    • rocksdb-memory-budget Non-zero human-readable bytes OR null
      anyOf
      type : string (non-zero human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Non-zero human-readable bytes

    • rocksdb-memory-ratio number
      type : number
      Default value : 0.5

      The memory budget for rocksdb memtables as ratio

      This defines the total memory for rocksdb as a ratio of all memory available to the log-server.

      (See rocksdb-total-memtables-ratio in common).

    • rocksdb-disable-wal-fsync boolean
      type : boolean
      Default value : false

      Disable fsync of WAL on every batch

    • rocksdb-max-sub-compactions integer
      type : integer
      Possible values : >= 0
      Default value : 0

      The maximum number of subcompactions to run in parallel.

      Setting this to 1 means no sub-compactions are allowed (i.e. only 1 thread will do the compaction).

      Default is 0 which maps to floor(number of CPU cores / 2)

    • rocksdb-max-wal-size Human-readable bytes
      type : string (human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Human-readable bytes

    • writer-batch-commit-count integer
      type : integer
      Possible values : >= 0
      Default value : 5000

      Trigger a commit when the batch size exceeds this threshold.

      Set to 0 or 1 to commit the write batch on every command.

    • incoming-network-queue-length integer
      type : integer
      Possible values : >= 1
      Default value : 1000

      The number of messages that can queue up on input network stream while request processor is busy.

    • rocksdb-disable-direct-io-for-reads Disable Direct IO for reads
      type : boolean
      ❓ Nullable

      Files will be opened in "direct I/O" mode which means that data r/w from the disk will not be cached or buffered. The hardware buffer of the devices may however still be used. Memory mapped files are not impacted by these parameters.

    • rocksdb-disable-direct-io-for-flush-and-compactions Disable Direct IO for flush and compactions
      type : boolean
      ❓ Nullable

      Use O_DIRECT for writes in background flush and compactions.

    • rocksdb-disable-wal Disable WAL
      type : boolean
      ❓ Nullable

      The default depends on the different rocksdb use-cases at Restate.

      Supports hot-reloading (Partial / Bifrost only)

    • rocksdb-disable-statistics boolean OR null
      type : boolean
      ❓ Nullable

      Disable rocksdb statistics collection

      Default: False (statistics enabled)

    • rocksdb-max-background-jobs RocksDB max background jobs (flushes and compactions)
      type : integer
      ❓ Nullable
      Possible values : >= 1

      Default: the number of CPU cores on this node.

    • rocksdb-compaction-readahead-size RocksDB compaction readahead size in bytes
      anyOf
      type : string (non-zero human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Non-zero human-readable bytes

    • rocksdb-statistics-level RocksDB statistics level
      anyOf
      oneOf
      type : string
      Possible values : 
      • disable-all

      Disable all metrics

    • rocksdb-log-level RocksDB log level
      anyOf
      type : string
      Possible values : 
      • debug
      • info
      • warn
      • error
      • fatal
      • header

      Verbosity of the LOG.

    • rocksdb-log-keep-file-num RocksDB log keep file num
      type : integer
      ❓ Nullable
      Possible values : >= 0
      Default value : 
      null

      Number of info LOG files to keep

      Default: 1

    • rocksdb-log-max-file-size RocksDB log max file size
      anyOf
      type : string (non-zero human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Non-zero human-readable bytes

    • rocksdb-block-size RocksDB block size
      anyOf
      type : string (non-zero human-bytes)
      Length : >= 1 character(s)
      Possible values : >= 1
      Pattern : ^\d+(\.\d+)? ?[KMG]B$

      Non-zero human-readable bytes

  • roles ("worker" XOR "admin" XOR "metadata-server" XOR "log-server" XOR "http-ingress",...)[]
    type : array

    Defines the roles which this Restate node should run, by default the node starts with all roles.

    • items[x] "worker" XOR "admin" XOR "metadata-server" XOR "log-server" XOR "http-ingress"
      oneOf
      type : string
      Possible values : 
      • worker

      A worker runs partition processor (journal, state, and drives invocations)

    Unique items : yes
    Default value : 
    ["worker","admin","metadata-server","log-server","http-ingress"]
  • node-name Node Name
    type : string
    ❓ Nullable
    Default value : 
    null

    Unique name for this node in the cluster. The node must not change unless it's started with empty local store. It defaults to the node's hostname.

  • location Node Location
    type : string

    [PREVIEW FEATURE] Setting the location allows Restate to form a tree-like cluster topology. The value is written in the format of "<region>[.zone]" to assign this node to a specific region, or to a zone within a region.

    The value of region and zone is arbitrary but whitespace and . are disallowed.

    NOTE: It's strongly recommended to not change the node's location string after its initial registration. Changing the location may result in data loss or data inconsistency if log-server is enabled on this node.

    When this value is not set, the node is considered to be in the default location. The default location means that the node is not assigned to any specific region or zone.

    Examples - us-west -- the node is in the us-west region. - us-west.a1 -- the node is in the us-west region and in the a1 zone. - `` -- [default] the node is in the default location

  • force-node-id integer OR null
    type : integer
    ❓ Nullable
    Possible values : >= 0
    Default value : 
    null

    If set, the node insists on acquiring this node ID.

  • cluster-name Cluster name
    type : string
    Default value : localcluster

    A unique identifier for the cluster. All nodes in the same cluster should have the same.

  • auto-provision Auto cluster provisioning
    type : boolean
    Default value : true

    If true, then this node is allowed to automatically provision as a new cluster. This node must have an admin role and a new nodes configuration will be created that includes this node.

    auto-provision is allowed by default in development mode and is disabled if restate-server runs with --production flag to prevent cluster nodes from forming their own clusters, rather than forming a single cluster.

    Use restatectl to provision the cluster/node if automatic provisioning is disabled.

    This can also be explicitly disabled by setting this value to false.

    Default: true

  • base-dir string OR null
    type : string
    ❓ Nullable
    Default value : 
    null

    The working directory which this Restate node should use for relative paths. The default is restate-data under the current working directory.

  • metadata-client Metadata client options
    type : object

    The metadata client type to store metadata

    • connect-timeout Connect timeout
      type : string
      Default value : 3s

      TCP connection timeout for connecting to the metadata store.

    • keep-alive-interval Metadata Store Keep Alive Interval
      type : string
      Default value : 5s
    • keep-alive-timeout Metadata Store Keep Alive Timeout
      type : string
      Default value : 5s
    • backoff-policy Retry policy
      oneOf
      type : object

      No retry strategy.

      • type "none" required constant
        type : string
        Possible values : 
        • none
    oneOf
    type : object

    Store metadata on the replicated metadata store that runs on nodes with the metadata-server role.

    • type "replicated" required constant
      type : string
      Possible values : 
      • replicated
    • addresses Restate metadata server address list required
      type : array
      • items[x] string
        type : string
  • bind-address string
    type : string

    Address to bind for the Node server. Derived from the advertised address, defaulting to 0.0.0.0:$PORT (where the port will be inferred from the URL scheme).

  • advertised-address string
    type : string
    Default value : http://127.0.0.1:5122/

    Address that other nodes will use to connect to this node. Default is http://127.0.0.1:5122/

  • default-num-partitions Partitions
    type : integer
    Possible values : >= 0
    Default value : 24

    Number of partitions that will be provisioned during initial cluster provisioning. partitions are the logical shards used to process messages.

    Cannot be higher than 65535 (You should almost never need as many partitions anyway)

    NOTE 1: This config entry only impacts the initial number of partitions, the value of this entry is ignored for provisioned nodes/clusters.

    NOTE 2: This will be renamed to default-num-partitions by default as of v1.3+

    Default: 24

  • default-replication Default replication factor
    type : string
    Default value : 1

    Configures the global default replication factor to be used by the the system.

    Note that this value only impacts the cluster initial provisioning and will not be respected after the cluster has been provisioned.

    To update existing clusters use the restatectl utility.

  • shutdown-timeout Shutdown grace timeout
    type : string
    Default value : 1m

    This timeout is used when shutting down the various Restate components to drain all the internal queues.

    Can be configured using the humantime format.

  • default-thread-pool-size Default async runtime thread pool
    type : integer
    ❓ Nullable
    Possible values : >= 0
    Default value : 
    null

    Size of the default thread pool used to perform internal tasks. If not set, it defaults to the number of CPU cores.

  • log-filter Logging Filter
    type : string
    Default value : warn,restate=info

    Log filter configuration. Can be overridden by the RUST_LOG environment variable. Check the RUST_LOG documentation for more details how to configure it.

  • log-format Log format
    oneOf
    type : string
    Possible values : 
    • pretty

    Enables verbose logging. Not recommended in production.

  • log-disable-ansi-codes Disable ANSI in log output
    type : boolean
    Default value : false

    Disable ANSI terminal codes for logs. This is useful when the log collector doesn't support processing ANSI terminal codes.

  • tokio-console-bind-address string
    type : string

    Address to bind for the tokio-console tracing subscriber. If unset and restate-server is built with tokio-console support, it'll listen on 0.0.0.0:6669.

  • histogram-inactivity-timeout string OR null
    type : string
    ❓ Nullable
    Default value : 3m

    Timeout for idle histograms.

    The duration after which a histogram is considered idle and will be removed from metric responses to save memory. This value should be configured higher than the scrape interval of the telemetry collection system (e.g. Prometheus).

  • disable-prometheus boolean
    type : boolean
    Default value : false

    Disable prometheus metric recording and reporting. Default is false.

  • storage-high-priority-bg-threads integer OR null
    type : integer
    ❓ Nullable
    Possible values : >= 1

    Storage high priority thread pool

    This configures the restate-managed storage thread pool for performing high-priority or latency-sensitive storage tasks when the IO operation cannot be performed on in-memory caches.

  • storage-low-priority-bg-threads integer OR null
    type : integer
    ❓ Nullable
    Possible values : >= 1

    Storage low priority thread pool

    This configures the restate-managed storage thread pool for performing low-priority or latency-insensitive storage tasks.

  • rocksdb-total-memory-size Non-zero human-readable bytes
    type : string (non-zero human-bytes)
    Length : >= 1 character(s)
    Possible values : >= 1
    Pattern : ^\d+(\.\d+)? ?[KMG]B$

    Non-zero human-readable bytes

  • rocksdb-total-memtables-ratio Rocksdb total memtable size ratio
    type : number
    Default value : 0.5

    The memory size used across all memtables (ratio between 0 to 1.0). This limits how much memory memtables can eat up from the value in rocksdb-total-memory-limit. When set to 0, memtables can take all available memory up to the value specified in rocksdb-total-memory-limit. This value will be sanitized to 1.0 if outside the valid bounds.

  • rocksdb-bg-threads Rocksdb Background Threads
    type : integer
    ❓ Nullable
    Possible values : >= 1

    The number of threads to reserve to Rocksdb background tasks. Defaults to the number of cores on the machine.

  • rocksdb-high-priority-bg-threads Rocksdb High Priority Background Threads
    type : integer
    Possible values : >= 1
    Default value : 2

    The number of threads to reserve to high priority Rocksdb background tasks.

  • rocksdb-write-stall-threshold Rocksdb stall detection threshold
    type : string
    Default value : 3s

    This defines the duration after which a write is to be considered in "stall" state. For every write that meets this threshold, the system will increment the restate.rocksdb_stall_flare gauge, if the write is unstalled, the guage will be updated accordingly.

  • rocksdb-enable-stall-on-memory-limit Allow rocksdb writes to stall if memory limit is reached
    type : boolean
    Default value : false

    Note if automatic memory budgeting is enabled, it should be safe to allow rocksdb to stall if it hits the limit. However, if rocksdb stall kicked in, it's unlikely that the system will recover from this without intervention.

  • rocksdb-perf-level "disable" XOR "enable-count" XOR "enable-time-except-for-mutex" XOR "enable-time-and-c-p-u-time-except-for-mutex" XOR "enable-time"
    oneOf
    type : string
    Possible values : 
    • disable

    Disable perf stats

  • metadata-update-interval Metadata update interval
    type : string
    Default value : 10s

    The idle time after which the node will check for metadata updates from metadata store. This helps the node detect if it has been operating with stale metadata for extended period of time, primarily because it didn't interact with other peers in the cluster during that period.

  • metadata-fetch-from-peer-timeout Timeout for metadata peer-to-peer fetching
    type : string
    Default value : 3s

    When a node detects that a new metadata version exists, it'll attempt to fetch it from its peers. After this timeout duration has passed, the node will attempt to fetch the metadata from metadata store as well. This is to ensure that the nodes converge quickly while reducing the load on the metadata store.

  • network-error-retry-policy Retry policy
    oneOf
    type : object

    No retry strategy.

    • type "none" required constant
      type : string
      Possible values : 
      • none
  • initialization-timeout Initialization timeout
    type : string
    Default value : 5m

    The timeout until the node gives up joining a cluster and initializing itself.

  • disable-telemetry Disable telemetry
    type : boolean
    Default value : false

    Restate uses Scarf to collect anonymous usage data to help us understand how the software is being used. You can set this flag to true to disable this collection. It can also be set with the environment variable DO_NOT_TRACK=1.

  • tracing-endpoint Tracing Endpoint
    type : string
    ❓ Nullable

    This is a shortcut to set both [Self::tracing_runtime_endpoint], and [Self::tracing_services_endpoint].

    Specify the tracing endpoint to send runtime traces to. Traces will be exported using OTLP gRPC through opentelemetry_otlp.

    To configure the sampling, please refer to the opentelemetry autoconfigure docs.

  • tracing-runtime-endpoint Runtime Tracing Endpoint
    type : string
    ❓ Nullable

    Overrides [Self::tracing_endpoint] for runtime traces

    Specify the tracing endpoint to send runtime traces to. Traces will be exported using OTLP gRPC through opentelemetry_otlp.

    To configure the sampling, please refer to the opentelemetry autoconfigure docs.

  • tracing-services-endpoint Services Tracing Endpoint
    type : string
    ❓ Nullable

    Overrides [Self::tracing_endpoint] for services traces

    Specify the tracing endpoint to send services traces to. Traces will be exported using OTLP gRPC through opentelemetry_otlp.

    To configure the sampling, please refer to the opentelemetry autoconfigure docs.

  • tracing-json-path Distributed Tracing JSON Export Path
    type : string
    ❓ Nullable

    If set, an exporter will be configured to write traces to files using the Jaeger JSON format. Each trace file will start with the trace prefix.

    If unset, no traces will be written to file.

    It can be used to export traces in a structured format without configuring a Jaeger agent.

    To inspect the traces, open the Jaeger UI and use the Upload JSON feature to load and inspect them.

  • tracing-filter Tracing Filter required
    type : string

    Distributed tracing exporter filter. Check the RUST_LOG documentation for more details how to configure it.

  • tracing-headers Additional tracing headers
    type : object

    Specify additional headers you want the system to send to the tracing endpoint (e.g. authentication headers).

    • property name* string
      type : string
  • request-identity-private-key-pem-file Request identity private key PEM file
    type : string
    ❓ Nullable
    Default value : 
    null

    A path to a file, such as "/var/secrets/key.pem", which contains exactly one ed25519 private key in PEM format. Such a file can be generated with openssl genpkey -algorithm ed25519. If provided, this key will be used to attach JWTs to requests from this client which SDKs may optionally verify, proving that the caller is a particular Restate instance.

    This file is currently only read on client creation, but this may change in future. Parsed public keys will be logged at INFO level in the same format that SDKs expect.

  • http-keep-alive-options HTTP/2 Keep alive options
    type : object

    Configuration for the HTTP/2 keep-alive mechanism, using PING frames.

    Please note: most gateways don't propagate the HTTP/2 keep-alive between downstream and upstream hosts. In those environments, you need to make sure the gateway can detect a broken connection to the upstream deployment(s).

    • interval HTTP/2 Keep-alive interval
      type : string
      Default value : 40s

      Sets an interval for HTTP/2 PING frames should be sent to keep a connection alive.

      You should set this timeout with a value lower than the abort_timeout.

    • timeout Timeout
      type : string
      Default value : 20s

      Sets a timeout for receiving an acknowledgement of the keep-alive ping.

      If the ping is not acknowledged within the timeout, the connection will be closed.

  • http-proxy Proxy URI
    type : string
    ❓ Nullable
    Default value : 
    null

    A URI, such as http://127.0.0.1:10001, of a server to which all invocations should be sent, with the Host header set to the deployment URI. HTTPS proxy URIs are supported, but only HTTP endpoint traffic will be proxied currently. Can be overridden by the HTTP_PROXY environment variable.

  • no-proxy No proxy
    type : array

    HTTP authorities eg localhost, restate.dev, 127.0.0.1 that should not be proxied by the http_proxy. Ports are ignored. Subdomains are also matched. An entry “*” matches all hostnames. Can be overridden by the NO_PROXY environment variable, which supports comma separated values.

    • items[x] string
      type : string
    Default value : 
    []
  • connect-timeout Connect timeout
    type : string
    Default value : 10s

    How long to wait for a TCP connection to be established before considering it a failed attempt.

  • initial-max-send-streams Initial Max Send Streams
    type : integer
    ❓ Nullable
    Possible values : >= 0
    Default value : 
    null

    Sets the initial maximum of locally initiated (send) streams.

    This value will be overwritten by the value included in the initial SETTINGS frame received from the peer as part of a [connection preface].

    Default: None

    NOTE: Setting this value to None (default) users the default recommended value from HTTP2 specs

  • aws-profile AWS Profile
    type : string
    ❓ Nullable
    Default value : 
    null

    Name of the AWS profile to select. Defaults to 'AWS_PROFILE' env var, or otherwise the default profile.

  • aws-assume-role-external-id AssumeRole external ID
    type : string
    ❓ Nullable
    Default value : 
    null

    An external ID to apply to any AssumeRole operations taken by this client. https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html Can be overridden by the AWS_EXTERNAL_ID environment variable.

  • rocksdb-disable-direct-io-for-reads Disable Direct IO for reads
    type : boolean
    ❓ Nullable

    Files will be opened in "direct I/O" mode which means that data r/w from the disk will not be cached or buffered. The hardware buffer of the devices may however still be used. Memory mapped files are not impacted by these parameters.

  • rocksdb-disable-direct-io-for-flush-and-compactions Disable Direct IO for flush and compactions
    type : boolean
    ❓ Nullable

    Use O_DIRECT for writes in background flush and compactions.

  • rocksdb-disable-wal Disable WAL
    type : boolean
    ❓ Nullable

    The default depends on the different rocksdb use-cases at Restate.

    Supports hot-reloading (Partial / Bifrost only)

  • rocksdb-disable-statistics boolean OR null
    type : boolean
    ❓ Nullable

    Disable rocksdb statistics collection

    Default: False (statistics enabled)

  • rocksdb-max-background-jobs RocksDB max background jobs (flushes and compactions)
    type : integer
    ❓ Nullable
    Possible values : >= 1

    Default: the number of CPU cores on this node.

  • rocksdb-compaction-readahead-size RocksDB compaction readahead size in bytes
    anyOf
    type : string (non-zero human-bytes)
    Length : >= 1 character(s)
    Possible values : >= 1
    Pattern : ^\d+(\.\d+)? ?[KMG]B$

    Non-zero human-readable bytes

  • rocksdb-statistics-level RocksDB statistics level
    anyOf
    oneOf
    type : string
    Possible values : 
    • disable-all

    Disable all metrics

  • rocksdb-log-level RocksDB log level
    anyOf
    type : string
    Possible values : 
    • debug
    • info
    • warn
    • error
    • fatal
    • header

    Verbosity of the LOG.

  • rocksdb-log-keep-file-num RocksDB log keep file num
    type : integer
    ❓ Nullable
    Possible values : >= 0
    Default value : 
    null

    Number of info LOG files to keep

    Default: 1

  • rocksdb-log-max-file-size RocksDB log max file size
    anyOf
    type : string (non-zero human-bytes)
    Length : >= 1 character(s)
    Possible values : >= 1
    Pattern : ^\d+(\.\d+)? ?[KMG]B$

    Non-zero human-readable bytes

  • rocksdb-block-size RocksDB block size
    anyOf
    type : string (non-zero human-bytes)
    Length : >= 1 character(s)
    Possible values : >= 1
    Pattern : ^\d+(\.\d+)? ?[KMG]B$

    Non-zero human-readable bytes

  • gossip-tick-interval Gossip tick interval
    type : string
    Default value : 100ms

    The interval at which the failure detector will tick. Decrease this value for faster reaction to node failures. Note, that every tick comes with an overhead.

  • gossip-failure-threshold Gossip failure threshold
    type : integer
    Possible values : >= 1
    Default value : 10

    Specifies how many gossip intervals of inactivity need to pass before considering a node as dead.

  • gossip-num-peers Number of peers to gossip
    type : integer
    Possible values : >= 1
    Default value : 2

    On every gossip interval, how many peers each node attempts to gossip with. The default is optimized for small clusters (<5). On larger clusters, if gossip overhead is noticeable, consider reducing this value to 1.

  • gossip-fd-stability-threshold Gossips before failure detector is stable
    type : integer
    Possible values : >= 1
    Default value : 3
  • gossip-suspect-interval Suspect duration
    type : string
    Default value : 5s

    How long to keep a node in a transient state (suspect) before marking it as available. Larger values mean that the cluster is less prone to flaky nodes but extends the time it takes for a node to participate.

    A node becomes a suspect if it has been previously marked as dead for a given generation number. If the node incremented its generation number, it will not be impacted by this threshold.

  • gossip-loneliness-threshold Gossip loneliness threshold
    type : integer
    Possible values : >= 1
    Default value : 30

    How many intervals need to pass without receiving any gossip messages before considering this node as potentially isolated/dead. This threshold is used in the case where the node can still send gossip messages but did not receive any. This can rarely happen in asymmetric network partitions.

    In this case, the node will advertise itself as dead in the gossip messages it sends out.

    Note: this threshold does not apply to a cluster that's configured with a single node.

  • gossip-extras-exchange-frequency Gossip extras exchange frequency
    type : integer
    Possible values : >= 1
    Default value : 10

    In addition to basic health/liveness information, the gossip protocol is used to exchange extra information about the roles hosted by this node. For instance, which partitions are currently running, their configuration versions, and the durable LSN of the corresponding partition databases. This information is sent every Nth gossip message. This setting controls the frequency of this exchange. For instance, 10 means that every 10th gossip message will contain the extra information about.

  • gossip-time-skew-threshold Gossips time skew threshold
    type : string
    Default value : 1s

    The time skew is the maximum acceptable time difference between the local node and the time reported by peers via gossip messages. The time skew is also used to ignore gossip messages that are too old.