实际上我以前简单说明过，下边包含一个官方文档完整的，方便参数学习
官方提供的参考配置
- dremio-reference.conf
include classpath("dremio-reference-ext.conf")
paths: {
# the local path for dremio to store data.
local: "/tmp/dremio",
# 此配置对于21 之后版本是比较重要的，需要使用分布式存储
# the distributed path Dremio data including job results, downloads, uploads, etc
dist: "pdfs://"${paths.local}"/pdfs"
# location for catalog database (if master node)
db: ${paths.local}/db,
spilling: [${paths.local}/spill]
# storage area for the accelerator cache.
accelerator: ${paths.dist}/accelerator
# staging area for json and csv ui downloads
downloads: ${paths.dist}/downloads
# stores uploaded data associated with user home directories
uploads: ${paths.dist}/uploads
# stores data associated with the job results cache.
results: ${paths.dist}/results
# shared scratch space for creation of tables.
scratch: ${paths.dist}/scratch
# storage area for internal Iceberg tables.
metadata: ${paths.dist}/metadata
}
# dremio 服务定义的，包含了协调节点，执行节点，以及候选节点（会执行查询，但是不能进行元数据操作）
services: {node-tag: "",
coordinator: {enabled: true,
# Auto-upgrade Dremio at startup if needed
auto-upgrade: false,
master: {enabled: true,
# configure an embedded ZooKeeper server on the same node as master
embedded-zookeeper: {enabled: true,
port: 2181,
path: ${paths.local}/zk}
},
# web 配置
web: {enabled: true,
port: 9047,
ssl: {# If SSL for communication path between browsers (or REST clients) and Dremio should be enabled.
enabled: false,
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: true
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},
# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}},
auth: {type: "internal"
}
ui: {# Configuration for Intercom
intercom: {enabled: true
appid: "@dremio.ui.intercom.appid@"
}
}
tokens: {cache: {# number of tokens to store locally on this coordinator node
# (set to 0, if all requests should be made directly to the database of tokens)
size: 100
# time (in minutes) after which the token needs to be obtained from the token store
expiration_minutes: 5
}
}
},
client-endpoint: {port: 31010
},
scheduler: {threads: 24
},
command-pool: {releasable: true,
enabled: true,
size: 0 # 0 defaults to the machine's number of cores
},
},
executor: {enabled: true
# enable/disable local cache manager
# storage space for cache manager
# control max percentage of disk cache manager db instance and fs mount points can consume
cache: {enabled: true,
path: {db: ${paths.local},fs: [${services.executor.cache.path.db}]},
pctquota: {db: 70,
fs: [${services.executor.cache.pctquota.db}]},
ensurefreespace: {fs: [10]
}
}
},
fabric: {port: 45678,
memory: {reservation: 100M
}
ssl: {# If SSL for communication path between Dremio instances should be enabled.
enabled: false,
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: false,
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},
# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}}
},
nessie: {enabled: true,
remote-uri: "",
backend: {in-memory: false
},
kvstore: {commit-timeout-ms: 30000
}
},
flight: {enabled: true
port: 32010
# Authentication mode to configure the FlightServer Endpoint to use.
# There are two modes:
# - legacy.arrow.flight.auth
# - arrow.flight.auth2
# legacy.arrow.flight.auth is backwards compatible and will be deprecated in the future.
# arrow.flight.auth2 is the new and preferred Arrow Flight authentication method.
auth.mode: "arrow.flight.auth2"
use_session_service: true
ssl: {# If SSL for communication path between Flight clients and Dremio should be enabled.
enabled: false,
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: true
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},
# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}}
},
usersessions.ttl.minutes: 120,
conduit: {# If set to 0, a port is automatically allocated (typically in ephemeral range). Otherwise, the configured value
# is used.
port: 0
ssl: {# If SSL for communication path between Dremio instances should be enabled.
enabled: ${services.fabric.ssl.enabled},
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: ${services.fabric.ssl.auto-certificate.enabled},
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${services.fabric.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${services.fabric.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${services.fabric.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${services.fabric.ssl.keyPassword},
# TrustStore type
trustStoreType: ${services.fabric.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${services.fabric.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${services.fabric.ssl.trustStorePassword}}
}
# Set up kerberos credentials in server (applicable for both coordinator and executor)
kerberos: {principal: "",
keytab.file.path: ""
}
web-admin: {enabled: true,
# IP of hostname to which liveness HTTP server will be bound to. Default is loopback interface.
host: "127.0.0.1"
# Port, on which the daemon responds to liveness HTTP requests (0 == auto-allocated)
port: 0
}
jobs: {# if jobs server should be enabled in a coorinator
enabled: true
}
}
security: {# Password to access Dremio keystore
keystore-password: ""
}
provisioning: {yarn: {jvmoptions: "",
# list of jars to be added to the main container classpath
classpath: []
app {# list of jars to be added to the Dremio application classpath
classpath: []
# list of jars to be added to the Dremio application classpath (at the front)
classpath-prefix: []
}
watchdog: {# how long to wait for server reply before considering it failed
poll.timeout: 1000ms
# interval between two server polls
poll.interval: 10000ms
# how many consecutive failed attempts before killing server
missed.polls.before.kill: 6
# maximum attempts at killing server
max.kill.attempts: 10
# how long before reattempting killing server
kill.reattempt.interval: 1000ms
}
nodemanager: {# Enable certificate validation when connecting to Yarn Node-manager REST API
certificate-validation.enabled: true
}
enabled:false
}
}
# the zookeeper quorum for the cluster
zookeeper: "localhost:"${services.coordinator.master.embedded-zookeeper.port}zk.client.session.timeout: 90000
zk.client.retry.unlimited: true
zk.client.retry.limit: -1
zk.client.retry.initial_timeout_ms: -1
zk.client.connection_handle.enabled: false
zk.client.supervisor.interval_ms: 30000
zk.client.supervisor.read_timeout_ms: 10000
zk.client.supervisor.max_failures: 5
# 此处一些配置还是比较有用的，比如开启debug 模式
debug: {enabled: false,
autoPort: false,
prepopulate: false,
singleNode: false,
verboseAccessLog: false,
allowTestApis: false,
forceRemote: false,
useMemoryStorage: false,
addDefaultUser: false,
allowNewerKVStore: false,
# to enable remote debugging of the DremioDaemon running in YARN container
yarnremote.enabled: false
# UI Red Screen Of Death
rsod.enabled: false
# UI File A Bug option
bug.filing.enabled: false
# DX-9126: enable on-idle load shedding
task.on_idle_load_shed: true
# DX-23620: enable eager load shedding
task.eager_load_shed: true
# DX-9911: enable rescheduling task on unblock
task.reschedule_on_unblock: true
# Use election service to elect between multiple master candidates
# has to be set to false if multiple master candidates
master.election.disabled: false,
alloc.max_occupancy_percent: 75,
alloc.est_heap_buf_size_bytes: 800,
# Enable caching for distributed storage, it is turned off by default
dist.caching.enabled: false,
# Max percent of total available cache space to use when possible for distributed storage
dist.max.cache.space.percent: 100,
# Enable asynchronous access for distributed stores when possible
dist.async.enabled: true,
# Enable asynchronous access for uploaded data associated with user home directories
uploads.async.enabled: true,
# Enable asynchronous access for temporary support stores
support.async.enabled: true,
# Enable asynchronous access for the job results cache store
results.async.enabled: true,
# Enable asynchronous access for the create table scratch space
scratch.async.enabled: true,
# Enable asynchronous access for the json and csv download staging space
download.async.enabled: true,
# Enable asynchronous access for the logs store
logs.async.enabled: true,
# Enable asynchronous access for the metadata store
debug.metadata.async.enabled: true,
# NoOp cluster coordinator
noop.cluster.coordinator.enabled: false,
remote.datastore.rpc_timeout_seconds: 60,
dist.s3_file_status_check.enabled: true,
}
# Plan Cache configuration
dremio.plan.cache.timeout_minutes: 14400
dremio.plan.cache.max_entries: 10000
# Statistics Cache configuration
dremio.statistics.cache.timeout_minutes: 1
dremio.statistics.cache.max_entries: 500
# These system properties are listed here to allow substitution of system property values for DAC Web SSL properties
# listed in services.web.ssl section. Currently we consider only the system properties listed in this file for
# substitution.
javax.net.ssl {keyStoreType: "",
keyStore: "",
keyStorePassword: "",
keyPassword: "",
trustStoreType: "",
trustStore:"",
trustStorePassword: ""
}
registration.publish-host: ""
说明
以上一些配置还是比较重要的，比如调度，rpc 配置，以及不同执行角色的
