Note:
This topic has been translated from a Chinese forum by GPT and might contain errors.Original topic: TIkv无法正常启动
|
username: 小码农-小牛马
【TiDB Usage Environment】Test
【TiDB Version】Release Version: 6.1.0
【Reproduction Path】What operations were performed when the issue occurred: Followed the k8s process for normal deployment
【Encountered Issue: Problem Phenomenon and Impact】TIkv won’t start
【Resource Configuration】Nfs as the k8s cluster’s StorageClass, disk is ext4, deployment process according to the official documentation: 在 Kubernetes 上快速上手 TiDB | PingCAP 文档中心
【Attachments: Screenshots/Logs/Monitoring】
[2022/11/28 02:35:21.408 +00:00] [INFO] [server.rs:369] [“connect to PD cluster”] [cluster_id=7170886967293250264]
[2022/11/28 02:35:21.408 +00:00] [INFO] [config.rs:2041] [“readpool.storage.use-unified-pool is not set, set to true by default”]
[2022/11/28 02:35:21.408 +00:00] [INFO] [config.rs:2064] [“readpool.coprocessor.use-unified-pool is not set, set to true by default”]
[2022/11/28 02:35:21.419 +00:00] [WARN] [config.rs:2977] [“memory_usage_limit:ReadableSize(6492433066) > recommended:ReadableSize(6298492416), maybe page cache isn’t enough”]
[2022/11/28 02:35:21.469 +00:00] [INFO] [server.rs:1541] [“beginning system configuration check”]
[2022/11/28 02:35:21.470 +00:00] [INFO] [config.rs:891] [“data dir”] [mount_fs=“FsInfo { tp: "nfs4", opts: "rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.2.155,local_lock=none,addr=192.168.2.152", mnt_dir: "/var/lib/tikv", fsname: "192.168.2.152:/nfs/share/tidb-cluster-tikv-basic-tikv-0-pvc-5572c5c9-772c-4ba0-84ea-39fb4a18af16" }”] [data_path=/var/lib/tikv]
[2022/11/28 02:35:21.471 +00:00] [WARN] [server.rs:1557] [“check: rocksdb-data-dir”] [err=“config fs: data-dir.rotation.get: "192.168.2.152:/nfs/share/tidb-cluster-tikv-basic-tikv-0-pvc-5572c5c9-772c-4ba0-84ea-39fb4a18af16" no device find in block”] [path=/var/lib/tikv]
[2022/11/28 02:35:21.472 +00:00] [INFO] [config.rs:891] [“data dir”] [mount_fs=“FsInfo { tp: "nfs4", opts: "rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=192.168.2.155,local_lock=none,addr=192.168.2.152", mnt_dir: "/var/lib/tikv", fsname: "192.168.2.152:/nfs/share/tidb-cluster-tikv-basic-tikv-0-pvc-5572c5c9-772c-4ba0-84ea-39fb4a18af16" }”] [data_path=/var/lib/tikv/raft]
[2022/11/28 02:35:21.472 +00:00] [WARN] [server.rs:1565] [“check: raftdb-path”] [err=“config fs: data-dir.rotation.get: "192.168.2.152:/nfs/share/tidb-cluster-tikv-basic-tikv-0-pvc-5572c5c9-772c-4ba0-84ea-39fb4a18af16" no device find in block”] [path=/var/lib/tikv/raft]
[2022/11/28 02:35:21.475 +00:00] [INFO] [server.rs:338] [“using config”] [config="{"log-level":"info","log-file":"","log-format":"text","log-rotation-timespan":"0s","log-rotation-size":"300MiB","slow-log-file":"","slow-log-threshold":"1s","panic-when-unexpected-key-or-data":false,"abort-on-panic":false,"memory-usage-limit":6492433066,"memory-usage-high-water":0.9,"log":{"level":"info","format":"text","enable-timestamp":true,"file":{"filename":"","max-size":300,"max-days":0,"max-backups":0}},"quota":{"foreground-cpu-time":0,"foreground-write-bandwidth":"0KiB","foreground-read-bandwidth":"0KiB","max-delay-duration":"500ms"},"readpool":{"unified":{"min-thread-count":1,"max-thread-count":4,"stack-size":"10MiB","max-tasks-per-worker":2000},"storage":{"use-unified-pool":true,"high-concurrency":4,"normal-concurrency":4,"low-concurrency":4,"max-tasks-per-worker-high":2000,"max-tasks-per-worker-normal":2000,"max-tasks-per-worker-low":2000,"stack-size":"10MiB"},"coprocessor":{"use-unified-pool":true,"high-concurrency":3,"normal-concurrency":3,"low-concurrency":3,"max-tasks-per-worker-high":2000,"max-tasks-per-worker-normal":2000,"max-tasks-per-worker-low":2000,"stack-size":"10MiB"}},"server":{"addr":"0.0.0.0:20160","advertise-addr":"basic-tikv-0.basic-tikv-peer.tidb-cluster.svc:20160","status-addr":"0.0.0.0:20180","advertise-status-addr":"basic-tikv-0.basic-tikv-peer.tidb-cluster.svc:20180","status-thread-pool-size":1,"max-grpc-send-msg-len":10485760,"raft-client-grpc-send-msg-buffer":524288,"raft-client-queue-size":8192,"raft-msg-max-batch-size":128,"grpc-compression-type":"none","grpc-concurrency":5,"grpc-concurrent-stream":1024,"grpc-raft-conn-num":1,"grpc-memory-pool-quota":9223372036854775807,"grpc-stream-initial-window-size":"2MiB","grpc-keepalive-time":"10s","grpc-keepalive-timeout":"3s","concurrent-send-snap-limit":32,"concurrent-recv-snap-limit":32,"end-point-recursion-limit":1000,"end-point-stream-channel-size":8,"end-point-batch-row-limit":64,"end-point-stream-batch-row-limit":128,"end-point-enable-batch-if-possible":true,"end-point-request-max-handle-duration":"1m","end-point-max-concurrency":4,"end-point-perf-level":0,"snap-max-write-bytes-per-sec":"100MiB","snap-max-total-size":"0KiB","stats-concurrency":1,"heavy-load-threshold":75,"heavy-load-wait-duration":null,"enable-request-batch":true,"background-thread-count":2,"end-point-slow-log-threshold":"1s","forward-max-connections-per-address":4,"reject-messages-on-memory-ratio":0.2,"labels":{}},"storage":{"data-dir":"/var/lib/tikv","gc-ratio-threshold":1.1,"max-key-size":8192,"scheduler-concurrency":524288,"scheduler-worker-pool-size":4,"scheduler-pending-write-threshold":"100MiB","reserve-space":"0KiB","enable-async-apply-prewrite":false,"api-version":1,"enable-ttl":false,"background-error-recovery-window":"1h","ttl-check-poll-interval":"12h","flow-control":{"enable":true,"soft-pending-compaction-bytes-limit":"192GiB","hard-pending-compaction-bytes-limit":"1TiB","memtables-threshold":5,"l0-files-threshold":20},"block-cache":{"shared":true,"capacity":"3715MiB","num-shard-bits":6,"strict-capacity-limit":false,"high-pri-pool-ratio":0.8,"memory-allocator":"nodump"},"io-rate-limit":{"max-bytes-per-sec":"0KiB","mode":"write-only","strict":false,"foreground-read-priority":"high","foreground-write-priority":"high","flush-priority":"high","level-zero-compaction-priority":"medium","compaction-priority":"low","replication-priority":"high","load-balance-priority":"high","gc-priority":"high","import-priority":"medium","export-priority":"medium","other-priority":"high"}},"pd":{"endpoints":["http://basic-pd:2379"],"retry-interval":"300ms","retry-max-count":9223372036854775807,"retry-log-every":10,"update-interval":"10m","enable-forwarding":false},"metric":{"job":"tikv"},"raftstore":{"prevote":true,"raftdb-path":"/var/lib/tikv/raft","capacity":"0KiB","raft-base-tick-interval":"1s","raft-heartbeat-ticks":2,"raft-election-timeout-ticks":10,"raft-min-election-timeout-ticks":10,"raft-max-election-timeout-ticks":20,"raft-max-size-per-msg":"1MiB","raft-max-inflight-msgs":256,"raft-entry-max-size":"8MiB","raft-log-compact-sync-interval":"2s","raft-log-gc-tick-interval":"3s","raft-log-gc-threshold":50,"raft-log-gc-count-limit":73728,"raft-log-gc-size-limit":"72MiB","raft-log-reserve-max-ticks":6,"raft-engine-purge-interval":"10s","raft-entry-cache-life-time":"30s","split-region-check-tick-interval":"10s","region-split-check-diff":"6MiB","region-compact-check-interval":"5m","region-compact-check-step":100,"region-compact-min-tombstones":10000,"region-compact-tombstones-percent":30,"pd-heartbeat-tick-interval":"1m","pd-store-heartbeat-tick-interval":"10s","snap-mgr-gc-tick-interval":"1m","snap-gc-timeout":"4h","lock-cf-compact-interval":"10m","lock-cf-compact-bytes-threshold":"256MiB","notify-capacity":40960,"messages-per-tick":4096,"max-peer-down-duration":"10m","max-leader-missing-duration":"2h","abnormal-leader-missing-duration":"10m","peer-stale-state-check-interval":"5m","leader-transfer-max-log-lag":128,"snap-apply-batch-size":"10MiB","consistency-check-interval":"0s","report-region-flow-interval":"1m","raft-store-max-leader-lease":"9s","check-leader-lease-interval":"2s250ms","renew-leader-lease-advance-duration":"2s250ms","right-derive-when-split":true,"merge-max-log-gap":10,"merge-check-tick-interval":"2s","use-delete-range":false,"snap-generator-pool-size":2,"cleanup-import-sst-interval":"10m","local-read-batch-size":1024,"apply-max-batch-size":256,"apply-pool-size":2,"apply-reschedule-duration":"5s","apply-low-priority-pool-size":1,"store-max-batch-size":256,"store-pool-size":2,"store-reschedule-duration":"5s","store-low-priority-pool-size":0,"store-io-pool-size":0,"store-io-notify-capacity":40960,"future-poll-size":1,"hibernate-regions":true,"dev-assert":false,"apply-yield-duration":"500ms","perf-level":0,"evict-cache-on-memory-ratio":0.0,"cmd-batch":true,"cmd-batch-concurrent-ready-max-count":1,"raft-write-size-limit":"1MiB","waterfall-metrics":true,"io-reschedule-concurrent-max-count":4,"io-reschedule-hotpot-duration":"5s","inspect-interval":"500ms","report-min-resolved-ts-interval":"0s","reactive-memory-lock-tick-interval":"2s","reactive-memory-lock-timeout-tick":5,"report-region-buckets-tick-interval":"10s","max-snapshot-file-raw-size":"100MiB"},"coprocessor":{"split-region-on-table":false,"batch-split-limit":10,"region-max-size":"144MiB","region-split-size":"96MiB","region-max-keys":1440000,"region-split-keys":960000,"consistency-check-method":"mvcc","enable-region-bucket":false,"region-bucket-size":"96MiB","region-size-threshold-for-approximate":"1440MiB","prefer-approximate-bucket":true,"region-bucket-merge-size-ratio":0.33},"coprocessor-v2":{"coprocessor-plugin-directory":null},"rocksdb":{"info-log-level":"info","wal-recovery-mode":2,"wal-dir":"","wal-ttl-seconds":0,"wal-size-limit":"0KiB","max-total-wal-size":"4GiB","max-background-jobs":3,"max-background-flushes":1,"max-manifest-file-size":"128MiB","create-if-missing":true,"max-open-files":256,"enable-statistics":true,"stats-dump-period":"10m","compaction-readahead-size":"0KiB","info-log-max-size":"1GiB","info-log-roll-time":"0s","info-log-keep-log-file-num":10,"info-log-dir":"","rate-bytes-per-sec":"10GiB","rate-limiter-refill-period":"100ms","rate-limiter-mode":2,"rate-limiter-auto-tuned":true,"bytes-per-sync":"1MiB","wal-bytes-per-sync":"512KiB","max-sub-compactions":1,"writable-file-max-buffer-size":"1MiB","use-direct-io-for-flush-and-compaction":false,"enable-pipelined-write":false,"enable-multi-batch-write":true,"enable-unordered-write":false,"defaultcf":{"block-size":"64KiB","block-cache-size":"2002MiB","disable-block-cache":false,"cache-index-and-filter-blocks":true,"pin-l0-filter-and-index-blocks":true,"use-bloom-filter":true,"optimize-filters-for-hits":true,"whole-key-filtering":true,"bloom-filter-bits-per-key":10,"block-based-bloom-filter":false,"read-amp-bytes-per-bit":0,"compression-per-level":["no","no","lz4","lz4","lz4","zstd","zstd"],"write-buffer-size":"128MiB","max-write-buffer-number":5,"min-write-buffer-number-to-merge":1,"max-bytes-for-level-base":"512MiB","target-file-size-base":"8MiB","level0-file-num-compaction-trigger":4,"level0-slowdown-writes-trigger":20,"level0-stop-writes-trigger":36,"max-compaction-bytes":"2GiB","compaction-pri":3,"dynamic-level-bytes":true,"num-levels":7,"max-bytes-for-level-multiplier":10,"compaction-style":0,"disable-auto-compactions":false,"disable-write-stall":true,"soft-pending-compaction-bytes-limit":"192GiB","hard-pending-compaction-bytes-limit":"1TiB","force-consistency-checks":false,"prop-size-index-distance":4194304,"prop-keys-index-distance":40960,"enable-doubly-skiplist":true,"enable-compaction-guard":true,"compaction-guard-min-output-file-size":"8MiB","compaction-guard-max-output-file-size":"128MiB","bottommost-level-compression":"zstd","bottommost-zstd-compression-dict-size":0,"bottommost-zstd-compression-sample-size":0,"titan":{"min-blob-size":"1KiB","blob-file-compression":"lz4","blob-cache-size":"0KiB","min-gc-batch-size":"16MiB","max-gc-batch-size":"64MiB","discardable-ratio":0.5,"sample-ratio":0.1,"merge-small-file-threshold":"8MiB","blob-run-mode":"normal","level-merge":false,"range-merge":true,"max-sorted-runs":20,"gc-merge-rewrite":false}},"writecf":{"block-size":"64KiB","block-cache-size":"1201MiB","disable-block-cache":false,"cache-index-and-filter-blocks":true,"pin-l0-filter-and-index-blocks":true,"use-bloom-filter":true,"optimize-filters-for-hits":false,"whole-key-filtering":false,"bloom-filter-bits-per-key":10,"block-based-bloom-filter":false,"read-amp-bytes-per-bit":0,"compression-per-level":["no","no","lz4","lz4","lz4","zstd","zstd"],"write-buffer-size":"128MiB","max-write-buffer-number":5,"min-write-buffer-number-to-merge":1,"max-bytes-for-level-base":"512MiB","target-file-size-base":"8MiB","level0-file-num-compaction-trigger":4,"level0-slowdown-writes-trigger":20,"level0-stop-writes-trigger":36,"max-compaction-bytes":"2GiB","compaction-pri":3,"dynamic-level-bytes":true,"num-levels":7,"max-bytes-for-level-multiplier":10,"compaction-style":0,"disable-auto-compactions":false,"disable-write-stall":true,"soft-pending-compaction-bytes-limit":"192GiB","hard-pending-compaction-bytes-limit":"1
|
username: 
|
username: 
|
username: