Error Messages During the Upgrade Process from TiDB 5.1 to 6.1

Translated
1

Note:
This topic has been translated from a Chinese forum by GPT and might contain errors.

Original topic: tidb 5.1升级到6.1检查过程中错误信息

| username: fshowmia

[TiDB Usage Environment] Production Environment
[TiDB Version] v5.1.0
[Encountered Problem]
When executing the upgrade check on the TiDB cluster v5.1.0 using the command tiup cluster check tidb-cluster --cluster, an ERROR is thrown, and the cause is currently unknown.
2022-09-21T17:02:47.722+0800 ERROR SSHCommand {“host”: “118.22.1.1”, “port”: “22”, “cmd”: “export LANG=C; PATH=$PATH:/usr/bin:/usr/sbin sudo -H bash -c "/tmp/tiup/bin/insight"”, “error”: “Run Command Timeout”, “stdout”: "{
"meta": {
"timestamp": "2022-09-21T17:01:47.760015551+08:00",
"uptime": 20391821.21,
"idle_time": 1586614724.41,
"sysinfo_ver": "0.9.4",
"git_branch": "tiup",
"git_commit": "v0.4.1-1-g5a79850",
"go_version": "go1.17 linux/amd64",
"tidb": null,
"tikv": null,
"pd": null
},
"sysinfo": {
"node": {
"hostname": "ptidblbtiser01",
"machineid": "fa28be13860146878e0840c6428603b8",
"timezone": "Asia/Shanghai"
},
"os": {
"name": "CentOS Linux 7 (Core)",
"vendor": "centos",
"version": "7",
"release": "7.8.2003",
"architecture": "amd64"
},
"kernel": {
"release": "3.10.0-1127.el7.x86_64",
"version": "#1 SMP Tue Mar 31 23:36:51 UTC 2020",
"architecture": "x86_64"
},
"product": {
"name": "PowerEdge R930",
"vendor": "Dell Inc.",
"serial": "54WNZH2"
},
"board": {
"name": "0Y0V4F",
"vendor": "Dell Inc.",
"version": "A02",
"serial": ".54WNZH2.CN747516CU0067."
},
"chassis": {
"type": 23,
"vendor": "Dell Inc.",
"serial": "54WNZH2"
},
"bios": {
"vendor": "Dell Inc.",
"version": "2.2.0",
"date": "09/29/2016"
},
"cpu": {
"vendor": "GenuineIntel",
"model": "Intel(R) Xeon(R) CPU E7-4820 v3 @ 1.90GHz",
"speed": 1900,
"cache": 25600,
"cpus": 4,
"cores": 40,
"threads": 80,
"governor": "performance"
},
"memory": {
"type": "DDR4",
"speed": 2400,
"size": 131072,
"swap": 32767
},
"storage": [
{
"name": "nvme0n1",
"model": "INTEL SSDPEDMD400G4",
"serial": "PHFT6344000M400BGN",
"size": 400
},
{
"name": "nvme1n1",
"model": "INTEL SSDPEDMD400G4",
"serial": "PHFT6344006A400BGN",
"size": 400
},
{
"name": "sda",
"driver": "sd",
"vendor": "DELL",
"model": "PERC H730P Adp",
"serial": "61866da09483050026039e8a0da373bb",
"size": 299
},
{
"name": "sdb",
"driver": "sd",
"vendor": "DELL",
"model": "PERC H730P Adp",
"serial": "61866da0948305002606c6c507fe8e42",
"size": 1798
}
],
"lvm": [
{
"lvname": "lv_data",
"vgname": "ptidbdatavg",
"lvsize": 1715324
},
{
"lvname": "lv_root",
"vgname": "ptidbosvg",
"lvsize": 98196
},
{
"lvname": "lv_tidb",
"vgname": "ptidbosvg",
"lvsize": 153600
}
],
"network": [
{
"name": "bond0",
"macaddress": "4a:13:ff:56:36:ff",
"mtu": 1500
},
{
"name": "bond1",
"macaddress": "a0:36:9f:e1:25:c0",
"ipaddress": [
"118.22.1.1/24",
"fe80::a236:9fff:fee1:25c0/64"
],
"speed": 10000,
"mtu": 1500
},
{
"name": "em1",
"driver": "tg3",
"macaddress": "18:66:da:f3:bc:28",
"port": "tp",
"speed": 1000,
"mtu": 1500
},
{
"name": "em2",
"driver": "tg3",
"macaddress": "18:66:da:f3:bc:29",
"port": "tp",
"speed": 1000,
"mtu": 1500
},
{
"name": "em3",
"driver": "tg3",
"macaddress": "18:66:da:f3:bc:2a",
"port": "tp",
"speed": 1000,
"mtu": 1500
},
{
"name": "em4",
"driver": "tg3",
"macaddress": "18:66:da:f3:bc:2b",
"port": "tp",
"speed": 1000,
"mtu": 1500
},
{
"name": "p6p1",
"driver": "ixgbe",
"macaddress": "a0:36:9f:e1:25:c0",
"port": "tp",
"speed": 10000,
"mtu": 1500
},
{
"name": "p6p2",
"driver": "ixgbe",
"macaddress": "a0:36:9f:e1:25:c0",
"port": "tp",
"speed": 10000,
"mtu": 1500
}
]
},
"ntp": {
"version": "ntpd 4.2.6p5@1.2349-o Thu Aug 8 11:47:59 UTC 2019 (1)",
"sync": "sync_ntp",
"stratum": 3,
"offset": 1.062,
"jitter": 0.612,
"status": "0648"
},
"partitions": [
{
"name": "sda",
"mount": {},
"sectors": 584843264,
"size": 299439751168,
"subdev": [
{
"name": "sda1",
"partition": true,
"mount": {
"mount_point": "/boot/efi",
"filesystem": "vfat",
"mount_options": "rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=ascii,shortname=winnt,errors=remount-ro"
},
"uuid": "735B-1BDD",
"sectors": 1024000,
"size": 524288000
},
{
"name": "sda2",
"partition": true,
"mount": {
"mount_point": "/boot",
"filesystem": "ext4",
"mount_options": "rw,relatime,data=ordered"
},
"uuid": "bc944a30-edc0-49b6-a040-92fd684440ca",
"sectors": 1024000,
"size": 524288000
},
{
"name": "sda3",
"partition": true,
"mount": {
"mount_point": "[SWAP]",
"filesystem": "swap"
},
"uuid": "f69e4954-ac58-4c0a-9b7e-69a5e1299675",
"sectors": 67108864,
"size": 34359738368
},
{
"name": "sda4",
"partition": true,
"mount": {},
"sectors": 515682304,
"size": 264029339648
}
],
"rotational": "1"
},
{
"name": "sdb",
"mount": {},
"sectors": 3512991744,
"size": 1798651772928,
"subdev": [
{
"name": "sdb1",
"partition": true,
"mount": {},
"sectors": 3512989696,
"size": 1798650724352
}
],
"rotational": "1"
},
{
"name": "sr0",
"mount": {},
"sectors": 2097151,
"size": 1073741312,
"rotational": "1"
},
{
"name": "dm-0",
"mount": {
"mount_point": "/",
"filesystem": "ext4",
"mount_options": "rw,relatime,data=ordered"
},
"uuid": "ce0cf471-6bfc-49d2-b96a-83d7a4084cb6",
"sectors": 201105408,
"size": 102965968896,
"slave_of": [
"sda4"
],
"rotational": "1"
},
{
"name": "dm-1",
"mount": {},
"uuid": "6958e267-0d5a-4e3b-87d1-c48aab7bc7f7",
"sectors": 314572800,
"size": 161061273600,
"slave_of": [
"sda4"
],
"rotational": "1"
},
{
"name": "dm-2",
"mount": {
"mount_point": "/data",
"filesystem": "ext4",
"mount_options": "rw,noatime,nodelalloc,data=ordered"
},
"uuid": "e1bf4aac-9b9d-43a6-b444-7c840ac8b933",
"sectors": 3512983552,
"size": 1798647578624,
"slave_of": [
"sdb1"
],
"rotational": "1"
},
{
"name": "nvme0n1",
"mount": {
"mount_point": "/tidb/tidb1",
"filesystem": "ext4",
"mount_options": "rw,noatime,nodelalloc,data=ordered"
},
"uuid": "030a3126-d15f-426f-b4ae-07af11ca0a1f",
"sectors": 781422768,
"size": 400088457216,
"rotational": "0"
},
{
"name": "nvme1n1",
"mount": {
"mount_point": "/tidb/tidb2",
"filesystem": "ext4",
"mount_options": "rw,noatime,nodelalloc,data=ordered"
},
"uuid": "ac3db71f-ac07-4096-99ae-c18c75d36f1e",
"sectors": 781422768,
"size": 400088457216,
"rotational": "0"
}
],
"epoll_exclusive": true,
"sockets": [
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 54056,
"dest_addr": "118.22.1.4",
"dest_port": 2379
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 55062,
"dest_addr": "118.22.1.11",
"dest_port": 20172
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 42044,
"dest_addr": "118.22.1.4",
"dest_port": 2379
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 42014,
"dest_addr": "118.22.1.10",
"dest_port": 20171
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 48940,
"dest_addr": "118.22.1.1",
"dest_port": 10081
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 38960,
"dest_addr": "118.22.1.12",
"dest_port": 20171
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 55060,
"dest_addr": "118.22.1.11",
"dest_port": 20172
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 56640,
"dest_addr": "118.22.1.15",
"dest_port": 10081
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 49784,
"dest_addr": "118.22.1.11",
"dest_port": 20171
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 49844,
"dest_addr": "118.22.1.11",
"dest_port": 20171
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 36800,
"dest_addr": "118.22.1.15",
"dest_port": 10082
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 38524,
"dest_addr": "118.22.1.6",
"dest_port": 20172
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 52982,
"dest_addr": "118.22.1.8",
"dest_port": 20171
},
{
"family": 2,
"state": 1,
"source_addr": "118.22.1.1",
"source_port": 33512,
"dest_addr": "118.22.1.10",
"dest_port": 20172
},\

2
| username: wuxiangdong | Original post link

The logs suggest that SSH did not connect.

3
| username: fshowmia | Original post link

Manually testing it works fine, SSH is connected, and /tmp/tiup/bin/insight can also be executed. I really can’t figure out where the problem is, suspecting it’s a bug.

4
| username: fshowmia | Original post link

The specific reason still couldn’t be identified.
Temporary solution: Add the --ssh system parameter during the cluster upgrade check.

5
| username: ealam_小羽 | Original post link

It should be due to this reason. The default version is “builtin,” which is incompatible with your system version. I remember someone encountered this issue before.