008_falcon磁盘io计算方法

一、falcon磁盘IO告警计算方法

(1)线上告警示例
【falcon】环境: prod 时间: 2018-11-10 22:29 共1条
【#主机磁盘io过高(appid)】主机hostname磁盘dfa io过高98.76>98%
(2)cat /proc/diskstats
252 0 dfa 2689642164 0 513826977162 100403006 6803204529 0 1348198360088 2893131263 0 934780608 3002409596
每项的具体含义如下:
1   252            主设备号    code:Major
2   0              次设备号    code:Minor
3   dfa            设备名称    code:Name 
4   2689642164     <1>读完成次数{读磁盘的次数,成功完成读的总次数}[2689642164]  code:ReadIOs
5   0              <2>合并读完成次数  code:ReadMerges
6   513826977162   <3>读扇区的次数,成功读过的扇区总次数  5.ReadSectors
7   100403006      <4>读花费的毫秒数;这是所有读操作所花费的毫秒数{从__make_request()到end_that_request_last()的测量} code:ReadTicks
8   6803204529     <5>写完成次数;写完成的次数,成功写完成的总次数  code:WriteIOs
9   0              <6>合并写完成次数。为了效率可能会合并相邻的读和写,从而两次4K的读在它最终被处理到磁盘上之前可能会变成一次8K的读,才被计数(和排队),因此只有一次I/O操作,这个域使你知道这样的操作有多频繁 code:WriteMerges
10  1348198360088  <7>写扇区次数;写扇区的次数,成功写扇区总次数  code:WriteSectors
11  2893131263     <8>写操作花费的毫秒数;写花费的毫秒数,这是所有写操作所花费的毫秒数{是从__make_request()到end_that_request_last()的测量}  code:WriteTicks
12  0              <9>正在处理的输入/输出请求数;I/O的当前进度,只有这个域应该是0.当请求被交给适当的request_queue_t时增加和请求完成时减小 code:InFlight
13  934780608      <10>输入/输出操作花费的毫秒数;花在I/O操作上的毫秒数,只要field9不为0这个域就会增长 code:IOTicks
14  3002409596     <11>输入/输出操作花费的加权毫秒数;花在I/O操作上的毫秒数,在每次I/O开始,I/O结束,I/O合并或读取自动上次更新这个域以来(第<9>列正在进行的io数量乘以花费在io上的毫秒数)时这个域都会增加.这可以给I/O完成时间和存储那些可以累积的提供一个便利的测量标准 code:TimeInQueue
(3)告警计算公式 
io_ticks{crt.IOTicks - last.IOTicks(上10s的值)}/100 > 98 会进行报警

二、实现代码

(1)
const diskStatPath = "/proc/diskstats"

type DiskStatCollector struct {
	lastDiskStats map[string]*linux.DiskStat
}

func (c *DiskStatCollector) Collect() ([]*model.Metric, error) {
	if c.lastDiskStats == nil {
		c.lastDiskStats = make(map[string]*linux.DiskStat)
	}
	disks, err := linux.ReadDiskStats(diskStatPath)
	if err != nil {
		return nil, fmt.Errorf("collect disk stat: %v", err)
	}
	var metrics []*model.Metric
	for _, crt := range disks {
		if len(crt.Name) == 3 && (strings.HasPrefix(crt.Name, "sd") || strings.HasPrefix(crt.Name, "vd") || strings.HasPrefix(crt.Name, "df")) {
			if last, ok := c.lastDiskStats[crt.Name]; ok {
				metrics = append(metrics, diskStatDiff(&crt, last)...)
			}
			tmp := crt
			c.lastDiskStats[crt.Name] = &tmp
		}
	}
	return metrics, nil
}

func diskStatDiff(crt *linux.DiskStat, last *linux.DiskStat) []*model.Metric {
	var metrics []*model.Metric
	if crt.IOTicks < last.IOTicks {
		return metrics
	}
	metrics = append(metrics, &model.Metric{
		Name: "disk.io",
		Fields: []*model.Field{
			{"read_ios", model.Gauge, crt.ReadIOs - last.ReadIOs},
			{"read_merges", model.Gauge, crt.ReadMerges - last.ReadMerges},
			{"read_sectors", model.Gauge, crt.ReadSectors - last.ReadSectors},
			{"read_ticks", model.Gauge, crt.ReadTicks - last.ReadTicks},
			{"write_ios", model.Gauge, crt.WriteIOs - last.WriteIOs},
			{"write_merges", model.Gauge, crt.WriteMerges - last.WriteMerges},
			{"write_sectors", model.Gauge, crt.WriteSectors - last.WriteSectors},
			{"write_ticks", model.Gauge, crt.WriteTicks - last.WriteTicks},
			{"in_flight", model.Gauge, crt.InFlight - last.InFlight},
			{"io_ticks", model.Gauge, crt.IOTicks - last.IOTicks},
			{"time_in_queue", model.Gauge, crt.TimeInQueue - last.TimeInQueue},
		},
		Tags:      map[string]string{"name": crt.Name},
		Timestamp: utils.Timestamp(),
	})
	return metrics
}

(2)

// ReadDiskStats reads and parses the file.
//
// Note:
// * Assumes a well formed file and will panic if it isn't.
func ReadDiskStats(path string) ([]DiskStat, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	devices := strings.Split(string(data), "\n")
	results := make([]DiskStat, len(devices)-1)

	for i := range results {
		fields := strings.Fields(devices[i])
		Major, _ := strconv.ParseInt(fields[0], 10, strconv.IntSize)
		results[i].Major = int(Major)
		Minor, _ := strconv.ParseInt(fields[1], 10, strconv.IntSize)
		results[i].Minor = int(Minor)
		results[i].Name = fields[2]
		results[i].ReadIOs, _ = strconv.ParseUint(fields[3], 10, 64)
		results[i].ReadMerges, _ = strconv.ParseUint(fields[4], 10, 64)
		results[i].ReadSectors, _ = strconv.ParseUint(fields[5], 10, 64)
		results[i].ReadTicks, _ = strconv.ParseUint(fields[6], 10, 64)
		results[i].WriteIOs, _ = strconv.ParseUint(fields[7], 10, 64)
		results[i].WriteMerges, _ = strconv.ParseUint(fields[8], 10, 64)
		results[i].WriteSectors, _ = strconv.ParseUint(fields[9], 10, 64)
		results[i].WriteTicks, _ = strconv.ParseUint(fields[10], 10, 64)
		results[i].InFlight, _ = strconv.ParseUint(fields[11], 10, 64)
		results[i].IOTicks, _ = strconv.ParseUint(fields[12], 10, 64)
		results[i].TimeInQueue, _ = strconv.ParseUint(fields[13], 10, 64)
	}

	return results, nil
}