全文主要讲解skywalking-3.x在elasticsearch中的数据模型及涉及到的数据操作,分为客户端注册、JVM性能指标、调用链跟踪片段三个部分。

skywalking-5.x即将推出,后续继续跟进


客户端注册

  • 【应用】/application/type

    • 映射

      1
      2
      3
      4
      5
      6
      curl -XGET /application/type/_mapping
      {
      _id:生成规则保证围绕零值正值递增、负值递减,
      application_id:同_id,
      application_code: 客户端配置的应用名称
      }
    • 获取主键最大最小值

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      curl -XGET /application/type/_search?pretty -d
      '{
      "aggs":{
      "application_id":{
      "min":{
      "field":"application_id"
      }
      },
      "max_application_id":{
      "max":{
      "field":"application_id"
      }
      }
      }
      }'
    • 新建

      1
      2
      3
      4
      5
      curl -XPUT /application/type/${_id}?pretty
      '{
      "application_id": 数值型,
      "application_code":"客户端指定"
      }'
  • 【应用实例】/instance/type

    • 映射

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      15
      curl -XGET /instance/type/_mapping
      {
      _id:从1自增,
      instance_id:同_id,
      agent_uuid:客户端jvm实例生成UUID,每重启一次就是新值
      heartbeat_time:每60秒发送一次心跳
      register_time:注册时间,客户端断开重连时会重置为当前时间
      application_id:应用ID,也就是外键,一个应用可以拥有多个实例
      os_info:{
      osName:操作系统版本,
      hostName:主机名,
      processId:jvm进程ID,
      ipv4s:[]
      }
      }
    • 获取主键最大最小值

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      curl -XGET /instance/type/_search?pretty -d
      '{
      "aggs":{
      "instance_id":{
      "min":{ "field":"instance_id" }
      },"max_application_id":{
      "max":{ "field":"instance_id" }
      }
      }
      }'
    • 新建

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      curl -XPOST /instance/type/${_id} -d '
      {
      instance_id:同_id,从1自增
      agent_uuid:客户端jvm实例生成UUID,每重启一次就是新值
      heartbeat_time:每60秒发送一次心跳
      register_time:注册时间,客户端断开重连时会重置为当前时间
      application_id:应用ID,也就是外键,一个应用可以拥有多个实例
      os_info:{
      osName:操作系统版本,
      hostName:主机名,
      processId:jvm进程ID,
      ipv4s:[]
      }
      }'
    • 更新心跳信息

      1
      2
      3
      4
      curl -XPUT /instance/type/${_id} -d
      '{
      "heartbeat_time":long value
      }'
    • 根据instance_id获取所属应用application_id

      1
      2
      3
      4
      5
      6
      7
      8
      curl -XGET /instance/type/_search?pretty -d
      '{
      "query":{
      "term":{
      "instance_id":instance_id
      }
      }
      }'
    • 恢复连接

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      curl -XPUT /instance/type/instance_id -d '
      {
      instance_id: int value,
      heartbeat_time:long value,
      register_time: long value,
      application_id:int value,
      os_info:{
      osName:String,
      hostName:String,
      processId:String,
      ipv4s:[]
      }
      }'
  • 【应用接口或服务或接口】/service_name/type

    • 映射

      1
      2
      3
      4
      5
      6
      7
      curl -XGET /service_name/type/_mapping
      {
      _id:生成规则IdAutoIncrement.INSTANCE.increment
      service_id:同_id,
      application_id:外键
      service_name:该应用所提供的外部可调用接口
      }
    • 获取主键最大最小值

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      curl -XGET /service_name/type/_search?pretty -d
      '{
      "aggs":{
      "max_service_id":{
      "min":{ "field":"service_id" }
      },"max_service_id":{
      "max":{ "field":"service_id" }
      }
      }
      }'
    • 新建

      1
      2
      3
      4
      5
      6
      curl -XPOST /service_name/type/${_id} -d
      '{
      service_id:service_id,
      application_id:外键,
      service_name:服务名
      }'
    • 根据service_id获取service_name

      1
      2
      3
      4
      5
      6
      7
      8
      curl -XGET /service_name/type/_search?pretty -d
      '{
      "query":{
      "term":{
      "service_id”:service_id
      }
      }
      }'
    • 根据service_name,application_id获取service_id

      1
      2
      3
      4
      5
      6
      7
      8
      9
      curl -XGET /service_name/type/_search?pretty -d
      '{
      "query":{
      "term":{
      "service_name":"",
      "application_id":application_id
      }
      }
      }'

JVM性能指标

  • 【CPU利用率】/cpu_metric/type

    • 映射

      1
      2
      3
      4
      5
      6
      7
      curl -XGET /cpu_metric/type/_mapping
      {
      "_id":"${time_bucket}_${instance_id}",
      "instance_id":外键
      "time_bucket":客户端提交的时间戳
      "usage_percent":(jvm当前占用cpu时间-上次收集时占用cpu时间)/(系统当前时间-上次收集时的时间)
      }
    • 批量插入

      1
      2
      3
      4
      5
      6
      curl -XPUT /cpu_metric/type/${_id}?pretty
      '{
      "instance_id": 数值型,
      "time_bucket":数值型,
      "usage_percent”:浮点数
      }'
  • 【instance心跳数据】/instance/type

    • 批量更新

      1
      2
      3
      4
      curl -XPUT /instance/type/${instance_id}?pretty
      '{
      "heartbeat_time": 客户端提交的时间戳
      }'
  • 【JVM堆/非堆数据】/memory_metric/type

    • 映射

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      curl -XGET /memory_metric/type/_mapping
      {
      "_id":"${time_bucket}_${instance_id}_${is_heap}",
      "application_instance_id":外键,
      "committed":"大于等于used,能够保证JVM使用的内存",
      "init":"初始内存大小",
      "is_heap":是否堆内存,
      "max":"最大可用内存大小",
      "time_bucket":"信息上班时间,格式yyyyMMddHHmmss",
      "used":"已使用的内存大小"
      }
    • 批量插入

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      curl -XPUT /memory_metric/type/${_id}?pretty
      '{
      "application_instance_id":String,
      "committed":long,
      "init":long,
      "is_heap”:boolean,
      "max":long,
      "time_bucket":String,
      "used":long
      }'
  • 【GC】/gc_metric/type

    • 映射

      1
      2
      3
      4
      5
      6
      7
      8
      9
      curl -XGET /gc_metric/type/_mapping
      {
      "_id":"${time_bucket}_${instance_id}_${phrase}",
      "instance_id":"",
      "phrase":"0,1;0新生代,1:老生代",
      "count":"收集次数",
      "time_bucket":"信息收集时间,yyyyMMddHHmmss",
      "time":"手机总耗时,单位是毫秒",
      }
    • 批量插入

      1
      2
      3
      4
      5
      6
      7
      8
      9
      curl -XPUT /gc_metric/type/${_id}?pretty
      '{
      "instance_id":String,
      "phrase":int,
      "count":long,
      "is_heap":boolean,
      "time":long,
      "time_bucket":String
      }'
  • 【JVM详细堆信息】/memory_pool_metric/type

    • 映射

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      curl -XGET /memory_pool_metric/type/_mapping
      {
      "_id":"${time_bucket}_${instance_id}_${is_heap}",
      "instance_id":外键,
      "committed":"大于等于used,能够保证JVM使用的内存",
      "init":"初始内存大小",
      "pool_type":"0:code_cache_usage;1:newgen_usage;2:olden_usage;3:survivor_usage;4:permgean_usage;5:metaspace;-1:其他",
      "max":"最大可用内存大小",
      "time_bucket":"信息上班时间,格式yyyyMMddHHmmss",
      "used":"已使用的内存大小"
      }
    • 批量插入

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      curl -XPUT /memory_pool_metric/type/${_id}?pretty
      '{
      "instance_id":String,
      "committed":long,
      "init":long,
      "pool_type":boolean,
      "max":long,
      "time_bucket":String,
      "used":long
      }'

调用链跟踪片段

  • 【调用链】【调用链中的某个节点】/global_trace/type/_mapping

    • 映射

      1
      2
      3
      4
      5
      6
      {
      "id":"${segment_id}_${global_trace_id}",
      "global_trace_id":"调用链全局ID",
      "segment_id":"调用链片段ID",
      "time_bucket":"当前segment开始时间,yyyyMMddHHmm"
      }
    • 批量插入

      1
      2
      3
      4
      5
      6
      curl -XPUT /global_trace/type/${_id}?pretty
      '{
      "global_trace_id":String,
      "segment_id":long,
      "time_bucket":String
      }'
  • 【节点映射】【处理调用方application字典与被调用方实际application_id的映射】

    • 映射

      1
      2
      3
      4
      5
      6
      7
      8
      /node_mapping/type/_mapping
      {
      "id":"${time_bucket}_${application_id}_${address_id或address}",
      "address”:"调用方传递过来的被调用方的地址",
      "address_id":”调用方传递过来的被调用方的application_id",
      "application_id”:"被调用方现在的application_id",
      "time_bucket":"当前segment开始时间,yyyyMMddHHmm"
      }
    • 批量插入

      1
      2
      3
      4
      5
      6
      7
      curl -XPUT /node_mapping/type/${_id}?pretty
      '{
      "address":String,
      "address_id":long,
      "application_id”:int,
      "time_bucket":String
      }'
  • 【instance某一秒的服务调用耗时】

    • 映射

      1
      2
      3
      4
      5
      6
      7
      8
      9
      /instance_performance/type/_mapping
      {
      "id":"${time_bucket}_${instance_id}",
      "application_id":"外键",
      "calls":调用次数,1,
      "cost_total":"调用总耗时,单位毫秒",
      "instance_id":"外键",
      "time_bucket":"当前segment开始时间,yyyyMMddHHmmss"
      }
    • 批量插入

      1
      2
      3
      4
      5
      6
      7
      8
      curl -XPUT /instance_performance/type/${_id}?pretty
      '{
      "cost_total”:long,
      "calls":long,
      "instance_id":String,
      "application_id”:int,
      "time_bucket":String
      }'
  • 【application节点的中间件类型】

    • 映射

      1
      2
      3
      4
      5
      6
      7
      8
      9
      /node_component/type/_mapping
      {
      "id":"${time_bucket}_${component_id或component_name}_${peer_id或peer}",
      "component_id":"节点ID",
      "component_name":"节点名字",
      "peer":'节点服务的ip:port',
      "peer_id":"application_id",
      "time_bucket":"当前segment开始时间,yyyyMMddHHmm"
      }
    • 批量插入

      1
      2
      3
      4
      5
      6
      7
      8
      curl -XPUT /node_component/type/${_id}?pretty
      '{
      "component_id”:long,
      "peer_id":long,
      "component_name":String,
      "peer”:String,
      "time_bucket":String
      }'
  • 【节点调用关联调用次数】

    • 映射

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      14
      /node_reference/type/_mapping
      {
      "id":"${time_bucket}_${front_application_id}_${behind_application_id}",
      "s1_lte": 耗时1秒的次数,
      "s3_lte": 耗时1秒到3秒的ishu,
      "s5_gt": 耗时大于5秒的次数,
      "s5_lte": 耗时大于3秒小于五秒的次数,
      "summary": 总调用次数,
      "error": 错误次数,
      "front_application_id": 调用方application_id,
      "behind_application_id": 被调用方application_id,
      "behind_peer": "",
      "time_bucket": yyyyMMddHHmm
      }
    • 批量插入

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      13
      curl -XPUT /node_reference/type/${_id}?pretty
      '{
      s1_lte": 耗时1秒的次数,
      "s3_lte": 耗时1秒到3秒的ishu,
      "s5_gt": 耗时大于5秒的次数,
      "s5_lte": 耗时大于3秒小于五秒的次数,
      "summary": 总调用次数,
      "error": 错误次数,
      "front_application_id": 调用方application_id,
      "behind_application_id": 被调用方application_id,
      "behind_peer": "",
      "time_bucket": yyyyMMddHHmm
      }'
  • 【segment耗时查询】

    • 映射

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      12
      /segment_cost/type/_mapping
      {
      "_id":"segment_id",
      "start_time": 开始时间,毫秒,
      "cost": 耗时,毫秒,
      "service_name": "服务名",
      "end_time": 结束时间,
      "time_bucket": segment开始时间,yyyyMMddHHmm,
      "is_error": 是否报错,boolean,
      "segment_id": "调用链片段ID",
      "application_id": "外键,应用ID"
      }
    • 批量插入

      1
      2
      3
      4
      5
      6
      7
      8
      9
      10
      11
      curl -XPUT /segment_cost/type/${_id}?pretty
      '{
      "start_time": "",
      "cost":"",
      "service_name": "",
      "end_time": "",
      "time_bucket": "",
      "is_error": "",
      "segment_id": "",
      "application_id": ""
      }'
  • 【segment数据】

    • 映射

      1
      2
      3
      4
      5
      6
      7
      /segment/type/_mapping
      {
      "_id":"segment_id",
      "data_binary": "binary"
      }
      ```
      - 批量插入

      curl -XPUT /segment/type/${_id}?pretty
      ‘{

      "data_binary": "binary"
      

      }’

      1
      2
      - 【无被调用关系的服务】
      - 映射

      /service_entry/type/_mapping
      {

      "_id":"${application_id}_${entry_service_name}",
      "entry_service_name": "节点entry服务名称",
      "newest_time": 最近一次注册时间yyyyMMddHHmm,
      "entry_service_id": 外键,
      "application_id": 外键,
      "register_time": 第一次注册时间
      

      }

      1
      - 批量插入

      curl -XPUT /service_entry/type/${_id}?pretty
      ‘{

      "entry_service_name": "节点entry服务名称",
      "newest_time": 最近一次注册时间yyyyMMddHHmm,
      "entry_service_id": 外键,
      "application_id": 外键,
      "register_time": 第一次注册时间
      

      }’

      1
      2
      - 【服务调用关系】
      - 映射

      /service_reference/type/_mapping
      {

      "_id":"${time_bucket}_${entry_service_name或entry_service_id}_$                "{front_service_name或front_service_id}_${behind_service_name或behind_service_id}",
      "behind_service_name": "${behind_application_id}_${behind_service_name}",
      "s1_lte": 耗时1秒及内的调用次数,
      "summary": 总调用次数,
      "front_service_id": 调用方服务ID,
      "entry_service_name": "${entry_application_id}_${entry_service_name}",
      "s5_gt": 大于5秒的调用次数,
      "entry_service_id": 入口服务ID,
      "s3_lte": 大于1秒小于等于3秒的调用次数,
      "cost_summary": 调用总耗时,
      "error": 错误次数,
      "s5_lte": 耗时大于3秒小于等于5秒的调用次数,
      "front_service_name": 调用方服务名字,
      "time_bucket": yyyyMMddHHmm,
      "behind_service_id": 被调用方服务ID
      

      }

      1
      - 批量插入
      curl -XPUT /service_reference/type/${_id}?pretty  '{}'
      

      ```