配置管理 #

获取运行实例的配置信息、修改配置信息等操作。

获取配置列表 #

HTTP 请求 #

GET /config/

请求示例

curl -X GET http://localhost:2900/config/

响应示例

{
  "main": {
    "location": "/Users/medcl/go/src/infini.sh/agent/agent.yml",
    "content": "env:\n  API_BINDING: \"0.0.0.0:2900\"\n\npath.data: data\npath.logs: log\npath.configs: \"config\"\nconfigs.auto_reload: true\n\nresource_limit.cpu.max_num_of_cpus: 1\nresource_limit:\n  memory:\n    max_in_bytes: 533708800 #50MB\n\ntask:\n  max_concurrent_tasks: 3\n\nstats:\n  include_storage_stats_in_api: false\n\nelastic:\n  skip_init_metadata_on_start: true\n  health_check:\n    enabled: true\n    interval: 60s\n  availability_check:\n    enabled: false\n    interval: 60s\n\ndisk_queue:\n  max_msg_size: 20485760\n  max_bytes_per_file: 20485760\n  max_used_bytes: 524288000\n  retention.max_num_of_local_files: 1\n  compress:\n    idle_threshold: 1\n    num_of_files_decompress_ahead: 0\n    segment:\n      enabled: true\n\napi:\n  enabled: true\n  network:\n    binding: $[[env.API_BINDING]]\n\nbadger:\n  mem_table_size: 1048576\n  value_log_max_entries: 1000000\n  value_log_file_size: 104857600\n  value_threshold: 1024\n\nconfig:\n  managed: true # managed by remote servers\n  polling_interval: 60s\n  servers: # config servers\n   - \"http://host:port\"",
    "updated": 1694325825,
    "size": 1010,
    "readonly": true
  },
  "configs": {
    "ingest_config.tpl": {
      "location": "/Users/medcl/go/src/infini.sh/agent/config/ingest_config.tpl",
      "content": """elasticsearch:
  - name: $[[INGEST_CLUSTER_ID]]
    enabled: true
    endpoints: $[[INGEST_CLUSTER_ENDPOINT]]
    discovery:
      enabled: false
    basic_auth:
      username: $[[INGEST_CLUSTER_USERNAME]]
      password: $[[keystore.ingest_cluster_password]]

metrics:
  enabled: true
  queue: metrics
  network:
    enabled: true
    summary: true
    details: true
  memory:
    metrics:
      - swap
      - memory
  disk:
    metrics:
      - iops
      - usage
  cpu:
    metrics:
      - idle
      - system
      - user
      - iowait
      - load
  instance:
    enabled: true

elastic:
  availability_check:
    enabled: false

pipeline:
  - name: merge_logs
    auto_start: true
    keep_running: true
    processor:
      - indexing_merge:
          index_name: ".infini_logs"
          elasticsearch: "$[[INGEST_CLUSTER_ID]]"
          input_queue: "logs"
          idle_timeout_in_seconds: 10
          output_queue:
            name: "merged_requests"
          worker_size: 1
          bulk_size_in_mb: 5
  - name: merge_metrics
    auto_start: true
    keep_running: true
    processor:
      - indexing_merge:
          elasticsearch: "$[[INGEST_CLUSTER_ID]]"
          index_name: ".infini_metrics"
          input_queue: "metrics"
          output_queue:
            name: "merged_requests"
          worker_size: 1
          bulk_size_in_mb: 5
  - name: ingest_merged_requests
    auto_start: true
    keep_running: true
    processor:
      - bulk_indexing:
          max_worker_size: 1
          bulk:
            batch_size_in_mb: 5
            batch_size_in_docs: 5000
            max_retry_times: 0
            invalid_queue: ""
            response_handle:
              include_index_stats: false
              include_action_stats: false
              output_bulk_stats: false
              include_error_details: false
              save_error_results: false
              save_success_results: false
              save_busy_results: false
          consumer:
            fetch_max_messages: 5
          queues:
            type: indexing_merge
          when:
            cluster_available: ["$[[INGEST_CLUSTER_ID]]"]""",
      "updated": 1694256173,
      "version": 333,
      "size": 2149
    },
    "ingest_variables.yml": {
      "location": "/Users/medcl/go/src/infini.sh/agent/config/ingest_variables.yml",
      "content": """configs.template:
  - name: "ingest"
    path: ./config/ingest_config.tpl
    variable:
      INGEST_CLUSTER_ID: "default_ingest_cluster"
      INGEST_CLUSTER_ENDPOINT: ["http://localhost:9200"]
      INGEST_CLUSTER_USERNAME: "admin"
""",
      "updated": 1694242713,
      "version": 333,
      "size": 234
    },
    "task_config.tpl": {
      "location": "/Users/medcl/go/src/infini.sh/agent/config/task_config.tpl",
      "content": """elasticsearch:
  - id: $[[CLUSTER_ID]]
    name: $[[CLUSTER_ID]]
    enabled: true
    endpoint: $[[CLUSTER_ENDPOINT]]
    discovery:
      enabled: false
    basic_auth:
      username: $[[CLUSTER_USERNAME]]
      password: $[[keystore.$[[CLUSTER_ID]]_password]]

pipeline:
#clsuter level metrics
- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
  enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
  keep_running: true
  singleton: true
  name: collect_$[[CLUSTER_ID]]_es_cluster_stats
  retry_delay_in_ms: 10000
  processor:
  - es_cluster_stats:
      elasticsearch: $[[CLUSTER_ID]]
      labels:
        cluster_id: $[[CLUSTER_ID]]
      when:
        cluster_available: ["$[[CLUSTER_ID]]"]

- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
  enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
  keep_running: true
  singleton: true
  name: collect_$[[CLUSTER_ID]]_es_index_stats
  retry_delay_in_ms: 10000
  processor:
  - es_index_stats:
      elasticsearch: $[[CLUSTER_ID]]
      labels:
        cluster_id: $[[CLUSTER_ID]]
      when:
        cluster_available: ["$[[CLUSTER_ID]]"]

- auto_start: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
  enabled: $[[CLUSTER_LEVEL_TASKS_ENABLED]]
  keep_running: true
  singleton: true
  name: collect_$[[CLUSTER_ID]]_es_cluster_health
  retry_delay_in_ms: 10000
  processor:
  - es_cluster_health:
      elasticsearch: $[[CLUSTER_ID]]
      labels:
        cluster_id: $[[CLUSTER_ID]]
      when:
        cluster_available: ["$[[CLUSTER_ID]]"]

#node level metrics
- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]]
  enabled: $[[NODE_LEVEL_TASKS_ENABLED]]
  keep_running: true
  name: collect_$[[CLUSTER_ID]]_es_node_stats
  retry_delay_in_ms: 10000
  processor:
  - es_node_stats:
      elasticsearch: $[[CLUSTER_ID]]
      labels:
        cluster_id: $[[CLUSTER_ID]]
      when:
        cluster_available: ["$[[CLUSTER_ID]]"]

#node logs
- auto_start: $[[NODE_LEVEL_TASKS_ENABLED]]
  enabled: $[[NODE_LEVEL_TASKS_ENABLED]]
  keep_running: true
  name: collect_$[[CLUSTER_ID]]_es_logs
  retry_delay_in_ms: 10000
  processor:
  - es_logs_processor:
      elasticsearch: $[[CLUSTER_ID]]
      labels:
        cluster_id: $[[CLUSTER_ID]]
      logs_path: $[[NODE_LOGS_PATH]]
      queue_name: logs
      when:
        cluster_available: ["$[[CLUSTER_ID]]"]
""",
      "updated": 1694312483,
      "version": 333,
      "size": 2264
    },
    "task_variables.yml": {
      "location": "/Users/medcl/go/src/infini.sh/agent/config/task_variables.yml",
      "content": "configs.template:\n  - name: \"node1\"\n    path: ./config/task_config.tpl\n    variable:\n      CLUSTER_ID: \"infini_default_system_cluster\"\n      CLUSTER_ENDPOINT: \"http://localhost:9200\"\n      CLUSTER_USERNAME: \"admin\"\n      CLUSTER_LEVEL_TASKS_ENABLED: false\n      NODE_LEVEL_TASKS_ENABLED: true\n      NODE_LOGS_PATH: \"/opt/easysearch/logs\"",
      "updated": 1694314422,
      "size": 337
    }
  }
}

main 代表程序主配置，configs 为 config 目录 path.configs 下的配置文件，支持动态更新。

获取运行时配置 #

运行时配置是程序合并各个配置文件并经过模版变量渲染之后的最终配置，也就是程序实际运行过程中使用的配置内容。

HTTP 请求 #

GET /config/runtime

请求示例

curl -X GET http://localhost:2900/config/runtime

响应示例

{
  "api": {
    "enabled": true,
    "network": {
      "binding": "0.0.0.0:2900"
    }
  },
  "badger": {
    "mem_table_size": 1048576,
    "value_log_file_size": 104857600,
    "value_log_max_entries": 1000000,
    "value_threshold": 1024
  },
  "configs": {
    "auto_reload": true,
    "template": [
      {
        "name": "ingest",
        "path": "./config/ingest_config.tpl",
        "variable": {
          "INGEST_CLUSTER_ENDPOINT": [
            "http://localhost:9200"
          ],
          "INGEST_CLUSTER_ID": "default_ingest_cluster",
          "INGEST_CLUSTER_USERNAME": "admin"
        }
      },
      {
        "name": "node1",
        "path": "./config/task_config.tpl",
        "variable": {
          "CLUSTER_ENDPOINT": "http://localhost:9200",
          "CLUSTER_ID": "infini_default_system_cluster",
          "CLUSTER_LEVEL_TASKS_ENABLED": false,
          "CLUSTER_USERNAME": "admin",
          "NODE_LEVEL_TASKS_ENABLED": true,
          "NODE_LOGS_PATH": "/opt/easysearch/logs"
        }
      }
    ]
  },
  "disk_queue": {
    "compress": {
      "idle_threshold": 1,
      "num_of_files_decompress_ahead": 0,
      "segment": {
        "enabled": true
      }
    },
    "max_bytes_per_file": 20485760,
    "max_msg_size": 20485760,
    "max_used_bytes": 524288000,
    "retention": {
      "max_num_of_local_files": 1
    }
  },
  "elastic": {
    "availability_check": {
      "enabled": false,
      "interval": "60s"
    },
    "health_check": {
      "enabled": true,
      "interval": "60s"
    },
    "skip_init_metadata_on_start": true
  },
  "elasticsearch": [
    {
      "basic_auth": {
        "password": "",
        "username": "admin"
      },
      "discovery": {
        "enabled": false
      },
      "enabled": true,
      "endpoints": [
        "http://localhost:9200"
      ],
      "name": "default_ingest_cluster"
    },
    {
      "basic_auth": {
        "password": "",
        "username": "admin"
      },
      "discovery": {
        "enabled": false
      },
      "enabled": true,
      "endpoint": "http://localhost:9200",
      "id": "infini_default_system_cluster",
      "name": "infini_default_system_cluster"
    }
  ],
  "env": {
    "API_BINDING": "0.0.0.0:2900"
  },
  "metrics": {
    "cpu": {
      "metrics": [
        "idle",
        "system",
        "user",
        "iowait",
        "load"
      ]
    },
    "disk": {
      "metrics": [
        "iops",
        "usage"
      ]
    },
    "enabled": true,
    "instance": {
      "enabled": true
    },
    "memory": {
      "metrics": [
        "swap",
        "memory"
      ]
    },
    "network": {
      "details": true,
      "enabled": true,
      "summary": true
    },
    "queue": "metrics"
  },
  "path": {
    "configs": "config",
    "data": "data",
    "logs": "log"
  },
  "pipeline": [
    {
      "auto_start": true,
      "keep_running": true,
      "name": "merge_logs",
      "processor": [
        {
          "indexing_merge": {
            "bulk_size_in_mb": 5,
            "elasticsearch": "default_ingest_cluster",
            "idle_timeout_in_seconds": 10,
            "index_name": ".infini_logs",
            "input_queue": "logs",
            "output_queue": {
              "name": "merged_requests"
            },
            "worker_size": 1
          }
        }
      ]
    },
    {
      "auto_start": true,
      "keep_running": true,
      "name": "merge_metrics",
      "processor": [
        {
          "indexing_merge": {
            "bulk_size_in_mb": 5,
            "elasticsearch": "default_ingest_cluster",
            "index_name": ".infini_metrics",
            "input_queue": "metrics",
            "output_queue": {
              "name": "merged_requests"
            },
            "worker_size": 1
          }
        }
      ]
    },
    {
      "auto_start": true,
      "keep_running": true,
      "name": "ingest_merged_requests",
      "processor": [
        {
          "bulk_indexing": {
            "bulk": {
              "batch_size_in_docs": 5000,
              "batch_size_in_mb": 5,
              "invalid_queue": "",
              "max_retry_times": 0,
              "response_handle": {
                "include_action_stats": false,
                "include_error_details": false,
                "include_index_stats": false,
                "output_bulk_stats": false,
                "save_busy_results": false,
                "save_error_results": false,
                "save_success_results": false
              }
            },
            "consumer": {
              "fetch_max_messages": 5
            },
            "max_worker_size": 1,
            "queues": {
              "type": "indexing_merge"
            },
            "when": {
              "cluster_available": [
                "default_ingest_cluster"
              ]
            }
          }
        }
      ]
    },
    {
      "auto_start": false,
      "enabled": false,
      "keep_running": true,
      "name": "collect_infini_default_system_cluster_es_cluster_stats",
      "processor": [
        {
          "es_cluster_stats": {
            "elasticsearch": "infini_default_system_cluster",
            "labels": {
              "cluster_id": "infini_default_system_cluster"
            },
            "when": {
              "cluster_available": [
                "infini_default_system_cluster"
              ]
            }
          }
        }
      ],
      "retry_delay_in_ms": 10000,
      "singleton": true
    },
    {
      "auto_start": false,
      "enabled": false,
      "keep_running": true,
      "name": "collect_infini_default_system_cluster_es_index_stats",
      "processor": [
        {
          "es_index_stats": {
            "elasticsearch": "infini_default_system_cluster",
            "labels": {
              "cluster_id": "infini_default_system_cluster"
            },
            "when": {
              "cluster_available": [
                "infini_default_system_cluster"
              ]
            }
          }
        }
      ],
      "retry_delay_in_ms": 10000,
      "singleton": true
    },
    {
      "auto_start": false,
      "enabled": false,
      "keep_running": true,
      "name": "collect_infini_default_system_cluster_es_cluster_health",
      "processor": [
        {
          "es_cluster_health": {
            "elasticsearch": "infini_default_system_cluster",
            "labels": {
              "cluster_id": "infini_default_system_cluster"
            },
            "when": {
              "cluster_available": [
                "infini_default_system_cluster"
              ]
            }
          }
        }
      ],
      "retry_delay_in_ms": 10000,
      "singleton": true
    },
    {
      "auto_start": true,
      "enabled": true,
      "keep_running": true,
      "name": "collect_infini_default_system_cluster_es_node_stats",
      "processor": [
        {
          "es_node_stats": {
            "elasticsearch": "infini_default_system_cluster",
            "labels": {
              "cluster_id": "infini_default_system_cluster"
            },
            "when": {
              "cluster_available": [
                "infini_default_system_cluster"
              ]
            }
          }
        }
      ],
      "retry_delay_in_ms": 10000
    },
    {
      "auto_start": true,
      "enabled": true,
      "keep_running": true,
      "name": "collect_infini_default_system_cluster_es_logs",
      "processor": [
        {
          "es_logs_processor": {
            "elasticsearch": "infini_default_system_cluster",
            "labels": {
              "cluster_id": "infini_default_system_cluster"
            },
            "logs_path": "/opt/easysearch/logs",
            "queue_name": "logs",
            "when": {
              "cluster_available": [
                "infini_default_system_cluster"
              ]
            }
          }
        }
      ],
      "retry_delay_in_ms": 10000
    }
  ],
  "resource_limit": {
    "cpu": {
      "max_num_of_cpus": 1
    },
    "memory": {
      "max_in_bytes": 533708800
    }
  },
  "stats": {
    "include_storage_stats_in_api": false
  },
  "task": {
    "max_concurrent_tasks": 3
  }
}

修改配置文件 #

HTTP 请求 #

PUT /config/

请求示例

curl -X PUT http://localhost:2900/config/ -d'
{
  "configs":{
    "ingest_config.tpl":"HELLO: WORLD"
  }
}
'

响应示例

{
  "acknowledged": true
}

删除配置文件 #

HTTP 请求 #

DELETE /config/

请求示例

curl -X DELETE http://localhost:2900/config/ -d'
{
  "configs": ["ingest_config.tpl"]
}
'

响应示例

{
  "acknowledged": true
}