技術標籤：Elasticsearchelasticsearch

背景

由於公司使用自己的編排系統（基於docker），所以在業務服務啟動時，需要對某些服務物件進行初始化，本文就對es索引的初始做一個小的分享；

大致流程

初始化步驟主要是基於docker啟動一個容器執行自己編寫好的sh指令碼

基礎映象選擇

由於es大部分操作都是基於es的rest api進行，所以，選擇alpine:latest這個輕量級作業系統進行操作
dockerfile如下：

FROM alpine:latest

RUN apk add --update curl && rm -rf /var/cache/apk/*

ADD ./*.sh /scripts/
ADD ./*.json /scripts/
ADD ./template/ /scripts/template/
RUN mkdir -p /devops_runonce

CMD sh -c 'echo start > /devops_runonce/status; /scripts/init-index.sh; echo ok > /devops_runonce/status'

 

映象中將字尾為sh、json,template目錄所有檔案拷貝到對應目錄，然後執行init-index.sh指令碼，主要操作就是init-index.sh，你的任何初始化邏輯都在這裡；

初始化指令碼

#!/bin/sh

init_elastic_address() {
  if [ ${MEGCITY_ELASTIC_ADDRESS} ];
  then
      echo "MEGCITY_ELASTIC_ADDRESS = $MEGCITY_ELASTIC_ADDRESS"
  else
      echo "no env: MEGCITY_ELASTIC_ADDRESS"
 

      exit 1
  fi

  if [ ${MEGCITY_ELASTIC_USER} ];
  then
      echo "MEGCITY_ELASTIC_USER = $MEGCITY_ELASTIC_USER"
  else
      MEGCITY_ELASTIC_USER=elastic
  fi
}

init_elastic_setting() {

  if [ ${NUMBER_OF_SHARDS} ];
  then
      echo "NUMBER_OF_SHARDS = $NUMBER_OF_SHARDS"
 

  else
      NUMBER_OF_SHARDS=1
  fi

  if [ ${NUMBER_OF_REPLICAS} ];
  then
      echo "NUMBER_OF_REPLICAS = $NUMBER_OF_REPLICAS"
  else
      NUMBER_OF_REPLICAS=0
  fi

  if [ ${MAX_RESULT_WINDOW} ];
  then
      echo "MAX_RESULT_WINDOW = $MAX_RESULT_WINDOW"
  else
      MAX_RESULT_WINDOW=5000000
  fi
}
# 初始化elastic地址配置
init_elastic_address
# 初始化elastic配置配置
init_elastic_setting

#根據.json字尾檔案資料建立索引
# 引數：索引名稱、分片數量、副本數量、最大返回結果數
create_index(){
  echo "create index:$1 with shards: $2 replicas: $3! window: $4!"

  mappings=`cat /scripts/"$1"'.json'`

  curl --insecure -u ${MEGCITY_ELASTIC_USER}:${DEVOPS_INFRA_PASSWORD} -XPUT $MEGCITY_ELASTIC_ADDRESS'/'"$1"'?pretty' -H 'Content-Type: application/json' -d'
  {
      "settings" : {
          "index" : {
              "number_of_shards" : '"$2"',
              "number_of_replicas" : '"$3"',
              "max_result_window": '"$4"'
          }
      },
      "mappings" : '"$mappings"'
  }
  '
  curl --insecure -u ${MEGCITY_ELASTIC_USER}:${DEVOPS_INFRA_PASSWORD} -XPOST $MEGCITY_ELASTIC_ADDRESS'/_aliases?pretty' -H 'Content-Type: application/json' -d'
  {
      "actions" : [
          { "add" : { "index" : "'"$1"'", "alias" : "alias_'"$1"'" } }
      ]
  }
  '
}

# 初始化索引：在執行create_field_template函式後執行
# 因為create_field_template只是建立一個索引模板，索引並沒有真正建立
init_index(){
    echo "create index:$2"

    mappings=`cat /scripts/"$1"'.json'`

    echo "init $2"

    curl --insecure -u ${MEGCITY_ELASTIC_USER}:${DEVOPS_INFRA_PASSWORD} -XPUT $MEGCITY_ELASTIC_ADDRESS'/'"$2"'?pretty' -H 'Content-Type: application/json' -d'
    {
        "mappings" : '"$mappings"'
    }
    '
}
# 根據模板檔案建立索引
#配置欄位的模板alarm_template.json
create_field_template(){
  echo "create template:/scripts/template/$1_template.json"
  echo "with shards: $2 replicas: $3! window: $4!"
  mappings=`cat /scripts/template/"$1"_template.json`

    mappings=`cat /scripts/template/"$1"_template.json \
    | sed 's/SHARDS_NUM/'"$2"'/g' |  sed 's/REPLICAS_NUM/'"$3"'/g' |  sed 's/WINDOW_NUM/'"$4"'/g' \
    `
  curl --insecure -u ${MEGCITY_ELASTIC_USER}:${DEVOPS_INFRA_PASSWORD} -XPUT ${MEGCITY_ELASTIC_ADDRESS}'/_template/'"$1"'?pretty' -H 'Content-Type: application/json' -d'
  '"$mappings"'
  '
}

# 執行建立索引操作：基於.json
create_index megcity-event ${NUMBER_OF_SHARDS} ${NUMBER_OF_REPLICAS} ${MAX_RESULT_WINDOW}
# 執行建立索引操作：基於template
create_field_template alarm ${NUMBER_OF_SHARDS} ${NUMBER_OF_REPLICAS} ${MAX_RESULT_WINDOW}
# 
init_index megcity-alarm megcity-alarm

以上shell指令碼中有一定的註釋說明

es索引模板

es索引模板：
就是把已經建立好的某個索引的引數設定(settings)和索引對映(mapping)儲存下來作為模板, 在建立新索引時, 指定要使用的模板名, 就可以直接重用已經定義好的模板中的設定和對映.

索引模板一般用在時間序列相關的索引中.
—— 也就是說, 如果你需要每間隔一定的時間就建立一次索引, 你只需要配置好索引模板, 以後就可以直接使用這個模板中的設定, 不用每次都設定settings和mappings.

{
  "index_patterns": [
    "megcity-alarm"
  ],
  "order": 0,
  "mappings": {
    "properties": {
      "causeId": {
        "type": "keyword"
      },
      "caseNumber": {
        "type": "text",
        "fields" : {
          "keyword" : {
            "type" : "keyword",
            "ignore_above" : 256
          }
        }
      },
      "videoId": {
        "type": "keyword"
      },
      "causeArea": {
        "properties":{
          "x":{
            "type":"float"
          },
          "y":{
            "type":"float"
          }
        }
      },
      "createTime": {
        "type": "date",
        "format": "epoch_millis"
      },
      "location": {
        "type": "geo_point"
      },
      "state": {
        "type": "integer"
      }
    }
  },
  "settings" : {
    "index" : {
      "number_of_shards": "SHARDS_NUM",
      "number_of_replicas": "REPLICAS_NUM",
      "max_result_window": "WINDOW_NUM",
      "sort": {
        "field": [
          "createTime",
          "caseNumber.keyword"
        ],
        "order": [
          "desc",
          "desc"
        ]
      }
    }
  },
  "aliases": {
    "alias_alarm": {}
  }
}

json

這個其實就是索引的mapping

{
  "properties": {
      "causeId": {
        "type": "keyword"
      },
      "caseNumber": {
        "type": "text",
        "fields" : {
          "keyword" : {
            "type" : "keyword",
            "ignore_above" : 256
          }
        }
      },
      "videoId": {
        "type": "keyword"
      },
      "causeArea": {
        "properties":{
          "x":{
            "type":"float"
          },
          "y":{
            "type":"float"
          }
        }
      },
      "createTime": {
        "type": "date",
        "format": "epoch_millis"
      },
      "location": {
        "type": "geo_point"
      },
      "state": {
        "type": "integer"
      }
    }
  }

目錄結構