chore(ci): Optimize Elasticsearch index init script, remove Docker im… (#106)

Makefile

@@ -13,6 +13,8 @@ MYSQL_SCHEMA := ./docker/volumes/mysql/schema.sql
 MYSQL_INIT_SQL := ./docker/volumes/mysql/sql_init.sql
 ENV_FILE := ./docker/.env
 STATIC_DIR := ./bin/resources/static
+ES_INDEX_SCHEMA := ./docker/volumes/elasticsearch/es_index_schema
+ES_SETUP_SCRIPT := ./docker/volumes/elasticsearch/setup_es.sh
 
 debug: env middleware python server
 
@@ -26,7 +28,7 @@ fe:
 	@echo "Building frontend..."
 	@bash $(BUILD_FE_SCRIPT)
 
-server: env
+server: env setup_es_index
 	@if [ ! -d "$(STATIC_DIR)" ]; then \
 		echo "Static directory '$(STATIC_DIR)' not found, building frontend..."; \
 		$(MAKE) fe; \
@@ -84,6 +86,10 @@ atlas-hash:
 	@echo "Rehash atlas migration files..."
 	@(cd ./docker/atlas && atlas migrate hash)
 
+setup_es_index:
+	@echo "Setting up Elasticsearch index..."
+	@bash $(ES_SETUP_SCRIPT)  --index-dir $(ES_INDEX_SCHEMA) --docker-host false
+
 help:
 	@echo "Usage: make [target]"
 	@echo ""
@@ -103,4 +109,5 @@ help:
 	@echo "  clean            - Stop the docker containers and clean volumes."
 	@echo "  python           - Setup python environment."
 	@echo "  atlas-hash       - Rehash atlas migration files."
+	@echo "  setup_es_index   - Setup elasticsearch index."
 	@echo "  help             - Show this help message."

backend/Dockerfile

@@ -31,7 +31,7 @@ WORKDIR /app
 # Install runtime dependencies for Go app and base for Python
 # pax-utils for scanelf, python3 for running Python, python3-dev for headers/shared libs
 # bind-tools for nslookup etc., file for debugging file types
-RUN apk add --no-cache pax-utils python3 python3-dev bind-tools file deno
+RUN apk add --no-cache pax-utils python3 python3-dev bind-tools file deno curl
 
 # Install Python build dependencies, create venv, install packages, then remove build deps
 RUN apk add --no-cache --virtual .python-build-deps build-base py3-pip git && \
@@ -81,6 +81,9 @@ EXPOSE 8888
 
 # Use a script to start both applications
 COPY backend/script/bootstrap.sh /app/bootstrap.sh
-RUN chmod +x /app/bootstrap.sh
+COPY docker/volumes/elasticsearch/setup_es.sh /app/setup_es.sh
+COPY docker/volumes/elasticsearch/es_index_schema /app/es_index_schemas
+
+RUN chmod +x /app/bootstrap.sh /app/setup_es.sh
 
 CMD ["/app/bootstrap.sh"]

backend/script/bootstrap.sh

@@ -1,5 +1,9 @@
 #!/bin/sh
 
+# Set up Elasticsearch
+echo "Setting up Elasticsearch..."
+/app/setup_es.sh --index-dir /app/es_index_schemas
+
 # Start the proxy application in the background
 echo "Starting proxy application..."
 /app/proxy >/tmp/proxy.log 2>&1 &

docker/docker-compose.yml

@@ -65,8 +65,6 @@ services:
     depends_on:
       minio-setup:
         condition: service_completed_successfully
-      elasticsearch-setup:
-        condition: service_completed_successfully
       mysql-setup-schema:
         condition: service_completed_successfully
       mysql-setup-init-sql:
@@ -409,27 +407,6 @@ services:
     networks:
       - coze-network
 
-  elasticsearch-setup:
-    image: alpine/curl:8.12.1
-    container_name: coze-elasticsearch-setup
-    profiles: ['middleware', 'volcano-setup']
-    env_file: *env_file
-    depends_on:
-      elasticsearch:
-        condition: service_healthy
-    volumes:
-      - ./volumes/elasticsearch/setup_es.sh:/setup_es.sh
-      - ./volumes/elasticsearch/es_index_schema:/es_index_schema
-    command:
-      - /bin/sh
-      - -c
-      - |
-        set -ex
-        /setup_es.sh
-        echo 'Elasticsearch setup complete.'
-    networks:
-      - coze-network
-    restart: 'no'
   minio-setup:
     image: minio/mc:RELEASE.2025-05-21T01-59-54Z-cpuv1
     container_name: coze-minio-setup
@@ -562,8 +539,6 @@ services:
         condition: service_healthy
       minio-setup:
         condition: service_completed_successfully
-      elasticsearch-setup:
-        condition: service_completed_successfully
       mysql-setup-init-sql:
         condition: service_completed_successfully
     command: ['/app/bootstrap.sh']

docker/volumes/elasticsearch/setup_es.sh

@@ -1,14 +1,61 @@
 #!/bin/sh
 set -e
 
-if [[ "$ES_ADDR" == *"localhost"* || "$ES_ADDR" == *"127.0.0.1"* ]]; then
-  echo "ES_ADDR is localhost, using docker address: http://elasticsearch:9200"
-  ES_ADDR="http://elasticsearch:9200"
+# Parse command-line arguments
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --es-address)
+      case "$2" in
+        "" | -*) echo "Error: Missing value for $1" >&2; exit 1;;
+      esac
+      ES_ADDR_ARG="$2"
+      shift 2
+      ;;
+    --index-dir)
+      case "$2" in
+        "" | -*) echo "Error: Missing value for $1" >&2; exit 1;;
+      esac
+      INDEX_DIR_ARG="$2"
+      shift 2
+      ;;
+    --docker-host)
+      case "$2" in
+        "true" | "false") REPLACE_ES_ADDR_WITH_DOCKER_HOST_ARG="$2" ;;
+        *) echo "Error: Invalid value for $1. Must be 'true' or 'false'" >&2; exit 1;;
+      esac
+      shift 2
+      ;;
+    *)
+      # unknown option
+      shift
+      ;;
+  esac
+done
+
+# If ES_ADDR_ARG argument is provided, use it. Otherwise, use environment variable or default logic.
+if [ -n "$ES_ADDR_ARG" ]; then
+  ES_ADDR=$ES_ADDR_ARG
+fi
+
+REPLACE_ES_ADDR=${REPLACE_ES_ADDR_WITH_DOCKER_HOST_ARG:-true}
+if [ "$REPLACE_ES_ADDR" = "true" ]; then
+  case "$ES_ADDR" in
+    *localhost*|*"127.0.0.1"*)
+      echo "ES_ADDR is localhost, using docker address: http://elasticsearch:9200"
+      ES_ADDR="http://elasticsearch:9200"
+      ;;
+  esac
+fi
+
+# If INDEX_DIR_ARG argument is provided, use it. Otherwise, use the default.
+if [ -n "$INDEX_DIR_ARG" ]; then
+  INDEX_DIR=$INDEX_DIR_ARG
+else
+  INDEX_DIR=/es_index_schema
 fi
-# ES_ADDR=http://localhost:31160
-INDEX_DIR=/es_index_schema
 
 echo "ES_ADDR: $ES_ADDR"
+echo "INDEX_DIR: $INDEX_DIR"
 
 AUTH_PARAM=""
 if [ -n "$ES_USERNAME" ]; then
@@ -38,30 +85,32 @@ if [ -z "$ES_TEMPLATES" ]; then
   exit 1
 else
   # Add index creation logic
-  echo -e "🔄 Creating Elasticsearch indexes..."
   for template_file in $ES_TEMPLATES; do
-
     template_name=$(basename "$template_file" | sed 's/\.index-template\.json$//')
-    echo -e "➡️ Registering template: $template_name"
+    # Check if index template exists
+    if curl -s -f $AUTH_PARAM -I "${ES_ADDR}/_index_template/$template_name" >/dev/null; then
+      echo -e "ℹ️ Index template $template_name already exists"
+    else
+      echo -e "➡️ Registering template: $template_name"
 
-    # Attempt to register index template
-    response=$(curl -s $AUTH_PARAM -X PUT "${ES_ADDR}/_index_template/$template_name" \
-      -H "Content-Type: application/json" \
-      -d @"$template_file" 2>&1)
+      # Attempt to register index template
+      response=$(curl -s $AUTH_PARAM -X PUT "${ES_ADDR}/_index_template/$template_name" \
+        -H "Content-Type: application/json" \
+        -d @"$template_file" 2>&1)
 
-    # Check if successful
-    if echo "$response" | grep -q '"acknowledged":true'; then
-      echo -e "✅ Template $template_name registered successfully"
-    else
-      echo -e "❌ Failed to register template $template_name. Response: $response"
-      exit 1
+      # Check if successful
+      if echo "$response" | grep -q '"acknowledged":true'; then
+        echo -e "✅ Template $template_name registered successfully"
+      else
+        echo -e "❌ Failed to register template $template_name. Response: $response"
+        exit 1
+      fi
     fi
 
     index_name=$(basename "$template_file" | sed 's/\.index-template\.json$//')
-    echo -e "➡️ Creating index: $index_name"
-
     # Check if index exists
     if ! curl -s -f $AUTH_PARAM "${ES_ADDR}/_cat/indices/$index_name" >/dev/null; then
+      echo -e "➡️ Creating index: $index_name"
       # Create index (matching template's index_patterns)
       curl $AUTH_PARAM -X PUT "${ES_ADDR}/$index_name" -H "Content-Type: application/json"
       echo ""
@@ -90,4 +139,4 @@ else
 fi
 
 echo "Elasticsearch setup completed."
-exit 0
+exit 0