From 878a7ef15b8596d1f0ad278e7a10a4f3c8e4c0f9 Mon Sep 17 00:00:00 2001 From: Ryo Date: Wed, 10 Sep 2025 10:34:16 +0800 Subject: [PATCH] feat(infra): remove minio local proxy (#2059) --- backend/infra/contract/storage/storage.go | 12 ++-- .../impl/storage/internal/proxy/proxy.go | 63 ------------------- backend/infra/impl/storage/minio/minio.go | 7 --- backend/infra/impl/storage/s3/s3.go | 6 -- backend/infra/impl/storage/tos/tos.go | 6 -- backend/main.go | 59 ----------------- 6 files changed, 6 insertions(+), 147 deletions(-) delete mode 100644 backend/infra/impl/storage/internal/proxy/proxy.go diff --git a/backend/infra/contract/storage/storage.go b/backend/infra/contract/storage/storage.go index 27a3712c..eb62c366 100644 --- a/backend/infra/contract/storage/storage.go +++ b/backend/infra/contract/storage/storage.go @@ -68,10 +68,10 @@ type ListObjectsPaginatedOutput struct { } type FileInfo struct { - Key string - LastModified time.Time - ETag string - Size int64 - URL string - Tagging map[string]string + Key string `json:"key"` + LastModified time.Time `json:"last_modified"` + ETag string `json:"etag"` + Size int64 `json:"size"` + URL string `json:"url"` + Tagging map[string]string `json:"tagging"` } diff --git a/backend/infra/impl/storage/internal/proxy/proxy.go b/backend/infra/impl/storage/internal/proxy/proxy.go deleted file mode 100644 index a1368456..00000000 --- a/backend/infra/impl/storage/internal/proxy/proxy.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2025 coze-dev Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package proxy - -import ( - "context" - "net" - "net/url" - "os" - - "github.com/coze-dev/coze-studio/backend/pkg/ctxcache" - "github.com/coze-dev/coze-studio/backend/pkg/logs" - "github.com/coze-dev/coze-studio/backend/types/consts" -) - -func CheckIfNeedReplaceHost(ctx context.Context, originURLStr string) (ok bool, proxyURL string) { - // url parse - originURL, err := url.Parse(originURLStr) - if err != nil { - logs.CtxWarnf(ctx, "[CheckIfNeedReplaceHost] url parse failed, err: %v", err) - return false, "" - } - - proxyPort := os.Getenv(consts.MinIOProxyEndpoint) // :8889 - if proxyPort == "" { - return false, "" - } - - currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx) - if !ok { - return false, "" - } - - currentScheme, ok := ctxcache.Get[string](ctx, consts.RequestSchemeKeyInCtx) - if !ok { - return false, "" - } - - host, _, err := net.SplitHostPort(currentHost) - if err != nil { - host = currentHost - } - - minioProxyHost := host + proxyPort - originURL.Host = minioProxyHost - originURL.Scheme = currentScheme - logs.CtxDebugf(ctx, "[CheckIfNeedReplaceHost] reset originURL.String = %s", originURL.String()) - return true, originURL.String() -} diff --git a/backend/infra/impl/storage/minio/minio.go b/backend/infra/impl/storage/minio/minio.go index 2784de0a..60882d4a 100644 --- a/backend/infra/impl/storage/minio/minio.go +++ b/backend/infra/impl/storage/minio/minio.go @@ -30,7 +30,6 @@ import ( "github.com/coze-dev/coze-studio/backend/infra/contract/storage" "github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/fileutil" - "github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/proxy" "github.com/coze-dev/coze-studio/backend/pkg/logs" ) @@ -232,12 +231,6 @@ func (m *minioClient) GetObjectUrl(ctx context.Context, objectKey string, opts . return "", fmt.Errorf("GetObjectUrl failed: %v", err) } - // logs.CtxDebugf(ctx, "[GetObjectUrl] origin presignedURL.String = %s", presignedURL.String()) - ok, proxyURL := proxy.CheckIfNeedReplaceHost(ctx, presignedURL.String()) - if ok { - return proxyURL, nil - } - return presignedURL.String(), nil } diff --git a/backend/infra/impl/storage/s3/s3.go b/backend/infra/impl/storage/s3/s3.go index 6670f401..98bdb640 100644 --- a/backend/infra/impl/storage/s3/s3.go +++ b/backend/infra/impl/storage/s3/s3.go @@ -32,7 +32,6 @@ import ( "github.com/coze-dev/coze-studio/backend/infra/contract/storage" "github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/fileutil" - "github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/proxy" "github.com/coze-dev/coze-studio/backend/pkg/goutil" "github.com/coze-dev/coze-studio/backend/pkg/logs" "github.com/coze-dev/coze-studio/backend/pkg/taskgroup" @@ -240,11 +239,6 @@ func (t *s3Client) GetObjectUrl(ctx context.Context, objectKey string, opts ...s return "", fmt.Errorf("get object presigned url failed: %v", err) } - ok, proxyURL := proxy.CheckIfNeedReplaceHost(ctx, req.URL) - if ok { - return proxyURL, nil - } - return req.URL, nil } diff --git a/backend/infra/impl/storage/tos/tos.go b/backend/infra/impl/storage/tos/tos.go index 991facb5..adc4c727 100644 --- a/backend/infra/impl/storage/tos/tos.go +++ b/backend/infra/impl/storage/tos/tos.go @@ -30,7 +30,6 @@ import ( "github.com/coze-dev/coze-studio/backend/infra/contract/storage" "github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/fileutil" - "github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/proxy" "github.com/coze-dev/coze-studio/backend/pkg/goutil" "github.com/coze-dev/coze-studio/backend/pkg/lang/conv" "github.com/coze-dev/coze-studio/backend/pkg/logs" @@ -257,11 +256,6 @@ func (t *tosClient) GetObjectUrl(ctx context.Context, objectKey string, opts ... return "", err } - ok, proxyURL := proxy.CheckIfNeedReplaceHost(ctx, output.SignedUrl) - if ok { - return proxyURL, nil - } - return output.SignedUrl, nil } diff --git a/backend/main.go b/backend/main.go index 55f1c355..3fb02c06 100755 --- a/backend/main.go +++ b/backend/main.go @@ -22,10 +22,6 @@ import ( "context" "crypto/tls" "fmt" - "log" - "net/http" - "net/http/httputil" - "net/url" "os" "runtime/debug" "strings" @@ -41,7 +37,6 @@ import ( "github.com/coze-dev/coze-studio/backend/pkg/lang/conv" "github.com/coze-dev/coze-studio/backend/pkg/lang/ternary" "github.com/coze-dev/coze-studio/backend/pkg/logs" - "github.com/coze-dev/coze-studio/backend/pkg/safego" "github.com/coze-dev/coze-studio/backend/types/consts" ) @@ -60,7 +55,6 @@ func main() { panic("InitializeInfra failed, err=" + err.Error()) } - asyncStartMinioProxyServer(ctx) startHttpServer() } @@ -160,56 +154,3 @@ func setCrashOutput() { crashFile, _ := os.Create("crash.log") debug.SetCrashOutput(crashFile, debug.CrashOptions{}) } - -// TODO: remove me later -func asyncStartMinioProxyServer(ctx context.Context) { - storageType := getEnv(consts.StorageType, "minio") - proxyURL := getEnv(consts.MinIOAPIHost, "http://localhost:9000") - - if storageType == "tos" { - proxyURL = getEnv(consts.TOSBucketEndpoint, "https://opencoze.tos-cn-beijing.volces.com") - } - - if storageType == "s3" { - proxyURL = getEnv(consts.S3BucketEndpoint, "") - } - - minioProxyEndpoint := getEnv(consts.MinIOProxyEndpoint, "") - if len(minioProxyEndpoint) == 0 { - return - } - - safego.Go(ctx, func() { - target, err := url.Parse(proxyURL) - if err != nil { - log.Fatal(err) - } - - proxy := httputil.NewSingleHostReverseProxy(target) - originDirector := proxy.Director - proxy.Director = func(req *http.Request) { - q := req.URL.Query() - q.Del("x-wf-file_name") - req.URL.RawQuery = q.Encode() - - originDirector(req) - req.Host = req.URL.Host - } - useSSL := getEnv(consts.UseSSL, "0") - if useSSL == "1" { - logs.Infof("Minio proxy server is listening on %s with SSL", minioProxyEndpoint) - err := http.ListenAndServeTLS(minioProxyEndpoint, - getEnv(consts.SSLCertFile, ""), - getEnv(consts.SSLKeyFile, ""), proxy) - if err != nil { - log.Fatal(err) - } - } else { - logs.Infof("Minio proxy server is listening on %s", minioProxyEndpoint) - err := http.ListenAndServe(minioProxyEndpoint, proxy) - if err != nil { - log.Fatal(err) - } - } - }) -}