diff --git a/backend/domain/workflow/internal/nodes/knowledge/knowledge_deleter.go b/backend/domain/workflow/internal/nodes/knowledge/knowledge_deleter.go index 9a2d4c7e..502f8f47 100644 --- a/backend/domain/workflow/internal/nodes/knowledge/knowledge_deleter.go +++ b/backend/domain/workflow/internal/nodes/knowledge/knowledge_deleter.go @@ -22,6 +22,8 @@ import ( "fmt" "strconv" + "github.com/spf13/cast" + "github.com/coze-dev/coze-studio/backend/api/model/crossdomain/knowledge" crossknowledge "github.com/coze-dev/coze-studio/backend/crossdomain/contract/knowledge" "github.com/coze-dev/coze-studio/backend/domain/workflow/entity" @@ -29,7 +31,6 @@ import ( "github.com/coze-dev/coze-studio/backend/domain/workflow/internal/canvas/convert" "github.com/coze-dev/coze-studio/backend/domain/workflow/internal/nodes" "github.com/coze-dev/coze-studio/backend/domain/workflow/internal/schema" - "github.com/spf13/cast" ) type DeleterConfig struct { diff --git a/backend/infra/contract/storage/storage.go b/backend/infra/contract/storage/storage.go index f3497561..517e0078 100644 --- a/backend/infra/contract/storage/storage.go +++ b/backend/infra/contract/storage/storage.go @@ -35,6 +35,8 @@ type Storage interface { // GetObjectUrl returns a presigned URL for the object. // The URL is valid for the specified duration. GetObjectUrl(ctx context.Context, objectKey string, opts ...GetOptFn) (string, error) + // HeadObject returns the object metadata with the specified key. + HeadObject(ctx context.Context, objectKey string, withTagging bool) (*FileInfo, error) // ListAllObjects returns all objects with the specified prefix. // It may return a large number of objects, consider using ListObjectsPaginated for better performance. ListAllObjects(ctx context.Context, prefix string, withTagging bool) ([]*FileInfo, error) @@ -66,6 +68,7 @@ type ListObjectsPaginatedOutput struct { // true: There are more results to return IsTruncated bool } + type FileInfo struct { Key string LastModified time.Time diff --git a/backend/infra/impl/storage/minio/minio.go b/backend/infra/impl/storage/minio/minio.go index 0a32953f..af67d33a 100644 --- a/backend/infra/impl/storage/minio/minio.go +++ b/backend/infra/impl/storage/minio/minio.go @@ -108,6 +108,17 @@ func (m *minioClient) test() { logs.CtxErrorf(ctx, "upload file failed: %v", err) } + f, err := m.HeadObject(ctx, objectName, true) + if err != nil { + logs.CtxErrorf(ctx, "head object failed: %v", err) + } + if f != nil { + logs.CtxInfof(ctx, "head object success, f: %v, tagging: %v", *f, f.Tagging) + } + + f, err = m.HeadObject(ctx, "not_exit.txt", true) + logs.CtxInfof(context.Background(), "HeadObject not exit success, f: %v, err: %v", f, err) + logs.CtxInfof(ctx, "upload file success") files, err := m.ListAllObjects(ctx, "test-file-", true) @@ -278,3 +289,32 @@ func (m *minioClient) ListAllObjects(ctx context.Context, prefix string, withTag return files, nil } + +func (m *minioClient) HeadObject(ctx context.Context, objectKey string, withTagging bool) (*storage.FileInfo, error) { + stat, err := m.client.StatObject(ctx, m.bucketName, objectKey, minio.StatObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, nil + } + + return nil, fmt.Errorf("HeadObject failed for key %s: %w", objectKey, err) + } + + f := &storage.FileInfo{ + Key: objectKey, + LastModified: stat.LastModified, + ETag: stat.ETag, + Size: stat.Size, + } + + if withTagging { + tags, err := m.client.GetObjectTagging(ctx, m.bucketName, objectKey, minio.GetObjectTaggingOptions{}) + if err != nil { + return nil, err + } + + f.Tagging = tags.ToMap() + } + + return f, nil +} diff --git a/backend/infra/impl/storage/s3/s3.go b/backend/infra/impl/storage/s3/s3.go index 4f62659b..e7d9d166 100644 --- a/backend/infra/impl/storage/s3/s3.go +++ b/backend/infra/impl/storage/s3/s3.go @@ -19,6 +19,7 @@ package s3 import ( "bytes" "context" + "errors" "fmt" "io" "time" @@ -360,6 +361,49 @@ func (t *s3Client) ListObjectsPaginated(ctx context.Context, input *storage.List return output, nil } +func (t *s3Client) HeadObject(ctx context.Context, objectKey string, withTagging bool) (*storage.FileInfo, error) { + obj, err := t.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(t.bucketName), + Key: aws.String(objectKey), + }) + if err != nil { + var nsk *types.NotFound + if errors.As(err, &nsk) { + return nil, nil + } + return nil, err + } + + f := &storage.FileInfo{ + Key: objectKey, + } + if obj.LastModified != nil { + f.LastModified = *obj.LastModified + } + + if obj.ETag != nil { + f.ETag = *obj.ETag + } + + if obj.ContentLength != nil { + f.Size = *obj.ContentLength + } + + if withTagging { + tagging, err := t.client.GetObjectTagging(ctx, &s3.GetObjectTaggingInput{ + Bucket: aws.String(t.bucketName), + Key: aws.String(objectKey), + }) + if err != nil { + return nil, err + } + + f.Tagging = tagsToMap(tagging.TagSet) + } + + return f, nil +} + func tagsToMap(tags []types.Tag) map[string]string { if len(tags) == 0 { return nil diff --git a/backend/infra/impl/storage/tos/tos.go b/backend/infra/impl/storage/tos/tos.go index fa5c7241..45607d6b 100644 --- a/backend/infra/impl/storage/tos/tos.go +++ b/backend/infra/impl/storage/tos/tos.go @@ -87,6 +87,17 @@ func (t *tosClient) test() { logs.CtxErrorf(context.Background(), "PutObject failed, objectKey: %s, err: %v", objectKey, err) } + f, err := t.HeadObject(ctx, objectKey, true) + if err != nil { + logs.CtxErrorf(context.Background(), "HeadObject failed, objectKey: %s, err: %v", objectKey, err) + } + if f != nil { + logs.CtxInfof(context.Background(), "HeadObject success, f: %v, tagging: %v", *f, f.Tagging) + } + + f, err = t.HeadObject(ctx, "not_exit.txt", true) + logs.CtxInfof(context.Background(), "HeadObject not exit success, f: %v, err: %v", f, err) + t.ListAllObjects(ctx, "", true) // test download @@ -360,6 +371,39 @@ func (t *tosClient) ListAllObjects(ctx context.Context, prefix string, withTaggi return files, nil } +func (t *tosClient) HeadObject(ctx context.Context, objectKey string, withTagging bool) (*storage.FileInfo, error) { + output, err := t.client.HeadObjectV2(ctx, &tos.HeadObjectV2Input{Bucket: t.bucketName, Key: objectKey}) + if err != nil { + if serverErr, ok := err.(*tos.TosServerError); ok { + if serverErr.StatusCode == http.StatusNotFound { + return nil, nil + } + } + return nil, err + } + + fileInfo := &storage.FileInfo{ + Key: objectKey, + LastModified: output.LastModified, + ETag: output.ETag, + Size: output.ContentLength, + } + + if withTagging { + tagging, err := t.client.GetObjectTagging(ctx, &tos.GetObjectTaggingInput{ + Bucket: t.bucketName, + Key: objectKey, + }) + if err != nil { + return nil, err + } + + fileInfo.Tagging = tagsToMap(tagging.TagSet.Tags) + } + + return fileInfo, nil +} + func tagsToMap(tags []tos.Tag) map[string]string { if len(tags) == 0 { return nil