From 054cd6cf3403bf7427db49675a0e531e0f93439f Mon Sep 17 00:00:00 2001 From: ayflying Date: Thu, 30 Oct 2025 12:19:31 +0800 Subject: [PATCH] =?UTF-8?q?=E9=A6=96=E6=AC=A1=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitattributes | 1 + .github/workflows/release.yml | 60 ++++ .gitignore | 19 ++ LICENSE | 21 ++ Makefile | 7 + README.md | 95 ++++++ api/hello/hello.go | 15 + api/hello/v1/hello.go | 12 + go.mod | 45 +++ go.sum | 83 +++++ hack/config.yaml | 24 ++ hack/hack-cli.mk | 20 ++ hack/hack.mk | 75 +++++ internal/cmd/cmd.go | 126 ++++++++ internal/consts/consts.go | 1 + internal/controller/hello/hello.go | 5 + internal/controller/hello/hello_new.go | 16 + internal/controller/hello/hello_v1_hello.go | 13 + internal/dao/.gitkeep | 0 internal/logic/.gitkeep | 0 internal/logic/logic.go | 9 + internal/logic/s3/s3.go | 294 ++++++++++++++++++ internal/model/.gitkeep | 0 internal/model/do/.gitkeep | 0 internal/model/entity/.gitkeep | 0 internal/packed/packed.go | 1 + internal/service/.gitkeep | 0 internal/service/s_3.go | 70 +++++ main.go | 15 + manifest/config/config.yaml | 15 + manifest/config/local.yaml | 10 + .../deploy/kustomize/base/deployment.yaml | 21 ++ .../deploy/kustomize/base/kustomization.yaml | 8 + manifest/deploy/kustomize/base/service.yaml | 12 + .../kustomize/overlays/develop/configmap.yaml | 14 + .../overlays/develop/deployment.yaml | 10 + .../overlays/develop/kustomization.yaml | 14 + manifest/docker/Dockerfile | 16 + manifest/docker/docker.sh | 8 + manifest/i18n/.gitkeep | 0 manifest/protobuf/.keep-if-necessary | 0 resource/public/html/.gitkeep | 0 resource/public/plugin/.gitkeep | 0 resource/public/resource/css/.gitkeep | 0 resource/public/resource/image/.gitkeep | 0 resource/public/resource/js/.gitkeep | 0 resource/template/.gitkeep | 0 utility/.gitkeep | 0 48 files changed, 1155 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 api/hello/hello.go create mode 100644 api/hello/v1/hello.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hack/config.yaml create mode 100644 hack/hack-cli.mk create mode 100644 hack/hack.mk create mode 100644 internal/cmd/cmd.go create mode 100644 internal/consts/consts.go create mode 100644 internal/controller/hello/hello.go create mode 100644 internal/controller/hello/hello_new.go create mode 100644 internal/controller/hello/hello_v1_hello.go create mode 100644 internal/dao/.gitkeep create mode 100644 internal/logic/.gitkeep create mode 100644 internal/logic/logic.go create mode 100644 internal/logic/s3/s3.go create mode 100644 internal/model/.gitkeep create mode 100644 internal/model/do/.gitkeep create mode 100644 internal/model/entity/.gitkeep create mode 100644 internal/packed/packed.go create mode 100644 internal/service/.gitkeep create mode 100644 internal/service/s_3.go create mode 100644 main.go create mode 100644 manifest/config/config.yaml create mode 100644 manifest/config/local.yaml create mode 100644 manifest/deploy/kustomize/base/deployment.yaml create mode 100644 manifest/deploy/kustomize/base/kustomization.yaml create mode 100644 manifest/deploy/kustomize/base/service.yaml create mode 100644 manifest/deploy/kustomize/overlays/develop/configmap.yaml create mode 100644 manifest/deploy/kustomize/overlays/develop/deployment.yaml create mode 100644 manifest/deploy/kustomize/overlays/develop/kustomization.yaml create mode 100644 manifest/docker/Dockerfile create mode 100644 manifest/docker/docker.sh create mode 100644 manifest/i18n/.gitkeep create mode 100644 manifest/protobuf/.keep-if-necessary create mode 100644 resource/public/html/.gitkeep create mode 100644 resource/public/plugin/.gitkeep create mode 100644 resource/public/resource/css/.gitkeep create mode 100644 resource/public/resource/image/.gitkeep create mode 100644 resource/public/resource/js/.gitkeep create mode 100644 resource/template/.gitkeep create mode 100644 utility/.gitkeep diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1fbf887 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* linguist-language=GO \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7f41868 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,60 @@ +name: Release + +on: + push: + tags: + - 'v*.*.*' + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.24.x' + + - name: Install gf CLI + run: | + go install github.com/gogf/gf/cmd/gf@latest + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + + - name: Verify gf + run: gf -v + + - name: Build with gf + run: gf build -ew -v "${{ github.ref_name }}" + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: uploads3-${{ github.ref_name }} + path: | + bin/** + + release: + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: uploads3-${{ github.ref_name }} + path: bin + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ github.ref_name }} + name: ${{ github.ref_name }} + draft: false + prerelease: false + files: | + bin/** \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fc2faef --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +.buildpath +.hgignore.swp +.project +.orig +.swp +.idea/ +.settings/ +.vscode/ +bin/ +**/.DS_Store +gf +main +main.exe +output/ +manifest/output/ +temp/ +temp.yaml +bin +config/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..bcc8106 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 ayflying + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2a6e6e9 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +ROOT_DIR = $(shell pwd) +NAMESPACE = "default" +DEPLOY_NAME = "template-single" +DOCKER_NAME = "template-single" + +include ./hack/hack-cli.mk +include ./hack/hack.mk \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..8d630ef --- /dev/null +++ b/README.md @@ -0,0 +1,95 @@ +# uploads3 + +一个用于批量上传本地目录文件到 S3 兼容对象存储的命令行工具,基于 GoFrame 与 minio-go SDK。 + +## 功能特性 +- 批量扫描本地目录并保持原有层级结构上传至 S3 +- 并发上传,默认 `10`,可配置,最大 `50` +- 通过 `config/local.yaml` 管理多套 S3 配置(如 MinIO、七牛 S3) +- 统一路径映射:自动将 Windows `\` 转换为对象存储使用的 `/` + +## 环境要求 +- Go `1.23+`(仓库声明了 `toolchain go1.24.8`,本地使用 Go 1.23/1.24 均可) +- 可访问你的 S3 兼容对象存储(如 MinIO、七牛云 S3) + +## 安装与构建 +- 直接构建(Windows/Linux/macOS 均可) + - Windows: `go build -o uploads3.exe .` + - Linux/macOS: `go build -o uploads3 .` +- 运行 + - Windows: `./uploads3.exe -p <本地目录> -u -w <并发数>` + - Linux/macOS: `./uploads3 -p <本地目录> -u -w <并发数>` + +可选(需安装 GoFrame CLI 与 Make):`make build` 会在 `bin/` 下生成各平台二进制。 + +## 配置说明 +应用读取 `config/local.yaml` 的 `s3` 配置,示例: + +```yaml +s3: + type: "default" # 当前使用的配置名称(default/qiniu/...) + default: + provider: "minio" + accessKey: "<你的AccessKey>" + secretKey: "<你的SecretKey>" + address: "<你的S3地址>:<端口>" # 如 minio: ay.cname.com:9000 + ssl: false # https 用 true + url: "http://host/bucket/" # 直链或 CDN 前缀,可选,用于拼接访问地址 + bucketName: "<桶名>" + qiniu: + provider: "qiniu" + accessKey: "<你的AccessKey>" + secretKey: "<你的SecretKey>" + address: "s3.cn-south-1.qiniucs.com" + ssl: true + url: "https://attachment.example.com/" + bucketName: "<桶名>" +``` + +- `s3.type` 决定使用哪套配置(上例为 `default`)。 +- `address` 是 S3/MinIO 的服务地址;`ssl` 选择是否 https。 +- `bucketName` 是目标桶名;上传时的对象 Key 将由本地路径与 `upload_path` 映射生成。 + +## 使用指南 +命令行参数: +- `-p, --path` 本地文件夹路径(必须) +- `-u, --upload_path` S3 上传根路径(必须) +- `-w, --worker` 并发数,默认 `10`,最大 `50` + +运行示例(Windows): +``` +./uploads3.exe -p D:\data\images -u cdn.yoyaworld.com/upload/images -w 20 +``` +运行示例(Linux/macOS): +``` +./uploads3 -p /data/images -u cdn.yoyaworld.com/upload/images -w 20 +``` + +### 路径映射规则 +- 工具会扫描 `path` 下所有文件,并以相对层级映射到 `upload_path`。 +- 对象 Key 生成规则:用 `upload_path` 替换本地绝对路径中的 `path` 前缀,并统一改为 `/`。 + - 例如: + - 本地文件:`D:\data\images\a\b.jpg` + - 参数:`-p D:\data\images`,`-u cdn.yoyaworld.com/upload/images` + - 对象 Key:`cdn.yoyaworld.com/upload/images/a/b.jpg` + +### 并发与性能 +- `-w` 控制同时上传的并行 worker 数,范围 `1~50`。 +- 大量小文件场景建议适度提高并发;网络与目标 S3 的限流会影响实际吞吐。 + +## 日志输出 +- 使用 GoFrame 日志,默认打印到控制台,包含当前上传进度 `(已传/总数)` 与对象 Key。 +- 失败时会记录错误日志;可根据输出排查凭据与网络问题。 + +## 常见问题 +- 连接失败:检查 `address` 与 `ssl` 是否匹配、端口是否可达。 +- 认证失败:核对 `accessKey/secretKey`、桶权限与跨区 endpoint 设置。 +- Key 不符合预期:确认命令行的 `path` 与 `upload_path` 是否书写正确,尤其是 Windows 路径需要使用转义或用双引号包裹。 + +## 开发者提示 +- 入口:`main.go` 调用 `internal/cmd/cmd.go` 的命令。 +- S3 封装:`internal/logic/s3/s3.go` 基于 `minio-go`,从 `config/local.yaml` 读取配置。 +- 若需要扩展不同提供商,可在 `config/local.yaml` 增加新节,并通过 `s3.type` 切换。 + +## 许可证 +- 使用 `MIT` 许可,详见 `LICENSE` 文件。 \ No newline at end of file diff --git a/api/hello/hello.go b/api/hello/hello.go new file mode 100644 index 0000000..5c71d2a --- /dev/null +++ b/api/hello/hello.go @@ -0,0 +1,15 @@ +// ================================================================================= +// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT. +// ================================================================================= + +package hello + +import ( + "context" + + "uploads3/api/hello/v1" +) + +type IHelloV1 interface { + Hello(ctx context.Context, req *v1.HelloReq) (res *v1.HelloRes, err error) +} diff --git a/api/hello/v1/hello.go b/api/hello/v1/hello.go new file mode 100644 index 0000000..b4dd233 --- /dev/null +++ b/api/hello/v1/hello.go @@ -0,0 +1,12 @@ +package v1 + +import ( + "github.com/gogf/gf/v2/frame/g" +) + +type HelloReq struct { + g.Meta `path:"/hello" tags:"Hello" method:"get" summary:"You first hello api"` +} +type HelloRes struct { + g.Meta `mime:"text/html" example:"string"` +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..45b4fed --- /dev/null +++ b/go.mod @@ -0,0 +1,45 @@ +module uploads3 + +go 1.23.0 + +toolchain go1.24.8 + +require ( + github.com/gogf/gf/v2 v2.7.1 + github.com/minio/minio-go/v7 v7.0.95 +) + +require ( + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/clbanning/mxj/v2 v2.7.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.1 // indirect + github.com/grokify/html-strip-tags-go v0.1.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + go.opentelemetry.io/otel v1.14.0 // indirect + go.opentelemetry.io/otel/sdk v1.14.0 // indirect + go.opentelemetry.io/otel/trace v1.14.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0d9245f --- /dev/null +++ b/go.sum @@ -0,0 +1,83 @@ +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= +github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gogf/gf/v2 v2.7.1 h1:Ukp7vzwh6VKnivEEx/xiMc61dL1HVZqCCHl//3GBRxc= +github.com/gogf/gf/v2 v2.7.1/go.mod h1:3oyGjyLHtSSo8kQ57Nj1TPdUNc0e2HS0A2J+KkXoW+I= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4= +github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= +go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= +go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= +go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= +go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/hack/config.yaml b/hack/config.yaml new file mode 100644 index 0000000..58bfda7 --- /dev/null +++ b/hack/config.yaml @@ -0,0 +1,24 @@ + +# CLI tool, only in development environment. +# https://goframe.org/docs/cli +gfcli: + build: + name: "uploads3" + arch: "amd64,arm,arm64" + system: "windows,android,linux" + mod: "none" + packSrc: "hack,resource,manifest" + version: "v1.0.0" + path: "./bin" + extra: -trimpath -ldflags="-s -w" + # cgo: true + dumpEnv: true + gen: + dao: + - link: "mysql:root:12345678@tcp(127.0.0.1:3306)/test" + descriptionTag: true + + docker: + build: "-a amd64 -s linux -p temp -ew" + tagPrefixes: + - my.image.pub/my-app \ No newline at end of file diff --git a/hack/hack-cli.mk b/hack/hack-cli.mk new file mode 100644 index 0000000..f4e2ad2 --- /dev/null +++ b/hack/hack-cli.mk @@ -0,0 +1,20 @@ + +# Install/Update to the latest CLI tool. +.PHONY: cli +cli: + @set -e; \ + wget -O gf \ + https://github.com/gogf/gf/releases/latest/download/gf_$(shell go env GOOS)_$(shell go env GOARCH) && \ + chmod +x gf && \ + ./gf install -y && \ + rm ./gf + + +# Check and install CLI tool. +.PHONY: cli.install +cli.install: + @set -e; \ + gf -v > /dev/null 2>&1 || if [[ "$?" -ne "0" ]]; then \ + echo "GoFame CLI is not installed, start proceeding auto installation..."; \ + make cli; \ + fi; \ No newline at end of file diff --git a/hack/hack.mk b/hack/hack.mk new file mode 100644 index 0000000..2f68179 --- /dev/null +++ b/hack/hack.mk @@ -0,0 +1,75 @@ +.DEFAULT_GOAL := build + +# Update GoFrame and its CLI to latest stable version. +.PHONY: up +up: cli.install + @gf up -a + +# Build binary using configuration from hack/config.yaml. +.PHONY: build +build: cli.install + @gf build -ew + +# Parse api and generate controller/sdk. +.PHONY: ctrl +ctrl: cli.install + @gf gen ctrl + +# Generate Go files for DAO/DO/Entity. +.PHONY: dao +dao: cli.install + @gf gen dao + +# Parse current project go files and generate enums go file. +.PHONY: enums +enums: cli.install + @gf gen enums + +# Generate Go files for Service. +.PHONY: service +service: cli.install + @gf gen service + + +# Build docker image. +.PHONY: image +image: cli.install + $(eval _TAG = $(shell git rev-parse --short HEAD)) +ifneq (, $(shell git status --porcelain 2>/dev/null)) + $(eval _TAG = $(_TAG).dirty) +endif + $(eval _TAG = $(if ${TAG}, ${TAG}, $(_TAG))) + $(eval _PUSH = $(if ${PUSH}, ${PUSH}, )) + @gf docker ${_PUSH} -tn $(DOCKER_NAME):${_TAG}; + + +# Build docker image and automatically push to docker repo. +.PHONY: image.push +image.push: cli.install + @make image PUSH=-p; + + +# Deploy image and yaml to current kubectl environment. +.PHONY: deploy +deploy: cli.install + $(eval _TAG = $(if ${TAG}, ${TAG}, develop)) + + @set -e; \ + mkdir -p $(ROOT_DIR)/temp/kustomize;\ + cd $(ROOT_DIR)/manifest/deploy/kustomize/overlays/${_ENV};\ + kustomize build > $(ROOT_DIR)/temp/kustomize.yaml;\ + kubectl apply -f $(ROOT_DIR)/temp/kustomize.yaml; \ + if [ $(DEPLOY_NAME) != "" ]; then \ + kubectl patch -n $(NAMESPACE) deployment/$(DEPLOY_NAME) -p "{\"spec\":{\"template\":{\"metadata\":{\"labels\":{\"date\":\"$(shell date +%s)\"}}}}}"; \ + fi; + + +# Parsing protobuf files and generating go files. +.PHONY: pb +pb: cli.install + @gf gen pb + +# Generate protobuf files for database tables. +.PHONY: pbentity +pbentity: cli.install + @gf gen pbentity \ No newline at end of file diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go new file mode 100644 index 0000000..e78383e --- /dev/null +++ b/internal/cmd/cmd.go @@ -0,0 +1,126 @@ +package cmd + +import ( + "context" + "time" + "uploads3/internal/service" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/gcmd" + "github.com/gogf/gf/v2/os/gctx" + "github.com/gogf/gf/v2/os/gfile" + "github.com/gogf/gf/v2/text/gstr" +) + +// helpDescription 定义P2P命令的详细帮助信息 +const helpDescription = ` +S3上传工具使用帮助: +-p,path 本地文件夹路径 +-u,upload_path S3上传根路径 +-w,worker 并发数,默认10,最大50 +` + +var ( + path string + uploadPath string + maxCount int + uploadCount int = 0 + // 管道长度 + workerCount = 10 // 固定并发数100 + + Main = gcmd.Command{ + Name: "main", + Usage: "main", + Brief: "start http server", + // Description 提供命令的详细描述和使用帮助 + Description: helpDescription, + Func: func(ctx context.Context, parser *gcmd.Parser) (err error) { + parser, err = gcmd.Parse(g.MapStrBool{ + "p,path": true, + "u,upload_path": true, + "w,worker": true, + }) + + //s := g.Server() + //s.Group("/", func(group *ghttp.RouterGroup) { + // group.Middleware(ghttp.MiddlewareHandlerResponse) + // group.Bind( + // hello.NewV1(), + // ) + //}) + //s.Run() + workerCount = parser.GetOpt("worker", workerCount).Int() + if workerCount > 50 { + workerCount = 50 + } + path = parser.GetOpt("path").String() + uploadPath = parser.GetOpt("upload_path").String() + if path == "" || uploadPath == "" { + g.Log().Errorf(ctx, "path 或 upload_path 为空") + return + } + + S3(path) + + return nil + }, + } +) + +var UploadTask = make(chan string, workerCount*5) + +func S3(path string) { + list, _ := gfile.ScanDirFile(path, "*", true) + maxCount = len(list) + g.Log().Debugf(gctx.New(), "当前需要处理的文件数量:%v", len(list)) + go func() { + for _, v := range list { + UploadTask <- v + } + }() + time.Sleep(1 * time.Second) + startWorkers() +} + +// 启动100个worker,持续处理任务 +func startWorkers() { + + // 启动100个worker + for i := 0; i < workerCount; i++ { + ctx := gctx.New() + go func() { + // 持续从管道取任务,直到管道关闭且所有任务处理完毕 + for { + select { + case filename := <-UploadTask: + //执行上传任务 + uploadToS3(ctx, filename) + case <-ctx.Done(): + // 上下文取消时,退出循环 + return + } + } + }() + } + + // 等待所有任务处理完毕 + for { + if len(UploadTask) == 0 { + return + } + } + +} + +func uploadToS3(ctx context.Context, filename string) { + //todo 实现上传到S3的逻辑 + uploadCount++ + + filepath := gstr.Replace(filename, path, uploadPath) + filepath = gstr.Replace(filepath, "\\", "/") + g.Log().Debugf(ctx, "(%d,%d)上传到s3:%v", uploadCount, maxCount, filepath) + //time.Sleep(grand.D(10*time.Millisecond, time.Second)) + f, _ := gfile.Open(filename) + service.S3().PutObject(ctx, f, filepath) + return +} diff --git a/internal/consts/consts.go b/internal/consts/consts.go new file mode 100644 index 0000000..d709a2b --- /dev/null +++ b/internal/consts/consts.go @@ -0,0 +1 @@ +package consts diff --git a/internal/controller/hello/hello.go b/internal/controller/hello/hello.go new file mode 100644 index 0000000..f72082f --- /dev/null +++ b/internal/controller/hello/hello.go @@ -0,0 +1,5 @@ +// ================================================================================= +// This is auto-generated by GoFrame CLI tool only once. Fill this file as you wish. +// ================================================================================= + +package hello diff --git a/internal/controller/hello/hello_new.go b/internal/controller/hello/hello_new.go new file mode 100644 index 0000000..35cd47d --- /dev/null +++ b/internal/controller/hello/hello_new.go @@ -0,0 +1,16 @@ +// ================================================================================= +// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT. +// ================================================================================= + +package hello + +import ( + "uploads3/api/hello" +) + +type ControllerV1 struct{} + +func NewV1() hello.IHelloV1 { + return &ControllerV1{} +} + diff --git a/internal/controller/hello/hello_v1_hello.go b/internal/controller/hello/hello_v1_hello.go new file mode 100644 index 0000000..2c611de --- /dev/null +++ b/internal/controller/hello/hello_v1_hello.go @@ -0,0 +1,13 @@ +package hello + +import ( + "context" + "github.com/gogf/gf/v2/frame/g" + + "uploads3/api/hello/v1" +) + +func (c *ControllerV1) Hello(ctx context.Context, req *v1.HelloReq) (res *v1.HelloRes, err error) { + g.RequestFromCtx(ctx).Response.Writeln("Hello World!") + return +} diff --git a/internal/dao/.gitkeep b/internal/dao/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/internal/logic/.gitkeep b/internal/logic/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/internal/logic/logic.go b/internal/logic/logic.go new file mode 100644 index 0000000..e3ed519 --- /dev/null +++ b/internal/logic/logic.go @@ -0,0 +1,9 @@ +// ========================================================================== +// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT. +// ========================================================================== + +package logic + +import ( + _ "uploads3/internal/logic/s3" +) diff --git a/internal/logic/s3/s3.go b/internal/logic/s3/s3.go new file mode 100644 index 0000000..dad7c6a --- /dev/null +++ b/internal/logic/s3/s3.go @@ -0,0 +1,294 @@ +package s3 + +import ( + "context" + "fmt" + "io" + "log" + "net/url" + "path" + "time" + "uploads3/internal/service" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/gcache" + "github.com/gogf/gf/v2/os/gctx" + "github.com/gogf/gf/v2/util/gconv" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// ctx 全局上下文,用于在整个包中传递请求范围的数据 +var ( +//client *minio.Client + +) + +// DataType 定义了 S3 配置的数据结构,用于存储访问 S3 所需的各种信息 +type DataType struct { + AccessKey string `json:"access_key"` // 访问 S3 的密钥 ID + SecretKey string `json:"secret_key"` // 访问 S3 的密钥 + Address string `json:"address"` // S3 服务的地址 + Ssl bool `json:"ssl"` // 是否使用 SSL 加密连接 + Url string `json:"url"` // S3 服务的访问 URL + BucketName string `json:"bucket_name"` // 默认存储桶名称 +} + +var ( + clientList = make(map[string]*minio.Client) // Minio S3 客户端实例 + cfgList = make(map[string]*DataType) // S3 配置信息 +) + +// Mod 定义了 S3 模块的结构体,包含一个 S3 客户端实例和配置信息 +type sS3 struct { + client *minio.Client + cfg *DataType +} + +func init() { + service.RegisterS3(New()) +} + +// New 根据配置创建一个新的 S3 模块实例 +// 如果未提供名称,则从配置中获取默认的 S3 类型 +// 配置错误时会触发 panic +func New(_name ...string) *sS3 { + + var name string + if len(_name) > 0 { + name = _name[0] + } else { + getName, err := g.Cfg("local").Get(gctx.New(), "s3.type") + if err != nil { + return nil + } + name = getName.String() + } + + if _, ok := cfgList[name]; ok { + return &sS3{ + client: clientList[name], + cfg: cfgList[name], + } + } + + get, err := g.Cfg("local").Get(gctx.New(), "s3."+name) + if err != nil { + panic(err.Error()) + } + var cfg *DataType + get.Scan(&cfg) + + // 使用 minio-go 创建 S3 客户端 + obj, err := minio.New( + cfg.Address, + &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: cfg.Ssl, + //BucketLookup: minio.BucketLookupPath, + }, + ) + if err != nil { + log.Fatalln(err) + } + + // 复制初始化参数 + cfgList[name] = cfg + clientList[name] = obj + + mod := &sS3{ + client: clientList[name], + cfg: cfgList[name], + } + + return mod +} + +//// GetCfg 获取当前 S3 模块的配置信息 +//func (s *sS3) GetCfg() *DataType { +// return s.cfg +//} + +// GetFileUrl 生成指向 S3 存储桶中指定文件的预签名 URL +// 预签名 URL 可用于在有限时间内访问 S3 存储桶中的文件 +// 支持从缓存中获取预签名 URL,以减少重复请求 +func (s *sS3) GetFileUrl(ctx context.Context, name string, _expires ...time.Duration) (presignedURL *url.URL, err error) { + // 设置预签名 URL 的有效期为 1 小时,可通过参数覆盖 + expires := time.Hour * 1 + if len(_expires) > 0 { + expires = _expires[0] + } + // 生成缓存键 + cacheKey := fmt.Sprintf("s3:%v:%v", name, s.cfg.BucketName) + // 尝试从缓存中获取预签名 URL + get, _ := gcache.Get(ctx, cacheKey) + if !get.IsEmpty() { + // 将缓存中的值转换为 *url.URL 类型 + err = gconv.Struct(get.Val(), &presignedURL) + return + } + // 调用 S3 客户端生成预签名 URL + presignedURL, err = s.client.PresignedGetObject(ctx, s.cfg.BucketName, name, expires, nil) + // 将生成的预签名 URL 存入缓存 + err = gcache.Set(ctx, cacheKey, presignedURL, expires) + return +} + +// PutFileUrl 生成一个用于上传文件到指定存储桶的预签名 URL +// 预签名 URL 的有效期默认为 10 分钟 +func (s *sS3) PutFileUrl(ctx context.Context, name string) (presignedURL *url.URL, err error) { + // 设置预签名 URL 的有效期为 10 分钟 + expires := time.Minute * 10 + // 调用 S3 客户端生成预签名 URL + presignedURL, err = s.client.PresignedPutObject(ctx, s.cfg.BucketName, name, expires) + return +} + +// ListBuckets 获取当前 S3 客户端可访问的所有存储桶列表 +// 出错时返回 nil +func (s *sS3) ListBuckets(ctx context.Context, ) []minio.BucketInfo { + buckets, err := s.client.ListBuckets(ctx) + if err != nil { + return nil + } + return buckets +} + +// PutObject 上传文件到指定的存储桶中 +// 支持指定文件大小,未指定时将读取文件直到结束 +func (s *sS3) PutObject(ctx context.Context, f io.Reader, name string, _size ...int64) (res minio.UploadInfo, err error) { + // 初始化文件大小为 -1,表示将读取文件至结束 + var size = int64(-1) + //if len(_size) > 0 { + // size = _size[0] + //} + // 调用 S3 客户端上传文件,设置内容类型为 "application/octet-stream" + res, err = s.client.PutObject(ctx, s.cfg.BucketName, name, f, size, minio.PutObjectOptions{ + //ContentType: "application/octet-stream", + }) + if err != nil { + // 记录上传错误日志 + g.Log().Error(ctx, err) + } + return +} + +// RemoveObject 从指定存储桶中删除指定名称的文件 +func (s *sS3) RemoveObject(ctx context.Context, name string) (err error) { + opts := minio.RemoveObjectOptions{ + ForceDelete: true, + //GovernanceBypass: true, + //VersionID: "myversionid", + } + // 调用 S3 客户端删除文件 + err = s.client.RemoveObject(ctx, s.cfg.BucketName, name, opts) + return +} + +// ListObjects 获取指定存储桶中指定前缀的文件列表 +// 返回一个包含文件信息的通道 +func (s *sS3) ListObjects(ctx context.Context, prefix string) (res <-chan minio.ObjectInfo, err error) { + // 调用 S3 客户端获取文件列表 + res = s.client.ListObjects(ctx, s.cfg.BucketName, minio.ListObjectsOptions{ + Prefix: prefix, + }) + return +} + +// StatObject 获取指定存储桶中指定文件的元数据信息 +func (s *sS3) StatObject(ctx context.Context, objectName string) (res minio.ObjectInfo, err error) { + res, err = s.client.StatObject(ctx, s.cfg.BucketName, objectName, minio.StatObjectOptions{}) + return +} + +//// SetBucketPolicy 设置指定存储桶或对象前缀的访问策略 +//// 目前使用固定的策略,可根据需求修改 +//func (s *sS3) SetBucketPolicy(ctx context.Context, prefix string) (err error) { +// // 定义访问策略 +// policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}` +// // 调用 S3 客户端设置存储桶策略 +// err = s.client.SetBucketPolicy(ctx, s.cfg.BucketName, policy) +// return +//} + +// GetUrl 获取文件的访问地址 +// 支持返回默认文件地址,根据 SSL 配置生成不同格式的 URL +func (s *sS3) GetUrl(filePath string, defaultFile ...string) (url string) { + bucketName := s.cfg.BucketName + get := s.cfg.Url + + // 如果没有指定文件路径,且提供了默认文件路径,则使用默认路径 + if filePath == "" && len(defaultFile) > 0 { + filePath = defaultFile[0] + } + + //switch s.cfg.Provider { + //case "qiniu": + // url = get + path.Join(bucketName, filePath) + //default: + // url = get + filePath + //} + url = get + filePath + + if !s.cfg.Ssl { + url = get + path.Join(bucketName, filePath) + } + + return +} + +// GetPath 从文件访问 URL 中提取文件路径 +func (s *sS3) GetPath(url string) (filePath string) { + bucketName := s.cfg.BucketName + get := s.cfg.Url + + return url[len(get+bucketName)+1:] +} + +// GetCdnUrl 通过文件名,获取直连地址 +func (s *sS3) GetCdnUrl(file string) string { + urlStr, _ := url.JoinPath(s.cfg.Url, file) + return urlStr +} + +// CopyObject 在指定存储桶内复制文件 +// bucketName 存储桶名称 +// dstStr 目标文件路径 +// srcStr 源文件路径 +// 返回操作过程中可能出现的错误 +func (s *sS3) CopyObject(ctx context.Context, dstStr string, srcStr string) (err error) { + // 定义目标文件的复制选项,包含存储桶名称和目标文件路径 + var dst = minio.CopyDestOptions{ + Bucket: s.cfg.BucketName, + Object: dstStr, + } + + // 定义源文件的复制选项,包含存储桶名称和源文件路径 + var src = minio.CopySrcOptions{ + Bucket: s.cfg.BucketName, + Object: srcStr, + } + + // 调用 S3 客户端的 CopyObject 方法,将源文件复制到目标位置 + // 忽略返回的复制信息,仅关注是否发生错误 + _, err = s.client.CopyObject(ctx, dst, src) + return +} + +// Rename 重命名文件 +func (s *sS3) Rename(ctx context.Context, oldName string, newName string) (err error) { + // 复制文件到新的名称 + g.Log().Debugf(nil, "仓库=%v,rename %s to %s", s.cfg.BucketName, oldName, newName) + err = s.CopyObject(ctx, newName, oldName) + if err != nil { + g.Log().Error(ctx, err) + return + } + // 删除原始文件 + err = s.RemoveObject(ctx, oldName) + if err != nil { + g.Log().Error(ctx, err) + return + } + return +} diff --git a/internal/model/.gitkeep b/internal/model/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/internal/model/do/.gitkeep b/internal/model/do/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/internal/model/entity/.gitkeep b/internal/model/entity/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/internal/packed/packed.go b/internal/packed/packed.go new file mode 100644 index 0000000..e20ab1e --- /dev/null +++ b/internal/packed/packed.go @@ -0,0 +1 @@ +package packed diff --git a/internal/service/.gitkeep b/internal/service/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/internal/service/s_3.go b/internal/service/s_3.go new file mode 100644 index 0000000..778ad8c --- /dev/null +++ b/internal/service/s_3.go @@ -0,0 +1,70 @@ +// ================================================================================ +// Code generated and maintained by GoFrame CLI tool. DO NOT EDIT. +// You can delete these comments if you wish manually maintain this interface file. +// ================================================================================ + +package service + +import ( + "context" + "io" + "net/url" + "time" + + "github.com/minio/minio-go/v7" +) + +type ( + IS3 interface { + // GetFileUrl 生成指向 S3 存储桶中指定文件的预签名 URL + // 预签名 URL 可用于在有限时间内访问 S3 存储桶中的文件 + // 支持从缓存中获取预签名 URL,以减少重复请求 + GetFileUrl(ctx context.Context, name string, _expires ...time.Duration) (presignedURL *url.URL, err error) + // PutFileUrl 生成一个用于上传文件到指定存储桶的预签名 URL + // 预签名 URL 的有效期默认为 10 分钟 + PutFileUrl(ctx context.Context, name string) (presignedURL *url.URL, err error) + // ListBuckets 获取当前 S3 客户端可访问的所有存储桶列表 + // 出错时返回 nil + ListBuckets(ctx context.Context) []minio.BucketInfo + // PutObject 上传文件到指定的存储桶中 + // 支持指定文件大小,未指定时将读取文件直到结束 + PutObject(ctx context.Context, f io.Reader, name string, _size ...int64) (res minio.UploadInfo, err error) + // RemoveObject 从指定存储桶中删除指定名称的文件 + RemoveObject(ctx context.Context, name string) (err error) + // ListObjects 获取指定存储桶中指定前缀的文件列表 + // 返回一个包含文件信息的通道 + ListObjects(ctx context.Context, prefix string) (res <-chan minio.ObjectInfo, err error) + // StatObject 获取指定存储桶中指定文件的元数据信息 + StatObject(ctx context.Context, objectName string) (res minio.ObjectInfo, err error) + // GetUrl 获取文件的访问地址 + // 支持返回默认文件地址,根据 SSL 配置生成不同格式的 URL + GetUrl(filePath string, defaultFile ...string) (url string) + // GetPath 从文件访问 URL 中提取文件路径 + GetPath(url string) (filePath string) + // GetCdnUrl 通过文件名,获取直连地址 + GetCdnUrl(file string) string + // CopyObject 在指定存储桶内复制文件 + // bucketName 存储桶名称 + // dstStr 目标文件路径 + // srcStr 源文件路径 + // 返回操作过程中可能出现的错误 + CopyObject(ctx context.Context, dstStr string, srcStr string) (err error) + // Rename 重命名文件 + Rename(ctx context.Context, oldName string, newName string) (err error) + } +) + +var ( + localS3 IS3 +) + +func S3() IS3 { + if localS3 == nil { + panic("implement not found for interface IS3, forgot register?") + } + return localS3 +} + +func RegisterS3(i IS3) { + localS3 = i +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..bc1b9b0 --- /dev/null +++ b/main.go @@ -0,0 +1,15 @@ +package main + +import ( + _ "uploads3/internal/packed" + + _ "uploads3/internal/logic" + + "github.com/gogf/gf/v2/os/gctx" + + "uploads3/internal/cmd" +) + +func main() { + cmd.Main.Run(gctx.GetInitCtx()) +} diff --git a/manifest/config/config.yaml b/manifest/config/config.yaml new file mode 100644 index 0000000..5fe97a7 --- /dev/null +++ b/manifest/config/config.yaml @@ -0,0 +1,15 @@ +# https://goframe.org/docs/web/server-config-file-template +server: + address: ":8000" + openapiPath: "/api.json" + swaggerPath: "/swagger" + +# https://goframe.org/docs/core/glog-config +logger: + level : "all" + stdout: true + +# https://goframe.org/docs/core/gdb-config-file +database: + default: + link: "mysql:root:12345678@tcp(127.0.0.1:3306)/test" \ No newline at end of file diff --git a/manifest/config/local.yaml b/manifest/config/local.yaml new file mode 100644 index 0000000..a0d9fdd --- /dev/null +++ b/manifest/config/local.yaml @@ -0,0 +1,10 @@ +s3: + type: "default" + default: + provider: "minio" + accessKey: "p6AuTRDWShet6ywGxzsU" + secretKey: "YPOHpSTUqFi4oZX44WsWFGvncx8iI4nc3GPzNcBf" + address: "ay.cname.com:9000" + ssl: false + url: "http://ay.cname.com:9000/cdn.yoyaworld.com/" + bucketName: "cdn.yoyaworld.com" diff --git a/manifest/deploy/kustomize/base/deployment.yaml b/manifest/deploy/kustomize/base/deployment.yaml new file mode 100644 index 0000000..28f1d69 --- /dev/null +++ b/manifest/deploy/kustomize/base/deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: template-single + labels: + app: template-single +spec: + replicas: 1 + selector: + matchLabels: + app: template-single + template: + metadata: + labels: + app: template-single + spec: + containers: + - name : main + image: template-single + imagePullPolicy: Always + diff --git a/manifest/deploy/kustomize/base/kustomization.yaml b/manifest/deploy/kustomize/base/kustomization.yaml new file mode 100644 index 0000000..302d92d --- /dev/null +++ b/manifest/deploy/kustomize/base/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- deployment.yaml +- service.yaml + + + diff --git a/manifest/deploy/kustomize/base/service.yaml b/manifest/deploy/kustomize/base/service.yaml new file mode 100644 index 0000000..608771c --- /dev/null +++ b/manifest/deploy/kustomize/base/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: template-single +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 8000 + selector: + app: template-single + diff --git a/manifest/deploy/kustomize/overlays/develop/configmap.yaml b/manifest/deploy/kustomize/overlays/develop/configmap.yaml new file mode 100644 index 0000000..3b1d0af --- /dev/null +++ b/manifest/deploy/kustomize/overlays/develop/configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: template-single-configmap +data: + config.yaml: | + server: + address: ":8000" + openapiPath: "/api.json" + swaggerPath: "/swagger" + + logger: + level : "all" + stdout: true diff --git a/manifest/deploy/kustomize/overlays/develop/deployment.yaml b/manifest/deploy/kustomize/overlays/develop/deployment.yaml new file mode 100644 index 0000000..04e4851 --- /dev/null +++ b/manifest/deploy/kustomize/overlays/develop/deployment.yaml @@ -0,0 +1,10 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: template-single +spec: + template: + spec: + containers: + - name : main + image: template-single:develop \ No newline at end of file diff --git a/manifest/deploy/kustomize/overlays/develop/kustomization.yaml b/manifest/deploy/kustomize/overlays/develop/kustomization.yaml new file mode 100644 index 0000000..4731c47 --- /dev/null +++ b/manifest/deploy/kustomize/overlays/develop/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base +- configmap.yaml + +patchesStrategicMerge: +- deployment.yaml + +namespace: default + + + diff --git a/manifest/docker/Dockerfile b/manifest/docker/Dockerfile new file mode 100644 index 0000000..d3abe8f --- /dev/null +++ b/manifest/docker/Dockerfile @@ -0,0 +1,16 @@ +FROM loads/alpine:3.8 + +############################################################################### +# INSTALLATION +############################################################################### + +ENV WORKDIR /app +ADD resource $WORKDIR/ +ADD ./temp/linux_amd64/main $WORKDIR/main +RUN chmod +x $WORKDIR/main + +############################################################################### +# START +############################################################################### +WORKDIR $WORKDIR +CMD ./main diff --git a/manifest/docker/docker.sh b/manifest/docker/docker.sh new file mode 100644 index 0000000..ff393f9 --- /dev/null +++ b/manifest/docker/docker.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# This shell is executed before docker build. + + + + + diff --git a/manifest/i18n/.gitkeep b/manifest/i18n/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/manifest/protobuf/.keep-if-necessary b/manifest/protobuf/.keep-if-necessary new file mode 100644 index 0000000..e69de29 diff --git a/resource/public/html/.gitkeep b/resource/public/html/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/resource/public/plugin/.gitkeep b/resource/public/plugin/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/resource/public/resource/css/.gitkeep b/resource/public/resource/css/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/resource/public/resource/image/.gitkeep b/resource/public/resource/image/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/resource/public/resource/js/.gitkeep b/resource/public/resource/js/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/resource/template/.gitkeep b/resource/template/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/utility/.gitkeep b/utility/.gitkeep new file mode 100644 index 0000000..e69de29