diff options
| author | tsne <tsne.dev@outlook.com> | 2026-07-06 16:52:20 +0200 |
|---|---|---|
| committer | tsne <tsne.dev@outlook.com> | 2026-07-06 21:37:42 +0200 |
| commit | 84da43998f7cc42cc72652333ac6e3d293abb23e (patch) | |
| tree | 6610f82df854ae86f7f34152bb05fc712ef6f314 | |
| download | hopper-84da43998f7cc42cc72652333ac6e3d293abb23e.tar.gz | |
39 files changed, 4870 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5e56e04 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/bin @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Thomas Nitsche + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..33c4df8 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +BINDIR := bin +GOCMD := go +PREFIX := /usr/local + + +all: build + +build: + $(GOCMD) build -o $(BINDIR)/hopper ./cmd/hopper + +install: build + install -d $(DESTDIR)$(PREFIX)/bin + install -m 0755 $(BINDIR)/hopper $(DESTDIR)$(PREFIX)/bin/hopper + +uninstall: + rm -f $(DESTDIR)$(PREFIX)/bin/hopper + +test: + $(GOCMD) test ./... + +freebsd: + GOOS=freebsd GOARCH=amd64 $(GOCMD) build -o $(BINDIR)/hopper + +clean: + rm -rf $(BINDIR) diff --git a/README.md b/README.md new file mode 100644 index 0000000..714d523 --- /dev/null +++ b/README.md @@ -0,0 +1,168 @@ +# hopper + +`hopper` is a small command-line tool that deploys content-hashed web assets +from a gzipped tarball to [Bunny CDN Storage](https://bunny.net/storage/), and +prunes obsolete files from a storage zone. + + + +## How hopper Thinks About Your Assets + +Modern site builders produce content-hashed asset filenames. The name embeds +a hash of the file's contents, e.g. `assets/app.a1b2c3.js`. Because the hash +changes whenever the bytes change, a given filename always maps to exactly the +same bytes forever. `hopper` calls these files *immutable assets*. + +The exception is *HTML files* (e.g. `index.html`, `404.html`). They are the +*entry points* that reference the immutable assets, they keep the same name +across releases, and their contents change on every deploy. `hopper` calls +these *mutable assets*. + +`hopper` categorizes every `*.html` file (case-insensitive) as a *mutable +asset* and every other file as an *immutable asset*. To save bandwidth, +immutable assets are uploaded only if the zone does not contain this asset +already or the sizes differ. Mutable assets, on the other hand, are always +uploaded, and uploaded only after all immutable assets succeeded. This ensures +that an application stays in a consistent state when an immutable asset upload +fails. Re-running the upload simply resumes. + +Because uploading accumulates files on the storage zone, `hopper` can prune old +files. Immutable assets that are not touched for a very long time still keep +their old modification date, so pruning files only on their age would be +dangerous. Instead pruning deletes only old files which are not in the current +release. + + + +## Installation + +`hopper` is distributed as source and built with the Go toolchain, which +compiles into a single static binary. To install `hopper` clone the repository +and run `make install` in its root. This installs the binary to +`$PREFIX/bin/hopper` (`$PREFIX` defaults to `/usr/local`). + + + +## Quickstart + +Deploy a release archive to a Bunny storage zone: + +```sh +export BUNNY_ZONE=my-zone +export BUNNY_ACCESS_KEY=your-storage-access-key + +hopper push dist.tar.gz +``` + +Later, clean up files from old releases that are no longer referenced and have +aged past the grace window (default: 30 days): + +```sh +hopper prune dist.tar.gz +# or define a custom grace window +hopper prune --older-than=90d dist.tar.gz +``` + +Always preview a destructive run first: + +```sh +hopper prune --dryrun dist.tar.gz +``` + + + +## Usage + +``` +hopper [global flags] ⟨subcommand⟩ [flags] ⟨args⟩ +``` + +`hopper` has exactly two subcommands: `push` and `prune`. The position of the +global flags are important. They must be before any subcommand. + + +### Global Flags + +Global flags must be defined before any subcommand. Some flags also have a +fallback to an environment variable, so command line arguments always overwrite +the environment variables. An empty environment variable is treated as unset. + +| Flag | Env Fallback | Default | Meaning | +|---------------------|-------------------------|--------------------------|---------------------------------------------------| +| `--zone` | `BUNNY_ZONE` | *(none)* | The Bunny storage zone name. | +| `--endpoint` | `BUNNY_ENDPOINT` | `storage.bunnycdn.com` | The Bunny storage endpoint hostname. | +| `--access-key-file` | `BUNNY_ACCESS_KEY_FILE` | *(none)* | Path to a file containing the storage access key. | +| `--verbose` | *(none)* | `false` | Extra detail (listing walk, retries, checksums). | +| `--quiet` | *(none)* | `false` | Print only the final summary and errors. | + +*Note:* The access key can also be provided directly using the `BUNNY_ACCESS_KEY` +environment variable. Both, `BUNNY_ACCESS_KEY_FILE` and `--access-key-file` can +overwrite this access key. + + +### Subcommand `push` + +``` +hopper [global flags] push [flags] ⟨source⟩ +``` + +Uploads a new release to the storage zone. It iterates over all files of the +`⟨source⟩`, uploads all new immutable assets, and then uploads all mutable +assets. After successfully uploading the assets, `hopper` checks the ratio of +the number of files contained in the storage zone to the number of files in +the given source. If this ratio becomes too large, a hint will be printed +suggesting to run `hopper prune`. + +*Note:* An empty source uploads nothing and prints a warning. It is not +considered an error (i.e. it exits with code 0), because the operation is +non-destructive. + +#### Flags +| Flag | Meaning | +|-----------------|---------------------------------------------------------------------------------------------------| +| `--concurrency` | The number of concurrent uploads. By default it auto-detects a value based on the number of CPUs. | +| `--prefix` | A path prefix in the storage zone. Every asset is uploaded into this directory. Default: none. | +| `--root` | The subtree of the source that should be uploaded. Default: All source files. | +| `--dryrun` | Run the command without any mutating operations. This flag can be used to verify the upload. | + + +### Subcommand `prune` + +``` +hopper [global flags] prune [flags] ⟨args⟩ +``` + +Deletes obsolete files. It builds the *live set* from the given source and a +view of the storage zone. It then deletes every remote file that is not in the +live set and whose last modification is older than a certain grace window. +Finally, it removes all empty directories it can find (failures here do not +affect the exit code). All deletes are independent. If one fails, `hopper` +keeps going deleting all other obsolete files. + +*Note:* If the live set is empty, the prune operation refuses to run, because +it could delete the entire zone. In this case, please check your `--root` flag +and the source. + +#### Flags +| Flag | Meaning | +|-----------------|--------------------------------------------------------------------------------------------------------------------------| +| `--concurrency` | The number of concurrent deletes. By default it auto-detects a value based on the number of CPUs. | +| `--prefix` | A path prefix in the storage zone. Only assets in this directory are deleted. Default: none. | +| `--root` | The subtree of the source that defines the live set. Default: All source files. | +| `--older-than` | The grace window that is used to determine obsolete files. Only files older than this will be deleted. Default: 30 days. | +| `--dryrun` | Run the command without any mutating operations. This flag can be used to verify the deletes. | + + + +## Retry Policy + +If Bunny rate limits your requests, `hopper` receives a `429` and reports it. +In case the response contains the `Retry-After` header, `hopper` honors this +header and automatically retries up to three times. Every other HTTP failure +or a `429` without `Retry-After` fails immediately and reports an error message. + + + +## License + +MIT. diff --git a/cmd/hopper/main.go b/cmd/hopper/main.go new file mode 100644 index 0000000..cc58d0d --- /dev/null +++ b/cmd/hopper/main.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "os" + "os/signal" + "strconv" + "sync/atomic" + "syscall" + + "tsne.dev/hopper/internal/cli" + "tsne.dev/hopper/internal/log" +) + +func main() { + log.Init(os.Stdout, os.Stderr) + + sigch := make(chan os.Signal, 1) + signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM) + + ctx, cancel := context.WithCancelCause(context.Background()) + defer cancel(nil) + + var receivedSignal int64 + go func() { + sig := (<-sigch).(syscall.Signal) + atomic.StoreInt64(&receivedSignal, int64(sig)) + + var signame string + switch sig { + case syscall.SIGINT: + signame = "SIGINT" + case syscall.SIGTERM: + signame = "SIGTERM" + default: + signame = "signal " + strconv.Itoa(int(sig)) + } + + log.Print("received %s", signame) + cancel(cli.CancellationError("operation aborted (" + signame + ")")) + }() + + code := cli.Run(ctx, os.Args[1:]) + if sig := atomic.LoadInt64(&receivedSignal); sig != 0 { + code = 128 + int(sig) + } + os.Exit(code) +} diff --git a/docs/bunny-api.json b/docs/bunny-api.json new file mode 100644 index 0000000..6453f48 --- /dev/null +++ b/docs/bunny-api.json @@ -0,0 +1,413 @@ +{ + "openapi": "3.0.0", + "info": { + "version": "1.0.0", + "title": "bunny.net Edge Storage API", + "description": "\n\nLearn how to use the [bunny.net](https://bunny.net \"bunny.net - The content delivery platform that truly hops\") Edge Storage API. It allows you to easily upload, download and delete files with just a few lines of code.\n<h2>API libraries:</h2> \nThese libraries allow quick integration for bunny.net Edge Storage.\n\n<ul>\n <li><a rel=\"nofollow\" href=\"https://github.com/BunnyWay/BunnyCDN.PHP.Storage\">BunnyCDN.PHP.Storage</a> - Official PHP storage library</li>\n <li><a rel=\"nofollow\" href=\"https://github.com/BunnyWay/BunnyCDN.Net.Storage\">BunnyCDN.Net.Storage</a> - Official .NET storage library</li>\n <li><a rel=\"nofollow\" href=\"https://github.com/BunnyWay/BunnyCDN.Java.Storage\">BunnyCDN.Java.Storage</a> - Official Java storage library</li>\n \n <li><a rel=\"nofollow\" href=\"https://github.com/NiveditJain/BunnyCDNStorageLib\">Python Storage Library</a> - (Unofficial)</li>\n</ul>\n\n<h2>Storage Endpoints:</h2> \n\nThe storage API endpoint depends on the primary storage region of your storage zone. You can also find this in the FTP & HTTP API Information of your storage zone.\n<ul>\n <li><strong>Falkenstein</strong>: storage.bunnycdn.com</li>\n <li><strong>New York</strong>: la.storage.bunnycdn.com</li>\n <li><strong>Los Angeles</strong>: la.storage.bunnycdn.com</li>\n <li><strong>Singapore</strong>: sg.storage.bunnycdn.com</li>\n <li><strong>Sydney</strong>: syd.storage.bunnycdn.com</li>\n</ul>\n", + "termsOfService": "http://swagger.io/terms/", + "contact": { + "name": "bunny.net API Team", + "email": "support@bunny.net", + "url": "http://bunny.net" + }, + "license": { + "name": "MIT", + "url": "http://github.com/gruntjs/grunt/blob/master/LICENSE-MIT" + } + }, + "security": [ + { + "AccessKeyAuthentication": [] + } + ], + "paths": { + "/{storageZoneName}/{path}/{fileName}": { + "get": { + "tags": [ + "Manage Files" + ], + "summary": "Download File", + "description": "Returns the stored file at the given path. If the file does not exist, a 404 response will be returned.\n", + "parameters": [ + { + "name": "storageZoneName", + "in": "path", + "description": "The name of your storage zone where you are connecting to.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "path", + "in": "path", + "description": "The directory path to your file. If this is the root of your storage zone, you can ignore this parameter.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "fileName", + "in": "path", + "description": "The name of the file that you wish to download.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "AccessKey", + "in": "header", + "required": true, + "description": "The API AccessKey used for authentication.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Returns the file stored at the given path." + }, + "404": { + "description": "The file on the given path does not exist.", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ErrorObject" + } + } + } + } + } + } + }, + "put": { + "tags": [ + "Manage Files" + ], + "summary": "Upload File", + "description": "Upload a file to a storage zone based on the URL path. If the directory tree does not exist, it will be created automatically. **The file content should be sent as the body of the request without any type of encoding.**", + "responses": { + "201": { + "description": "The file upload was successful." + }, + "400": { + "description": "Upload failed." + } + }, + "requestBody": { + "description": "Raw request body should contain the contents of the file. This should be raw file data without any sort of encoding.", + "required": true, + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "parameters": [ + { + "name": "storageZoneName", + "in": "path", + "description": "he name of your storage zone where you are connecting to.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "path", + "in": "path", + "description": "The directory path to where your file will be stored. If this is the root of your storage zone, you can ignore this parameter.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "fileName", + "in": "path", + "description": "The name that the file will be uploaded as.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "Checksum", + "in": "header", + "required": false, + "description": "The SHA256 checksum of the uploaded content in HEX format (UPPERCASE). The server will compare the final SHA256 to the checksum and reject the request in case the checksums do not match.", + "schema": { + "type": "string" + }, + "example": "D7A8FBB307D7809469CA9ABCB0082E4F8D5651E46D3CDB762D02D0BF37C9E592" + }, + { + "name": "AccessKey", + "in": "header", + "required": true, + "description": "The API AccessKey used for authentication.", + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "tags": [ + "Manage Files" + ], + "summary": "Delete File", + "description": "Delete an object from the storage zone. In case the object is a directory all the data in it will be recursively deleted as well.", + "parameters": [ + { + "name": "storageZoneName", + "in": "path", + "description": "The name of your storage zone where you are connecting to.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "path", + "in": "path", + "description": "The directory path to your file. If this is the root of your storage zone, you can ignore this parameter.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "fileName", + "in": "path", + "description": "The name of the file that you wish to delete.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "AccessKey", + "in": "header", + "required": true, + "description": "The API AccessKey used for authentication.", + "schema": { + "type": "string" + } + }, + { + "name": "allowRootDelete", + "in": "query", + "required": false, + "description": "Set to `true` to bypass the root directory deletion guard and delete the storage zone root (`/`).", + "schema": { + "type": "string", + "enum": [ + "true" + ] + } + }, + { + "name": "allowRootDelete", + "in": "header", + "required": false, + "description": "Set to `true` to bypass the root directory deletion guard and delete the storage zone root.", + "schema": { + "type": "string", + "enum": [ + "true" + ] + } + } + ], + "responses": { + "200": { + "description": "Object was successfully deleted" + }, + "400": { + "description": "Object delete failed" + } + } + } + }, + "/{storageZoneName}/{path}/": { + "get": { + "tags": [ + "Browse Files" + ], + "summary": "List Files", + "description": "Retrieve a list of files and directories located in the given directory.\n", + "parameters": [ + { + "name": "storageZoneName", + "in": "path", + "description": "he name of your storage zone where you are connecting to.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "path", + "in": "path", + "description": "The directory path that you want to list.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "A JSON encoded array of objects.", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/StorageObject" + } + } + }, + "application/json": { + "examples": { + "response": { + "value": [ + { + "Guid": "d6445d80-a797-4535-bf0e-d3819bcdf928", + "StorageZoneName": "testfiles", + "Path": "/testfiles/images/", + "ObjectName": "300kb.jpg", + "Length": 300000, + "LastChanged": "2017-03-10T03:06:48.203", + "IsDirectory": false, + "ServerId": 24, + "UserId": "6eb577ca-556c-4e23-970c-a5afde191042", + "DateCreated": "2017-03-10T03:06:48.203", + "StorageZoneId": 18201 + }, + { + "Guid": "bc0f1128-fa88-4c53-a09d-fc59b442391d", + "StorageZoneName": "testfiles", + "Path": "/testfiles/images/", + "ObjectName": "300kb.jpg", + "Length": 300000, + "LastChanged": "2017-03-10T03:06:48.203", + "IsDirectory": false, + "ServerId": 24, + "UserId": "1e733d2d-c0a4-4462-be8e-f8e63888d154", + "DateCreated": "2017-03-10T03:06:48.203", + "StorageZoneId": 18201 + } + ] + } + } + } + } + } + } + } + } + }, + "servers": [ + { + "url": "https://storage.bunnycdn.com/" + } + ], + "components": { + "securitySchemes": { + "AccessKeyAuthentication": { + "type": "apiKey", + "in": "header", + "name": "AccessKey", + "description": "The storage zone password also doubles as your API key. You can find it in your storage zone details page in the bunny.net dashboard." + } + }, + "schemas": { + "ErrorObject": { + "type": "object", + "properties": { + "HttpCode": { + "type": "integer", + "description": "The error status code of the request" + }, + "Message": { + "type": "string", + "description": "The error description message" + } + } + }, + "StorageObject": { + "type": "object", + "properties": { + "Guid": { + "type": "string", + "format": "guid", + "description": "The unique Guid that represents the file" + }, + "StorageZoneName": { + "type": "string", + "description": "The full hostname domain value" + }, + "Path": { + "type": "string", + "description": "True directory path where the object is located" + }, + "ObjectName": { + "type": "string", + "description": "The name of the object, file or directory name" + }, + "Length": { + "type": "integer", + "format": "int64", + "description": "The size of the file" + }, + "LastChanged": { + "type": "string", + "format": "datetime", + "description": "The time when the file was last changed" + }, + "IsDirectory": { + "type": "boolean", + "description": "True if this is a directory object, false if it's a file" + }, + "ServerId": { + "type": "integer", + "description": "The ID of the physical server that holds the file" + }, + "UserId": { + "type": "string", + "description": "The ID of the user that holds the file" + }, + "DateCreated": { + "type": "string", + "format": "datetime", + "description": "The date and time when the object was first created" + }, + "StorageZoneId": { + "type": "integer", + "description": "The storage zone ID to which the file belongs" + } + } + } + } + }, + "x-explorer-enabled": true, + "x-proxy-enabled": true, + "x-samples-enabled": true, + "x-samples-languages": [ + "curl", + "node", + "ruby", + "javascript", + "python" + ] +} @@ -0,0 +1,5 @@ +module tsne.dev/hopper + +go 1.24 + +require github.com/spf13/pflag v1.0.10 @@ -0,0 +1,2 @@ +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= diff --git a/internal/bunny/api.go b/internal/bunny/api.go new file mode 100644 index 0000000..1ffc1b5 --- /dev/null +++ b/internal/bunny/api.go @@ -0,0 +1,170 @@ +package bunny + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "tsne.dev/hopper/internal/log" +) + +// Object is a single node returned by a directory listing. +type Object struct { + Path string + Size int + IsDirectory bool + LastChanged time.Time +} + +type API struct { + endpoint string // hostname only (normalized, no scheme/trailing slash) + zone string + accessKey string + maxRetries int + http *http.Client +} + +func NewAPI(endpoint, zone, accessKey string) *API { + // normalize endpoint + endpoint = strings.TrimPrefix(endpoint, "https://") + endpoint = strings.TrimPrefix(endpoint, "http://") + endpoint = strings.TrimRight(endpoint, "/") + + return &API{ + endpoint: endpoint, + zone: zone, + accessKey: accessKey, + maxRetries: 3, + http: &http.Client{}, + } +} + +// do executes req under the shared retry policy and returns the final +// response (2xx) or a typed *apiError. It owns the universal parts: the +// AccessKey header, the 429+Retry-After retry loop, and status classification. +// Callers own method, body, and op-specific headers, and must read/close the +// returned body. method and path on any apiError are taken from req. The body +// is reset from req.GetBody before each attempt; callers that send a body MUST +// build the request with bytes.NewReader (so GetBody is set) or retries would +// resend an empty body. +func (api *API) do(ctx context.Context, req *http.Request) (*http.Response, error) { + req.Header.Set("AccessKey", api.accessKey) + + attempt := 0 + for { + attempt++ + if req.GetBody != nil { + body, err := req.GetBody() + if err != nil { + return nil, newAPIError(errorKindNetwork, err, req, 0) + } + req.Body = body + } + + resp, err := api.http.Do(req) + if err != nil { + if ctxErr := context.Cause(ctx); ctxErr != nil { + return nil, ctxErr + } + return nil, newAPIError(errorKindNetwork, err, req, 0) + } + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return resp, nil + } + + bunnyMsg := readResponseError(resp.Body) + resp.Body.Close() + + if resp.StatusCode == http.StatusTooManyRequests { + ra := resp.Header.Get("Retry-After") + if ra == "" { + return nil, newAPIError(errorKindRateLimited, errors.New("status 429, no retry-after header received"), req, 0) + } + if attempt > api.maxRetries { + return nil, newAPIError(errorKindRateLimited, errors.New("status 429, retries exhausted"), req, 0) + } + d, perr := parseRetryAfter(ra) + if perr != nil { + return nil, newAPIError(errorKindRateLimited, errors.New("status 429, "+perr.Error()), req, 0) + } + // Surface the cool-down so a long Retry-After wait doesn't look like a + // hang. Always warned (never suppressed): being rate-limited is + // abnormal and the wait is user-visible time. + log.Warn("rate-limited by Bunny (retry in %s; attempt %d/%d)", d.Round(time.Second), attempt, api.maxRetries) + select { + case <-ctx.Done(): + return nil, context.Cause(ctx) + case <-time.After(d): + } + continue + } + + errKind := errorKindFromStatus(resp.StatusCode) + var cause error + if bunnyMsg != "" { + cause = fmt.Errorf("http error %d: %s", resp.StatusCode, bunnyMsg) + } else { + cause = fmt.Errorf("http error %d", resp.StatusCode) + } + return nil, newAPIError(errKind, cause, req, resp.StatusCode) + } +} + +// readResponseError best-effort extracts Bunny's ErrorObject.Message from a +// non-2xx response body. It fully drains (bounded) the body so the connection +// can be reused, and returns "" when the body is absent, unparseable, or +// carries no Message. It never errors: enrichment is optional detail. +func readResponseError(body io.Reader) string { + data, err := io.ReadAll(io.LimitReader(body, 8<<10)) // 8 KiB + if err != nil || len(data) == 0 { + return "" + } + var e struct { + Message string `json:"Message"` + } + if err := json.Unmarshal(data, &e); err != nil { + return "" + } + return strings.TrimSpace(e.Message) +} + +func (api *API) url(path string) string { + return "https://" + api.endpoint + "/" + api.zone + "/" + path +} + +// parseTime parses Bunny's LastChanged timestamp, which is NOT RFC 3339 +// (it carries no timezone, e.g. "2023-06-29T12:00:00.000"), so it cannot be +// unmarshaled into a time.Time directly. +func parseTime(v string) time.Time { + if v == "" { + return time.Time{} + } + if t, err := time.Parse("2006-01-02T15:04:05.999", v); err == nil { + return t.Local() + } + if t, err := time.Parse(time.RFC3339, v); err == nil { + return t.Local() + } + return time.Time{} +} + +// parseRetryAfter handles the delay-seconds form and the HTTP-date form. +func parseRetryAfter(v string) (time.Duration, error) { + if secs, err := strconv.Atoi(v); err == nil { + if secs < 0 { + return 0, fmt.Errorf("negative retry-after header") + } + return time.Duration(secs) * time.Second, nil + } + if t, err := http.ParseTime(v); err == nil { + return max(0, time.Until(t)), nil + } + return 0, fmt.Errorf("unparseable retry-after header %q", v) +} diff --git a/internal/bunny/api_delete.go b/internal/bunny/api_delete.go new file mode 100644 index 0000000..868b0f3 --- /dev/null +++ b/internal/bunny/api_delete.go @@ -0,0 +1,46 @@ +package bunny + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" +) + +// Delete removes a single object with a raw DELETE, applying the shared retry +// policy. +func (api *API) Delete(ctx context.Context, path string) error { + return api.delete(ctx, api.url(path)) +} + +// DeleteDir removes a directory. Bunny's storage endpoints identify a directory +// by a trailing slash on the URL (the same quirk listDir relies on), so we +// append it here rather than leak the convention to callers. The delete is +// recursive. +// +// CAUTION: The delete is recursive AND prune decides emptiness from the +// pre-delete listing snapshot. If another actor uploads a file under this directory +// between the snapshot and this call, the recursive delete will remove it. +func (api *API) DeleteDir(ctx context.Context, path string) error { + u := api.url(path) + if !strings.HasSuffix(u, "/") { + u += "/" + } + return api.delete(ctx, u) +} + +func (api *API) delete(ctx context.Context, u string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, u, nil) + if err != nil { + return fmt.Errorf("build request: %w", err) + } + + resp, err := api.do(ctx, req) + if err != nil { + return err + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + return nil +} diff --git a/internal/bunny/api_list.go b/internal/bunny/api_list.go new file mode 100644 index 0000000..57a7817 --- /dev/null +++ b/internal/bunny/api_list.go @@ -0,0 +1,152 @@ +package bunny + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "tsne.dev/hopper/internal/log" +) + +// Iterator is an old-school pull iterator over the recursive remote listing under +// a starting directory. It yields every entry (files AND directories; a +// directory has Size 0) so consumers can build whatever structure they need: +// push collects files and verifies no file collides with a remote directory; +// prune derives orphans and empty directories. Usage: +// +// it := client.List(ctx, dir) +// for it.Next() { +// rec := it.Object() +// } +// if err := it.Err(); err != nil { ... } +// +// The walk is iterative: a stack holds directories still to list, each listed +// on demand. A 404 listing is an empty subtree (skipped, not fatal); any other +// listing failure stops iteration and surfaces via Err. +type Iterator struct { + ctx context.Context + listDir func(context.Context, string) ([]Object, error) + stack []string + pending []Object + cur Object + err error + done bool +} + +// NewIterator builds an Iterator rooted at dir, listing each directory via listDir. +func NewIterator(ctx context.Context, dir string, listDir func(context.Context, string) ([]Object, error)) *Iterator { + stack := make([]string, 0, 4) + stack = append(stack, dir) + return &Iterator{ctx: ctx, listDir: listDir, stack: stack} +} + +// Next advances to the next entry, returning false when the walk is exhausted +// or a listing failed (check Err). Directory entries are yielded and also queued +// for descent. +func (it *Iterator) Next() bool { + if it.err != nil || it.done { + return false + } + for { + if len(it.pending) > 0 { + o := it.pending[0] + it.pending = it.pending[1:] + if o.IsDirectory { + it.stack = append(it.stack, o.Path) + } + it.cur = o + return true + } + if len(it.stack) == 0 { + it.done = true + return false + } + dir := it.stack[len(it.stack)-1] + it.stack = it.stack[:len(it.stack)-1] + objs, err := it.listDir(it.ctx, dir) + if err != nil { + var apiErr *apiError + if errors.As(err, &apiErr) && apiErr.status == http.StatusNotFound { + continue // 404 = empty subtree + } + it.err = err + return false + } + it.pending = objs + } +} + +// Object returns the entry reached by the most recent Next. +func (it *Iterator) Object() Object { + return it.cur +} + +// Err returns the listing failure that stopped iteration, if any. +func (it *Iterator) Err() error { + return it.err +} + +// List returns an Iterator over the recursive listing rooted at dir. An empty +// dir denotes zone root. +func (api *API) List(ctx context.Context, dir string) *Iterator { + return NewIterator(ctx, dir, api.listDir) +} + +// listDir returns the immediate children of dir. Bunny's list endpoint is +// per-directory and expects a trailing slash on the directory URL. Returned +// Object.Path is the full storage key relative to the zone (matching source +// keys), so the skip-check can match it. +func (api *API) listDir(ctx context.Context, dir string) ([]Object, error) { + if dir == "" { + log.PrintVerbose("listing zone root") + } else { + log.PrintVerbose("listing %s", dir) + } + + u := api.url(dir) + if !strings.HasSuffix(u, "/") { + u += "/" + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + + resp, err := api.do(ctx, req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var raw []listObject + if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + return nil, newAPIError(errorKindInternal, errors.New("failed to decode list response ("+err.Error()+")"), req, resp.StatusCode) + } + + prefix := "/" + api.zone + "/" + objs := make([]Object, len(raw)) + for i, o := range raw { + objs[i] = Object{ + Path: strings.TrimPrefix(o.Path, prefix) + o.ObjectName, + Size: o.Length, + IsDirectory: o.IsDirectory, + LastChanged: parseTime(o.LastChanged), + } + } + return objs, nil +} + +// listObject mirrors a single entry of Bunny's per-directory list response. The +// full storage key is Path+ObjectName (Path includes the zone and trailing +// slash); see List. +type listObject struct { + Path string `json:"Path"` + ObjectName string `json:"ObjectName"` + Length int `json:"Length"` + LastChanged string `json:"LastChanged"` + IsDirectory bool `json:"IsDirectory"` +} diff --git a/internal/bunny/api_test.go b/internal/bunny/api_test.go new file mode 100644 index 0000000..e74a3b9 --- /dev/null +++ b/internal/bunny/api_test.go @@ -0,0 +1,349 @@ +package bunny + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func newTestAPI(srvURL string) *API { + api := NewAPI(srvURL, "zone", "key") + api.http = &http.Client{Transport: rewriteTransport{}} + return api +} + +// rewriteTransport downgrades https://host/... to http://host/... for tests. +type rewriteTransport struct{} + +func (rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.URL.Scheme = "http" + return http.DefaultTransport.RoundTrip(req) +} + +func TestUploadSuccess(t *testing.T) { + var gotKey, gotChecksum, gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotKey = r.Header.Get("AccessKey") + gotChecksum = r.Header.Get("Checksum") + gotPath = r.URL.Path + if r.Header.Get("Content-Type") != "" { + t.Errorf("unexpected Content-Type %q", r.Header.Get("Content-Type")) + } + w.WriteHeader(201) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + if err := api.Upload(context.Background(), "a/b.js", []byte("data")); err != nil { + t.Fatalf("Upload: %v", err) + } + sum := sha256.Sum256([]byte("data")) + wantChecksum := strings.ToUpper(hex.EncodeToString(sum[:])) + if gotKey != "key" || gotChecksum != wantChecksum { + t.Fatalf("headers: key=%q checksum=%q (want %q)", gotKey, gotChecksum, wantChecksum) + } + if gotPath != "/zone/a/b.js" { + t.Fatalf("path = %q", gotPath) + } +} + +func TestUploadRetriesOn429WithRetryAfter(t *testing.T) { + var n int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if atomic.AddInt32(&n, 1) <= 2 { + w.Header().Set("Retry-After", "0") + w.WriteHeader(429) + return + } + w.WriteHeader(200) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + if err := api.Upload(context.Background(), "x", []byte("d")); err != nil { + t.Fatalf("Upload: %v", err) + } + if n != 3 { + t.Fatalf("attempts = %d, want 3", n) + } +} + +func TestUploadFailsOn429NoRetryAfter(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(429) + })) + defer srv.Close() + api := newTestAPI(srv.URL) + if err := api.Upload(context.Background(), "x", []byte("d")); err == nil { + t.Fatal("expected error") + } +} + +func TestUploadFailsOn5xx(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer srv.Close() + api := newTestAPI(srv.URL) + if err := api.Upload(context.Background(), "x", []byte("d")); err == nil { + t.Fatal("expected error") + } +} + +func TestUploadRetriesExhausted(t *testing.T) { + var n int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&n, 1) + w.Header().Set("Retry-After", "0") + w.WriteHeader(429) + })) + defer srv.Close() + api := newTestAPI(srv.URL) + if err := api.Upload(context.Background(), "x", []byte("d")); err == nil { + t.Fatal("expected error") + } + if n != 4 { // initial + 3 retries + t.Fatalf("attempts = %d, want 4", n) + } +} + +func TestUploadClassifiesStatus(t *testing.T) { + cases := []struct { + status int + kind errorKind + }{ + {http.StatusUnauthorized, errorKindAuth}, + {http.StatusForbidden, errorKindAuth}, + {http.StatusBadRequest, errorKindInternal}, + {http.StatusNotFound, errorKindInternal}, + {http.StatusInternalServerError, errorKindServer}, + {http.StatusBadGateway, errorKindServer}, + } + for _, tc := range cases { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.status) + })) + api := newTestAPI(srv.URL) + err := api.Upload(context.Background(), "x", []byte("d")) + srv.Close() + + var apiErr *apiError + if !errors.As(err, &apiErr) { + t.Fatalf("status %d: want *APIError, got %T (%v)", tc.status, err, err) + } + if apiErr.kind != tc.kind { + t.Fatalf("status %d: kind = %d, want %d", tc.status, apiErr.kind, tc.kind) + } + if apiErr.status != tc.status { + t.Fatalf("status %d: APIError.Status = %d", tc.status, apiErr.status) + } + } +} + +func TestUploadSurfacesBunnyErrorMessage(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"HttpCode":400,"Message":"checksum mismatch"}`)) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + err := api.Upload(context.Background(), "x", []byte("d")) + + var apiErr *apiError + if !errors.As(err, &apiErr) { + t.Fatalf("want *apiError, got %T (%v)", err, err) + } + if got := apiErr.ErrorVerbose(); !strings.Contains(got, "checksum mismatch") { + t.Fatalf("verbose detail = %q, want it to contain Bunny's Message", got) + } +} + +func TestListParsesFullPaths(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Write([]byte(`[ + {"Path":"/zone/assets/","ObjectName":"app.abc.js","Length":14,"IsDirectory":false,"LastChanged":"2023-06-29T12:00:00.000"}, + {"Path":"/zone/assets/","ObjectName":"sub","Length":0,"IsDirectory":true,"LastChanged":"2023-06-29T12:00:00"} + ]`)) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + objs, err := api.listDir(context.Background(), "assets") + if err != nil { + t.Fatalf("listDir: %v", err) + } + if gotPath != "/zone/assets/" { + t.Fatalf("list path = %q, want trailing slash", gotPath) + } + if len(objs) != 2 { + t.Fatalf("objs = %v", objs) + } + if objs[0].Path != "assets/app.abc.js" || objs[0].Size != 14 || objs[0].IsDirectory { + t.Fatalf("file object = %+v", objs[0]) + } + if objs[1].Path != "assets/sub" || !objs[1].IsDirectory { + t.Fatalf("dir object = %+v", objs[1]) + } +} + +func TestListRootTrailingSlash(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Write([]byte(`[]`)) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + if _, err := api.listDir(context.Background(), ""); err != nil { + t.Fatalf("listDir: %v", err) + } + if gotPath != "/zone/" { + t.Fatalf("root list path = %q, want /zone/", gotPath) + } +} + +func TestWalkBuildsTreeAndDescends(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/zone/": + w.Write([]byte(`[ + {"Path":"/zone/","ObjectName":"index.html","Length":6,"IsDirectory":false}, + {"Path":"/zone/","ObjectName":"assets","IsDirectory":true} + ]`)) + case "/zone/assets/": + w.Write([]byte(`[{"Path":"/zone/assets/","ObjectName":"app.js","Length":3,"IsDirectory":false}]`)) + default: + t.Errorf("unexpected list path %q", r.URL.Path) + } + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + it := api.List(context.Background(), "") + var got []Object + for it.Next() { + got = append(got, it.Object()) + } + if err := it.Err(); err != nil { + t.Fatalf("Walk: %v", err) + } + if len(got) != 3 { + t.Fatalf("got %d entries: %+v", len(got), got) + } + var sawDir, sawNested bool + for _, o := range got { + if o.Path == "assets" && o.IsDirectory { + sawDir = true + } + if o.Path == "assets/app.js" && !o.IsDirectory && o.Size == 3 { + sawNested = true + } + } + if !sawDir || !sawNested { + t.Fatalf("expected descent into assets; got %+v", got) + } +} + +func TestWalk404IsEmpty(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + it := api.List(context.Background(), "") + for it.Next() { + t.Fatalf("unexpected entry %+v", it.Object()) + } + if err := it.Err(); err != nil { + t.Fatalf("Walk: %v", err) + } +} + +func TestWalkFatalOnNon404(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + it := api.List(context.Background(), "") + for it.Next() { + } + if it.Err() == nil { + t.Fatal("expected error") + } +} + +func TestUploadRateLimitedIsTyped(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(429) + })) + defer srv.Close() + api := newTestAPI(srv.URL) + err := api.Upload(context.Background(), "x", []byte("d")) + var apiErr *apiError + if !errors.As(err, &apiErr) || apiErr.kind != errorKindRateLimited { + t.Fatalf("want KindRateLimited APIError, got %v", err) + } +} + +func TestDeleteSuccess(t *testing.T) { + var gotMethod, gotKey, gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + gotKey = r.Header.Get("AccessKey") + gotPath = r.URL.Path + w.WriteHeader(200) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + if err := api.Delete(context.Background(), "a/b.js"); err != nil { + t.Fatalf("Delete: %v", err) + } + if gotMethod != http.MethodDelete { + t.Fatalf("method = %q", gotMethod) + } + if gotKey != "key" || gotPath != "/zone/a/b.js" { + t.Fatalf("key=%q path=%q", gotKey, gotPath) + } +} + +func TestDeleteFailsOn5xx(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + if err := api.Delete(context.Background(), "a/b.js"); err == nil { + t.Fatal("expected error on 5xx") + } +} + +// TestAPIErrorUnwrapsCause ensures a network apiError preserves its cause in the +// error chain, so errors.Is can detect e.g. context.Canceled from an aborted +// in-flight upload. Without this, the worker pool cannot filter cancellation +// collateral out of a fail-fast failure report. +func TestAPIErrorUnwrapsCause(t *testing.T) { + req, err := http.NewRequest(http.MethodPut, "https://example.test/zone/a.js", nil) + if err != nil { + t.Fatal(err) + } + // Simulate a transport error whose cause is context.Canceled. + apiErr := newAPIError(errorKindNetwork, context.Canceled, req, 0) + if !errors.Is(apiErr, context.Canceled) { + t.Fatalf("apiError must unwrap to its cause; got chain that is not context.Canceled") + } +} diff --git a/internal/bunny/api_upload.go b/internal/bunny/api_upload.go new file mode 100644 index 0000000..f2af534 --- /dev/null +++ b/internal/bunny/api_upload.go @@ -0,0 +1,50 @@ +package bunny + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "net/http" + "unsafe" +) + +// Upload performs a raw PUT with AccessKey and Checksum headers, applying the +// retry policy: a 429 with Retry-After is retried up to maxRetries times, +// honoring the header; every other non-2xx (429 without Retry-After, 5xx, +// other 4xx) and any network error fails. The Checksum header is the +// hex-encoded SHA256 of body, computed here so callers need not know Bunny's +// integrity scheme. +func (api *API) Upload(ctx context.Context, path string, body []byte) error { + sum := sha256.Sum256(body) + checksum := upperHexEncode(sum[:]) + + // bytes.NewReader lets net/http set req.GetBody automatically, so do can + // replay the body on each retry attempt. + req, err := http.NewRequestWithContext(ctx, http.MethodPut, api.url(path), bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("build request: %w", err) + } + req.Header.Set("Checksum", checksum) + req.ContentLength = int64(len(body)) + + resp, err := api.do(ctx, req) + if err != nil { + return err + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + return nil +} + +func upperHexEncode(src []byte) string { + const alphabet = "0123456789ABCDEF" + + dst := make([]byte, len(src)*2) + for i, b := range src { + dst[i*2] = alphabet[b>>4] + dst[i*2+1] = alphabet[b&0x0f] + } + return unsafe.String(unsafe.SliceData(dst), len(dst)) +} diff --git a/internal/bunny/error.go b/internal/bunny/error.go new file mode 100644 index 0000000..4c17bda --- /dev/null +++ b/internal/bunny/error.go @@ -0,0 +1,95 @@ +package bunny + +import ( + "fmt" + "net/http" +) + +type errorKind int + +const ( + // errorKindNetwork: the request never got a response (DNS, connection, TLS, + // context cancellation surfaced as a transport error). + errorKindNetwork errorKind = iota + // errorKindAuth: the access key was rejected (401/403). + errorKindAuth + // errorKindRateLimited: Bunny is throttling (429, in any retry-exhausted form). + errorKindRateLimited + // errorKindServer: Bunny-side failure (5xx). + errorKindServer + // errorKindInternal: an unexpected client error (non-auth, non-rate-limit 4xx). + // These almost always mean hopper built a bad request: a likely bug. + errorKindInternal + // errorKindTooLarge: Bunny rejected the asset as too large (413). This is the + // operator's payload, not a hopper bug, so it gets a distinct message. + errorKindTooLarge +) + +func errorKindFromStatus(status int) errorKind { + switch { + case status == http.StatusUnauthorized || status == http.StatusForbidden: + return errorKindAuth + case status == http.StatusRequestEntityTooLarge: + return errorKindTooLarge + case status >= 500: + return errorKindServer + default: + return errorKindInternal + } +} + +// apiError is the single error type returned by all API operations. Error() +// renders a friendly, kind-based message for the user; the technical detail +// (method, path, status, wrapped cause) is exposed separately via +// ErrorVerbose() and surfaced only under --verbose by the presentation layer. +// method and path are derived from the request itself (HTTP verb and +// req.URL.Path). +type apiError struct { + method string // HTTP verb, e.g. "PUT" + path string // req.URL.Path, e.g. "/zone/a/b.js" + status int // HTTP status code; 0 for transport-level (KindNetwork) errors + kind errorKind + cause error // wrapped technical cause, exposed via Unwrap (NOT via Error) +} + +func newAPIError(kind errorKind, err error, req *http.Request, statusCode int) *apiError { + return &apiError{ + method: req.Method, + path: req.URL.Path, + status: statusCode, + kind: kind, + cause: err, + } +} + +func (e *apiError) Unwrap() error { return e.cause } + +// ErrorVerbose returns the technical description (verb, path, status, cause) +// surfaced only under --verbose. It is the "re-run with --verbose for details" +// payload; the user-facing message stays in Error(). Any error type may +// implement this interface to contribute extra detail to failure reports. +func (e *apiError) ErrorVerbose() string { + if e.status != 0 { + return fmt.Sprintf("%s %s -> %d: %v", e.method, e.path, e.status, e.cause) + } + return fmt.Sprintf("%s %s -> %v", e.method, e.path, e.cause) +} + +func (e *apiError) Error() string { + switch e.kind { + case errorKindAuth: + return "The access key was rejected. Check your Bunny storage credentials." + case errorKindRateLimited: + return "Bunny is rate-limiting requests. Please try again shortly." + case errorKindServer: + return "Bunny had an internal server error. Please try again later." + case errorKindNetwork: + return "Couldn't reach Bunny. Check your network connection and endpoint." + case errorKindTooLarge: + return "The asset was rejected by Bunny as too large." + case errorKindInternal: + return "Unexpected response from Bunny. This is likely a bug in hopper. Please report it (re-run with --verbose for details)." + default: + return "An unknown Bunny error occurred (re-run with --verbose for details)." + } +} diff --git a/internal/cli/cmd.go b/internal/cli/cmd.go new file mode 100644 index 0000000..698cf7c --- /dev/null +++ b/internal/cli/cmd.go @@ -0,0 +1,73 @@ +package cli + +import ( + "context" + "errors" + "io" + "os" + + "github.com/spf13/pflag" + + "tsne.dev/hopper/internal/log" +) + +// Run executes the CLI and returns the process exit code. +// +// Note: Help paths will call exit directly. +func Run(ctx context.Context, args []string) int { + main := newMainCommand() + err := runCommand(ctx, main, args) + if err != nil { + var exit exitError + if errors.As(err, &exit) { + return exit.code + } + + var ev errorVerboser + if errors.As(err, &ev) { + log.Error("%s\n %s", err.Error(), ev.ErrorVerbose()) + } else { + log.Error(err.Error()) + } + return 1 + } + return 0 +} + +type command interface { + Usage() string + Run(ctx context.Context, args []string) error +} + +func parseFlags(cmd command, flags *pflag.FlagSet, args []string) { + err := flags.Parse(args) + if err != nil { + if errors.Is(err, pflag.ErrHelp) { + log.Print(cmd.Usage()) + os.Exit(0) + } else { + log.Error("%s\n\n%s", err.Error(), cmd.Usage()) + os.Exit(2) + } + } +} + +func runCommand(ctx context.Context, cmd command, args []string) error { + err := cmd.Run(ctx, args) + if err != nil { + var usageErr usageError + if errors.As(err, &usageErr) { + log.Error("%s\n\n%s", usageErr.msg, cmd.Usage()) + os.Exit(2) + } + return err + } + return nil +} + +func newFlagset(name string) *pflag.FlagSet { + fs := pflag.NewFlagSet(name, pflag.ContinueOnError) + fs.SetOutput(io.Discard) + fs.Usage = func() {} + return fs +} diff --git a/internal/cli/cmd_main.go b/internal/cli/cmd_main.go new file mode 100644 index 0000000..3d4c3db --- /dev/null +++ b/internal/cli/cmd_main.go @@ -0,0 +1,130 @@ +package cli + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/spf13/pflag" + + "tsne.dev/hopper/internal/log" +) + +var _ command = (*mainCommand)(nil) + +type mainCommand struct { + flags *pflag.FlagSet + + zone string + endpoint string + accessKeyFile string + accessKey string + verbose bool + quiet bool +} + +func newMainCommand() *mainCommand { + cmd := &mainCommand{} + cmd.flags = newFlagset("hopper") + cmd.flags.SetInterspersed(false) + cmd.flags.StringVar(&cmd.zone, + "zone", + lookupEnv("BUNNY_ZONE", ""), + "Bunny storage zone name (required) (env: BUNNY_ZONE)", + ) + cmd.flags.StringVar(&cmd.endpoint, + "endpoint", + lookupEnv("BUNNY_ENDPOINT", "storage.bunnycdn.com"), + "Bunny storage endpoint hostname (env: BUNNY_ENDPOINT)", + ) + cmd.flags.StringVar(&cmd.accessKeyFile, + "access-key-file", + lookupEnv("BUNNY_ACCESS_KEY_FILE", ""), + "path to a file containing the storage access key (env: BUNNY_ACCESS_KEY_FILE)", + ) + cmd.flags.BoolVar( + &cmd.verbose, + "verbose", + false, + "print more details", + ) + cmd.flags.BoolVar( + &cmd.quiet, + "quiet", + false, + "print only the final summary and errors", + ) + return cmd +} + +func (cmd *mainCommand) Usage() string { + return "Usage:" + + "\n hopper [global flags] ⟨command⟩ [command flags]" + + "\n" + + "\nCommands:" + + "\n push upload a release to Bunny Storage" + + "\n prune remove orphaned files from Bunny Storage" + + "\n" + + "\nGlobal flags:" + + "\n" + cmd.flags.FlagUsages() +} + +func (cmd *mainCommand) Run(ctx context.Context, args []string) error { + parseFlags(cmd, cmd.flags, args) + + log.SetVerbose(cmd.verbose) + log.SetQuiet(cmd.quiet) + + if cmd.accessKeyFile == "" { + cmd.accessKey = os.Getenv("BUNNY_ACCESS_KEY") + } + + switch { + case cmd.zone == "": + return usageError{"missing zone (provide --zone or BUNNY_ZONE)"} + case cmd.endpoint == "": + // --endpoint has a default, so an empty value here is always deliberate. + return usageError{"the endpoint must not be empty"} + case cmd.accessKey == "" && cmd.accessKeyFile == "": + return usageError{"missing access key (provide --access-key-file, BUNNY_ACCESS_KEY_FILE, or BUNNY_ACCESS_KEY)"} + } + + subargs := cmd.flags.Args() + if len(subargs) == 0 || subargs[0] == "" { + return usageError{"missing subcommand"} + } + + var sub command + switch subargs[0] { + case "push": + sub = newPushCommand(cmd) + case "prune": + sub = newPruneCommand(cmd) + default: + return usageError{fmt.Sprintf("unknown subcommand %q", subargs[0])} + } + + return runCommand(ctx, sub, subargs[1:]) +} + +func (cmd *mainCommand) lookupAccessKey() error { + if cmd.accessKey == "" { + accessKey, err := os.ReadFile(cmd.accessKeyFile) + if err != nil { + return fmt.Errorf("read access key file: %w", err) + } + cmd.accessKey = strings.TrimSpace(string(accessKey)) + } + if cmd.accessKey == "" { + return usageError{"empty access key"} + } + return nil +} + +func lookupEnv(env string, defaultValue string) string { + if val := os.Getenv(env); val != "" { + return val + } + return defaultValue +} diff --git a/internal/cli/cmd_main_test.go b/internal/cli/cmd_main_test.go new file mode 100644 index 0000000..d383970 --- /dev/null +++ b/internal/cli/cmd_main_test.go @@ -0,0 +1,126 @@ +package cli + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// clearGlobalEnv neutralizes the BUNNY_ env fallbacks so a test controls +// resolution entirely through flags. +func clearGlobalEnv(t *testing.T) { + t.Helper() + for _, k := range []string{"BUNNY_ZONE", "BUNNY_ENDPOINT", "BUNNY_ACCESS_KEY_FILE", "BUNNY_ACCESS_KEY"} { + t.Setenv(k, "") + } +} + +func writeKeyFile(t *testing.T) string { + t.Helper() + p := filepath.Join(t.TempDir(), "key") + if err := os.WriteFile(p, []byte("secret\n"), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func runMain(args []string) error { + return newMainCommand().Run(context.Background(), args) +} + +func TestMainMissingSubcommand(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + + var ue usageError + if err := runMain(nil); !errors.As(err, &ue) { + t.Fatalf("expected usageError, got %v", err) + } +} + +func TestMainMissingZone(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + + var ue usageError + err := runMain([]string{"push"}) + if !errors.As(err, &ue) || !strings.Contains(err.Error(), "zone") { + t.Fatalf("expected missing-zone usageError, got %v", err) + } +} + +func TestMainMissingAccessKey(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + + var ue usageError + err := runMain([]string{"--zone", "z", "push"}) + if !errors.As(err, &ue) || !strings.Contains(err.Error(), "access key") { + t.Fatalf("expected missing-access-key usageError, got %v", err) + } +} + +func TestMainUnknownSubcommand(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + kf := writeKeyFile(t) + + var ue usageError + err := runMain([]string{"--zone", "z", "--access-key-file", kf, "frob"}) + if !errors.As(err, &ue) || !strings.Contains(err.Error(), "unknown subcommand") { + t.Fatalf("expected unknown-subcommand usageError, got %v", err) + } +} + +func TestMainDispatchesToPrune(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + kf := writeKeyFile(t) + + // A fully-resolved global set reaches dispatch; the missing archive surfaces a + // plain (non-usage) error, proving parse + validation + dispatch worked. + err := runMain([]string{"--zone", "z", "--access-key-file", kf, "prune", "no-such.tgz"}) + if err == nil || !strings.Contains(err.Error(), "open archive") { + t.Fatalf("expected prune dispatch error, got %v", err) + } +} + +func TestMainAccessKeyFromEnv(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + t.Setenv("BUNNY_ACCESS_KEY", "literalkey") + + // With no --access-key-file, the literal BUNNY_ACCESS_KEY resolves the key + // and resolution reaches prune. + err := runMain([]string{"--zone", "z", "prune", "no-such.tgz"}) + if err == nil || !strings.Contains(err.Error(), "open archive") { + t.Fatalf("expected prune dispatch error (env access key resolved), got %v", err) + } +} + +func TestMainZoneFromEnv(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + t.Setenv("BUNNY_ZONE", "envzone") + kf := writeKeyFile(t) + + // With zone supplied via env, resolution succeeds and reaches prune. + err := runMain([]string{"--access-key-file", kf, "prune", "no-such.tgz"}) + if err == nil || !strings.Contains(err.Error(), "open archive") { + t.Fatalf("expected prune dispatch error (env zone resolved), got %v", err) + } +} + +func TestLookupEnv(t *testing.T) { + t.Setenv("BUNNY_TEST_LOOKUP", "value") + if got := lookupEnv("BUNNY_TEST_LOOKUP", "fallback"); got != "value" { + t.Fatalf("set env: got %q", got) + } + t.Setenv("BUNNY_TEST_LOOKUP", "") + if got := lookupEnv("BUNNY_TEST_LOOKUP", "fallback"); got != "fallback" { + t.Fatalf("empty env should fall back: got %q", got) + } +} diff --git a/internal/cli/cmd_prune.go b/internal/cli/cmd_prune.go new file mode 100644 index 0000000..14c3f0f --- /dev/null +++ b/internal/cli/cmd_prune.go @@ -0,0 +1,186 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "io" + "sync/atomic" + "time" + + "github.com/spf13/pflag" + + "tsne.dev/hopper/internal/log" + "tsne.dev/hopper/internal/source" +) + +var _ command = (*pruneCommand)(nil) + +type pruneCommand struct { + flags *pflag.FlagSet + main *mainCommand + + concurrency uint + prefix string + root string + olderThan duration +} + +func newPruneCommand(main *mainCommand) *pruneCommand { + cmd := &pruneCommand{ + flags: newFlagset("prune"), + main: main, + } + cmd.flags.UintVar(&cmd.concurrency, "concurrency", 0, "number of concurrent transfers (0 = auto-detect based on CPUs)") + registerPathPrefixFlag(cmd.flags, &cmd.prefix, "prefix", "", "path prefix within the zone to target") + registerPathPrefixFlag(cmd.flags, &cmd.root, "root", "", "subtree of the source to consider live") + registerDurationFlag(cmd.flags, &cmd.olderThan, "older-than", "30d", "grace window, only orphans older than this are deleted") + registerDryRunFlag(cmd.flags) + return cmd +} + +func (cmd *pruneCommand) Usage() string { + return "Usage:" + + "\n hopper [global flags] prune [flags] ⟨source⟩" + + "\n" + + "\nFlags:" + + "\n" + cmd.flags.FlagUsages() +} + +func (cmd *pruneCommand) Run(ctx context.Context, args []string) error { + parseFlags(cmd, cmd.flags, args) + + if cmd.olderThan.isZero() { + return usageError{"grace window must not be zero"} + } + + positionals := cmd.flags.Args() + if len(positionals) == 0 || positionals[0] == "" { + return usageError{"missing source"} + } + if len(positionals) > 1 { + return usageError{"only a single source is supported"} + } + + if err := cmd.main.lookupAccessKey(); err != nil { + return err + } + + api := newStorageAPI(cmd.main.endpoint, cmd.main.zone, cmd.main.accessKey) + return cmd.prune(ctx, api, positionals[0], time.Now()) +} + +func (cmd *pruneCommand) prune(ctx context.Context, api storageAPI, sourceArg string, now time.Time) error { + start := time.Now() + + live, err := cmd.liveSet(sourceArg) + if err != nil { + return err + } + if len(live) == 0 { + return errors.New("empty source (refusing to prune). check --root, --prefix, or the archive contents") + } + + entries, err := listRemoteEntries(ctx, api, cmd.prefix) + if err != nil { + return err + } + + cutoff := cmd.olderThan.cutoff(now) + candidates := make([]*remoteEntry, 0, len(entries)/4) + for i := range entries { + e := &entries[i] + if !e.isDir && !live[e.path] { + if e.lastChanged.IsZero() { + log.Warn("could not determine last modified date for %s (skipping)", e.path) + } else if e.lastChanged.Before(cutoff) { + candidates = append(candidates, e) + } + } + } + + // Each delete is independent, so the pool never fails fast. A successful + // delete tombstones its entry. + pool := newWorkerPool(ctx, cmd.concurrency, false) + freed := int64(0) + for _, e := range candidates { + action := fmt.Sprintf("deleting %s (%s)", e.path, humanSize(e.size)) + pool.submit(action, func(ctx context.Context) error { + if err := api.Delete(ctx, e.path); err != nil { + return fileError{path: e.path, err: err} + } + e.deleted = true + atomic.AddInt64(&freed, int64(e.size)) + return nil + }) + } + + deleted, errs := pool.wait() + + var cmdErr error + if reportFailures(ctx, "failed to delete files", errs) { + cmdErr = exitError{1} + } + + cmd.removeEmptyDirs(ctx, api, entries) + + failed := len(errs) + logSummary("deleted: %d"+ + "\nfailed: %d"+ + "\nfreed: %s"+ + "\nelapsed: %s", + deleted, failed, humanSize64(freed), time.Since(start).Round(time.Millisecond), + ) + + return cmdErr +} + +// removeEmptyDirs deletes directories left with no surviving children after the +// file deletes and returns the number successfully removed. It runs as a +// best-effort second pass: failures are reported but never propagated to the +// exit code. +func (cmd *pruneCommand) removeEmptyDirs(ctx context.Context, api storageAPI, entries []remoteEntry) { + dirs := emptyDirs(entries) + if len(dirs) == 0 { + return + } + + pool := newWorkerPool(ctx, cmd.concurrency, false) + for _, dir := range dirs { + action := fmt.Sprintf("removing empty directory %s", dir.path) + pool.submit(action, func(ctx context.Context) error { + err := api.DeleteDir(ctx, dir.path) + if err != nil { + return fileError{path: dir.path, err: err} + } + return nil + }) + } + + _, errs := pool.wait() + if len(errs) > 0 { + _ = reportFailures(ctx, "failed to remove empty directories", errs) + } +} + +// liveSet walks the source once and returns the set of prefixed storage keys in +// the current release. It iterates entries only and never reads content. +func (cmd *pruneCommand) liveSet(sourceArg string) (map[string]bool, error) { + src, err := source.NewTarGzSource(sourceArg, cmd.root) + if err != nil { + return nil, err + } + defer src.Close() + + live := map[string]bool{} + for { + entry, err := src.Next() + if err != nil { + if errors.Is(err, io.EOF) { + return live, nil + } + return nil, err + } + live[prefixedPath(cmd.prefix, entry.Path)] = true + } +} diff --git a/internal/cli/cmd_prune_test.go b/internal/cli/cmd_prune_test.go new file mode 100644 index 0000000..e3b9398 --- /dev/null +++ b/internal/cli/cmd_prune_test.go @@ -0,0 +1,244 @@ +package cli + +import ( + "context" + "errors" + "sort" + "strings" + "testing" + "time" + + "tsne.dev/hopper/internal/bunny" +) + +func runPrune(source string, api storageAPI, older duration, now time.Time) error { + cmd := &pruneCommand{concurrency: 8, olderThan: older} + return cmd.prune(context.Background(), api, source, now) +} + +func days(n int) duration { return duration{days: n} } + +func sortedDeletes(f *fakeAPI) []string { + f.mu.Lock() + defer f.mu.Unlock() + out := append([]string(nil), f.deletes...) + sort.Strings(out) + return out +} + +// TestPruneSelectsCandidates covers the core §10 correctness properties: an +// orphan older than the cutoff is deleted; a file still in the release is kept +// even with an old LastChanged; an orphan newer than the cutoff is kept. +func TestPruneSelectsCandidates(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/live.js", body: "x"}, + }) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + recent := now.AddDate(0, 0, -5) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "index.html", LastChanged: old}, // live, old -> kept + {Path: "assets", IsDirectory: true}, + {Path: "orphan-old.js", Size: 3, LastChanged: old}, // orphan, old -> deleted + {Path: "orphan-new.js", LastChanged: recent}, // orphan, new -> kept + } + fc.lists["assets"] = []bunny.Object{ + {Path: "assets/live.js", LastChanged: old}, // live asset, old -> kept + {Path: "assets/gone.js", Size: 4, LastChanged: old}, // orphan, old -> deleted + } + captureLogs(t) + + if err := runPrune(arc, fc, days(30), now); err != nil { + t.Fatalf("prune: %v", err) + } + + got := sortedDeletes(fc) + want := []string{"assets/gone.js", "orphan-old.js"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("deletes = %v, want %v", got, want) + } +} + +func TestPruneAppliesPrefix(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "app.js", body: "x"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists["foo"] = []bunny.Object{ + {Path: "foo/app.js", LastChanged: old}, // live (prefixed) -> kept + {Path: "foo/stale.js", LastChanged: old}, // orphan -> deleted + } + captureLogs(t) + + cmd := &pruneCommand{concurrency: 8, prefix: "foo", olderThan: days(30)} + if err := cmd.prune(context.Background(), fc, arc, now); err != nil { + t.Fatalf("prune: %v", err) + } + if got := sortedDeletes(fc); len(got) != 1 || got[0] != "foo/stale.js" { + t.Fatalf("deletes = %v, want [foo/stale.js]", got) + } +} + +func TestPruneEmptyLiveSetIsFatal(t *testing.T) { + arc := makeArchive(t, nil) + fc := newFakeAPI() + // A List failure would be reported if reached; the empty-guard must fire + // first, so no Bunny call happens. + fc.listErr = errors.New("should not be called") + captureLogs(t) + + err := runPrune(arc, fc, days(30), time.Now()) + if err == nil || !strings.Contains(err.Error(), "empty source") { + t.Fatalf("expected empty-live-set error, got %v", err) + } + if len(fc.deletes) != 0 { + t.Fatalf("expected no deletes, got %v", fc.deletes) + } +} + +func TestPruneListFailureIsFatal(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "a.js", body: "x"}}) + fc := newFakeAPI() + fc.listErr = errors.New("boom") + captureLogs(t) + + if err := runPrune(arc, fc, days(30), time.Now()); err == nil { + t.Fatal("expected error") + } + if len(fc.deletes) != 0 { + t.Fatalf("expected no deletes, got %v", fc.deletes) + } +} + +func TestPruneDrainAndReport(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "keep.js", body: "x"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "keep.js", LastChanged: old}, + {Path: "a.js", LastChanged: old}, + {Path: "b.js", LastChanged: old}, + {Path: "c.js", LastChanged: old}, + } + fc.failErr = errors.New("boom") + fc.failWhen = func(path string) bool { return path == "b.js" } + _, errb := captureLogs(t) + + err := runPrune(arc, fc, days(30), now) + var exit exitError + if !errors.As(err, &exit) { + t.Fatalf("expected exitError (non-zero exit) on delete failure, got %v", err) + } + // One failed delete must not stop the others: all three orphans are attempted. + if got := sortedDeletes(fc); strings.Join(got, ",") != "a.js,b.js,c.js" { + t.Fatalf("all candidates must be attempted, got %v", got) + } + if !strings.Contains(errb.String(), "b.js") { + t.Fatalf("failure report must name b.js, got %v", errb.String()) + } +} + +// TestPruneRemovesEmptyDirsBottomUp covers §10 step 5: a directory whose only +// file is deleted is removed, and its now-empty parent is removed too. +func TestPruneRemovesEmptyDirsBottomUp(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "index.html", body: "<html>"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "index.html", LastChanged: old}, + {Path: "a", IsDirectory: true}, + } + fc.lists["a"] = []bunny.Object{{Path: "a/b", IsDirectory: true}} + fc.lists["a/b"] = []bunny.Object{{Path: "a/b/gone.js", Size: 3, LastChanged: old}} + captureLogs(t) + + if err := runPrune(arc, fc, days(30), now); err != nil { + t.Fatalf("prune: %v", err) + } + + got := sortedDeletes(fc) + want := []string{"a", "a/b/gone.js"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("deletes = %v, want %v", got, want) + } +} + +// TestPruneKeepsDirWithFailedChild covers §10 step 5: a directory whose file +// delete FAILED must not be removed, because the node survives. +func TestPruneKeepsDirWithFailedChild(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "index.html", body: "<html>"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "index.html", LastChanged: old}, + {Path: "a", IsDirectory: true}, + } + fc.lists["a"] = []bunny.Object{{Path: "a/gone.js", Size: 3, LastChanged: old}} + fc.failWhen = func(path string) bool { return path == "a/gone.js" } + fc.failErr = errors.New("boom") + captureLogs(t) + + if err := runPrune(arc, fc, days(30), now); err == nil { + t.Fatal("expected non-zero exit on file-delete failure") + } + for _, d := range fc.deletes { + if d == "a" { + t.Fatalf("dir with failed child must not be removed, got %v", fc.deletes) + } + } +} + +// TestPruneEmptyDirFailureNonFatal covers §10 step 6: an empty-dir removal +// failure is warned and counted but never flips the exit code. +func TestPruneEmptyDirFailureNonFatal(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "index.html", body: "<html>"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "index.html", LastChanged: old}, + {Path: "a", IsDirectory: true}, + } + fc.lists["a"] = []bunny.Object{{Path: "a/gone.js", Size: 3, LastChanged: old}} + fc.failWhen = func(path string) bool { return path == "a" } + fc.failErr = errors.New("boom") + _, errb := captureLogs(t) + + if err := runPrune(arc, fc, days(30), now); err != nil { + t.Fatalf("empty-dir failure must be non-fatal, got %v", err) + } + if !strings.Contains(errb.String(), "Error:") { + t.Fatalf("expected error on empty-dir failure, got %q", errb.String()) + } +} + +func TestPruneDryRunDeletesNothing(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "keep.js", body: "x"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "keep.js", LastChanged: old}, + {Path: "stale.js", LastChanged: old}, + } + dryRun = true + + if err := runPrune(arc, dryRunStorageAPI{fc}, days(30), now); err != nil { + t.Fatalf("prune: %v", err) + } + if len(fc.deletes) != 0 { + t.Fatalf("dry-run must delete nothing, got %v", fc.deletes) + } +} diff --git a/internal/cli/cmd_push.go b/internal/cli/cmd_push.go new file mode 100644 index 0000000..c58f75c --- /dev/null +++ b/internal/cli/cmd_push.go @@ -0,0 +1,266 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/spf13/pflag" + + "tsne.dev/hopper/internal/bunny" + "tsne.dev/hopper/internal/log" + "tsne.dev/hopper/internal/source" +) + +// pruneHintFloor and pruneHintMultiple gate the post-push prune hint: the zone +// must hold at least floor files and more than multiple times the current +// release before stale assets are worth flagging. multiple covers the current +// release plus five retained rollbacks (six releases' worth of hashed assets). +const ( + pruneHintFloor = 1000 + pruneHintMultiple = 6 +) + +var _ command = (*pushCommand)(nil) + +type pushCommand struct { + flags *pflag.FlagSet + main *mainCommand + + concurrency uint + prefix string + root string +} + +func newPushCommand(main *mainCommand) *pushCommand { + cmd := &pushCommand{ + flags: newFlagset("push"), + main: main, + } + cmd.flags.UintVar(&cmd.concurrency, "concurrency", 0, "number of concurrent transfers (0 = auto-detect based on CPUs)") + registerPathPrefixFlag(cmd.flags, &cmd.prefix, "prefix", "", "path prefix within the zone to target") + registerPathPrefixFlag(cmd.flags, &cmd.root, "root", "", "subtree of the source to upload") + registerDryRunFlag(cmd.flags) + return cmd +} + +func (cmd *pushCommand) Usage() string { + return "Usage:" + + "\n hopper [global flags] push [flags] ⟨source⟩" + + "\n" + + "\nFlags:" + + "\n" + cmd.flags.FlagUsages() +} + +func (cmd *pushCommand) Run(ctx context.Context, args []string) error { + parseFlags(cmd, cmd.flags, args) + + positionals := cmd.flags.Args() + if len(positionals) == 0 || positionals[0] == "" { + return usageError{"missing source"} + } + if len(positionals) > 1 { + return usageError{"only a single source is supported"} + } + source := positionals[0] + + if err := cmd.main.lookupAccessKey(); err != nil { + return err + } + + api := newStorageAPI(cmd.main.endpoint, cmd.main.zone, cmd.main.accessKey) + return cmd.push(ctx, api, source) +} + +func (cmd *pushCommand) push(ctx context.Context, api storageAPI, sourceArg string) error { + src, err := source.NewTarGzSource(sourceArg, cmd.root) + if err != nil { + return err + } + defer src.Close() + + // Drain the complete remote listing before any writes. A List failure is + // fatal here, so push never proceeds on a partial picture. + // Index the remote listing by storage key so each source entry's skip / + // conflict decision is an O(1) lookup rather than a linear scan per file + // (the walk itself is O(remote)). A file and a directory can never share an + // exact key in object storage, so keying by path is unambiguous. + start := time.Now() + + remoteByPath := make(map[string]bunny.Object, 128) + remoteFiles := 0 + it := api.List(ctx, cmd.prefix) + for it.Next() { + o := it.Object() + if !o.IsDirectory { + remoteFiles++ + } + remoteByPath[o.Path] = o + } + if err := it.Err(); err != nil { + return err + } + + skipped := 0 + + // stashedMutables holds mutable entries (e.g. *.html files) read during the + // asset phase but not uploaded until all immutable assets are successfully + // uploaded. The stash is in-memory. A release normally has only a handful of + // mutable files only. + stashedMutables := make([]mutableEntry, 0, 8) + + // Upload every immutable asset via the pool and stash mutables. The single + // reader below buffers each entry's bytes and hands them to a fail-fast worker + // pool. The first asset that fails to upload cancels the pool. + assetsPool := newWorkerPool(ctx, cmd.concurrency, true) + var readErr error + for !assetsPool.cancelled() { + entry, err := src.Next() + if err != nil { + if !errors.Is(err, io.EOF) { + readErr = err + } + break + } + + // Decide against the remote listing: a directory at this exact path is a + // hard conflict; a matching-size file is skipped; any other case (absent, + // size mismatch, 0/missing) falls through to re-upload. + key := prefixedPath(cmd.prefix, entry.Path) + remoteObj, remoteExists := remoteByPath[key] + if remoteExists && remoteObj.IsDirectory { + readErr = fmt.Errorf("remote path %q is a directory, release has a file there", key) + break + } + + if entry.IsMutable { + data, err := entry.ReadContent(ctx) + if err != nil { + readErr = fmt.Errorf("read %q: %w", entry.Path, err) + break + } + stashedMutables = append(stashedMutables, mutableEntry{path: key, data: data}) + continue + } + + if remoteExists && remoteObj.Size == entry.Size { + logAction("skipping %s (exists)", key) + skipped++ + continue + } + + data, err := entry.ReadContent(ctx) + if err != nil { + readErr = fmt.Errorf("read %q: %w", entry.Path, err) + break + } + + action := fmt.Sprintf("uploading %s (%s)", key, humanSize(len(data))) + if !assetsPool.submit(action, uploadTask(api, key, data)) { + break + } + } + + // A reader error is fatal and abandons all in-flight uploads. + if readErr != nil { + assetsPool.cancel(errors.New("upload cancelled")) + } + + assetsUploaded, errs := assetsPool.wait() + reported := reportFailures(ctx, "failed to upload files", errs) + if readErr != nil { + return readErr + } else if reported { + return exitError{1} + } + + // Upload all stashed mutable files. This batch is neither atomic nor ordered. + // A partial mutable-phase failure can leave an entry point live while a mutable + // file it references is not yet uploaded. Mitigation: We report an error and + // a re-run is idempotent. + mutablesPool := newWorkerPool(ctx, cmd.concurrency, false) + for _, m := range stashedMutables { + action := fmt.Sprintf("uploading %s (%s)", m.path, humanSize(len(m.data))) + if !mutablesPool.submit(action, uploadTask(api, m.path, m.data)) { + break // the parent context was cancelled + } + } + + // Commit phase (HTML) is drain-and-report: every stashed mutable is attempted + // and all failures are aggregated. Unlike the pre-commit asset phase, uploads + // here genuinely happened, so we print a summary with counts even on partial + // failure — but the failure list goes out first, keeping the summary last. + mutablesUploaded, mutErrs := mutablesPool.wait() + mutablesFailed := len(mutErrs) + + sourceFiles := assetsUploaded + mutablesUploaded + skipped + mutablesFailed + uploaded := assetsUploaded + mutablesUploaded + + if sourceFiles == 0 { + log.Warn("source contains no files to upload (pushed nothing). check --root, --prefix, or the archive contents") + } + + var cmdErr error + if reportFailures(ctx, "failed to upload files", mutErrs) { + cmdErr = exitError{1} + } + + summary := "uploaded: %d" + + "\nskipped: %d" + + "\nremote: %d" + + "\nfailed: %d" + + "\nelapsed: %s" + if shouldHintPrune(sourceFiles, remoteFiles) { + summary += "\n\nHint: There are many stale files on the zone. Consider running hopper prune." + } + logSummary( + summary, + uploaded, skipped, remoteFiles, mutablesFailed, time.Since(start).Round(time.Millisecond), + ) + + return cmdErr +} + +func shouldHintPrune(sourceFiles, remoteFiles int) bool { + return sourceFiles > 0 && + remoteFiles >= pruneHintFloor && + remoteFiles > pruneHintMultiple*sourceFiles +} + +// uploadTask builds a pool task that PUTs data and, on failure, tags the error +// with the path so the report can name the file. +func uploadTask(api storageAPI, path string, data []byte) func(context.Context) error { + return func(ctx context.Context) error { + if err := api.Upload(ctx, path, data); err != nil { + return fileError{path: path, err: err} + } + return nil + } +} + +// mutableEntry is a mutable file (current rule: *.html) read during the asset +// phase and held in memory until the commit point. data is owned by this +// struct. +type mutableEntry struct { + path string + data []byte +} + +func humanSize(n int) string { + return humanSize64(int64(n)) +} + +func humanSize64(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for x := n / unit; x >= unit; x /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} diff --git a/internal/cli/cmd_push_test.go b/internal/cli/cmd_push_test.go new file mode 100644 index 0000000..9a199b2 --- /dev/null +++ b/internal/cli/cmd_push_test.go @@ -0,0 +1,633 @@ +package cli + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "strings" + "testing" + + "tsne.dev/hopper/internal/bunny" + "tsne.dev/hopper/internal/log" +) + +// runPush drives the push flow against a fake storage API for the given archive, +// using the default concurrency. +func runPush(source string, api storageAPI) error { + return runPushN(source, api, 8) +} + +func runPushN(source string, api storageAPI, concurrency uint) error { + cmd := &pushCommand{concurrency: concurrency} + return cmd.push(context.Background(), api, source) +} + +func TestPushAppliesPrefix(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/app.abc.js", body: "console.log(1)"}, + {name: "assets/skip.js", body: "same"}, + }) + fc := newFakeAPI() + // The remote walk starts under the prefix and returns already-prefixed keys. + fc.lists["foo"] = []bunny.Object{{Path: "foo/assets", IsDirectory: true}} + fc.lists["foo/assets"] = []bunny.Object{ + {Path: "foo/assets/skip.js", Size: int(len("same"))}, + } + out, _ := captureLogs(t) + + cmd := &pushCommand{concurrency: 8, prefix: "foo"} + if err := cmd.push(context.Background(), fc, arc); err != nil { + t.Fatalf("push: %v", err) + } + + got := fc.paths() + want := []string{"foo/assets/app.abc.js", "foo/index.html"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("paths = %v, want %v", got, want) + } + if !strings.Contains(out.String(), "skipping foo/assets/skip.js (exists)") { + t.Fatalf("skip-check must compare prefixed key: %q", out.String()) + } +} + +func TestPushSelectsRootAndStrips(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "README.md", body: "ignored"}, + {name: "app/index.html", body: "<html>"}, + {name: "app/assets/x.js", body: "x"}, + {name: "other/y.js", body: "ignored"}, + }) + fc := newFakeAPI() + captureLogs(t) + + cmd := &pushCommand{concurrency: 8, root: "app"} + if err := cmd.push(context.Background(), fc, arc); err != nil { + t.Fatalf("push: %v", err) + } + + got := fc.paths() + want := []string{"assets/x.js", "index.html"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func TestPushSelectsNestedRoot(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "dist/public/index.html", body: "<html>"}, + {name: "dist/public/assets/x.js", body: "x"}, + {name: "dist/private/secret", body: "ignored"}, + }) + fc := newFakeAPI() + captureLogs(t) + + cmd := &pushCommand{concurrency: 8, root: "dist/public"} + if err := cmd.push(context.Background(), fc, arc); err != nil { + t.Fatalf("push: %v", err) + } + + got := fc.paths() + want := []string{"assets/x.js", "index.html"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func TestPushRootWithPrefix(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "app/assets/x.js", body: "x"}, + }) + fc := newFakeAPI() + captureLogs(t) + + cmd := &pushCommand{concurrency: 8, root: "app", prefix: "foo"} + if err := cmd.push(context.Background(), fc, arc); err != nil { + t.Fatalf("push: %v", err) + } + + got := fc.paths() + want := []string{"foo/assets/x.js"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func TestPushRootMatchesNothing(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "app/index.html", body: "<html>"}, + }) + fc := newFakeAPI() + out, _ := captureLogs(t) + + cmd := &pushCommand{concurrency: 8, root: "nope"} + if err := cmd.push(context.Background(), fc, arc); err != nil { + t.Fatalf("push: %v", err) + } + + if got := fc.paths(); len(got) != 0 { + t.Fatalf("paths = %v, want none", got) + } + if !strings.Contains(out.String(), "uploaded: 0") { + t.Fatalf("summary missing: %q", out.String()) + } +} + +func TestPushUploadsRegularFiles(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/app.abc.js", body: "console.log(1)"}, + {name: "dir/", typeflag: tar.TypeDir}, + }) + fc := newFakeAPI() + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + + got := fc.paths() + want := []string{"assets/app.abc.js", "index.html"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("paths = %v, want %v", got, want) + } + if string(fc.puts["index.html"]) != "<html>" { + t.Fatalf("bad body for index.html: %q", fc.puts["index.html"]) + } + if string(fc.puts["assets/app.abc.js"]) != "console.log(1)" { + t.Fatalf("bad body for app.abc.js: %q", fc.puts["assets/app.abc.js"]) + } + if !strings.Contains(out.String(), "uploaded: 2") { + t.Fatalf("summary missing: %q", out.String()) + } +} + +func TestPushUploadsHTMLAfterAllAssets(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "a.js", body: "x"}, + {name: "nested/page.HTML", body: "<p>"}, // case-insensitive .html + {name: "b.css", body: "y"}, + }) + fc := newFakeAPI() + captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + + // Every HTML Put must happen after the last asset Put. + lastAsset := -1 + firstHTML := len(fc.order) + for i, p := range fc.order { + if strings.HasSuffix(strings.ToLower(p), ".html") { + if i < firstHTML { + firstHTML = i + } + } else if i > lastAsset { + lastAsset = i + } + } + if firstHTML <= lastAsset { + t.Fatalf("HTML uploaded before an asset; order = %v", fc.order) + } +} + +func TestPushDoesNotUploadHTMLWhenAssetFails(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "a.js", body: "x"}, + }) + fc := newFakeAPI() + fc.failOn = "a.js" + captureLogs(t) + + if err := runPush(arc, fc); err == nil { + t.Fatal("expected error") + } + for _, p := range fc.order { + if strings.HasSuffix(strings.ToLower(p), ".html") { + t.Fatalf("HTML must not be uploaded when an asset fails; order = %v", fc.order) + } + } +} + +func TestPushSkipsSymlinkWithWarning(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "a.js", body: "x"}, + {name: "link", typeflag: tar.TypeSymlink, linkname: "a.js"}, + }) + fc := newFakeAPI() + _, errb := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + if got := fc.paths(); len(got) != 1 || got[0] != "a.js" { + t.Fatalf("paths = %v", got) + } + if !strings.Contains(errb.String(), "Warning") { + t.Fatalf("expected warning on stderr, got %q", errb.String()) + } +} + +func TestPushFailsOnUploadError(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "a.js", body: "x"}, + {name: "b.js", body: "y"}, + }) + fc := newFakeAPI() + fc.failOn = "a.js" + captureLogs(t) + + if err := runPush(arc, fc); err == nil { + t.Fatal("expected error") + } +} + +func TestPushSkipsUnchanged(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/app.abc.js", body: "console.log(1)"}, + }) + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "index.html", Size: int(len("<html>"))}, + {Path: "assets", IsDirectory: true}, + } + fc.lists["assets"] = []bunny.Object{ + {Path: "assets/app.abc.js", Size: int(len("console.log(1)"))}, + } + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + // The asset is skipped (matching size); the *.html is the commit point and is + // always uploaded even when an identically-sized copy exists remotely. + if got := fc.paths(); len(got) != 1 || got[0] != "index.html" { + t.Fatalf("expected only index.html uploaded, got %v", got) + } + s := out.String() + if !strings.Contains(s, "skipping assets/app.abc.js (exists)") { + t.Fatalf("missing skip line: %q", s) + } + if strings.Contains(s, "skipping index.html") { + t.Fatalf("index.html must never be skipped: %q", s) + } + if !strings.Contains(s, "uploaded: 1") || !strings.Contains(s, "skipped: 1") { + t.Fatalf("summary missing: %q", s) + } +} + +func TestPushReuploadsOnSizeMismatch(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "a.js", body: "hello"}, // 5 bytes + {name: "b.js", body: "world"}, // present, matching size -> skip + }) + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "a.js", Size: 1}, // size mismatch -> upload + {Path: "b.js", Size: int(len("world"))}, + } + captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + if got := fc.paths(); len(got) != 1 || got[0] != "a.js" { + t.Fatalf("paths = %v, want [a.js]", got) + } +} + +func TestPushErrorsOnDirectoryFileCollision(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "data", body: ""}, // 0-byte asset colliding with a remote dir + }) + fc := newFakeAPI() + // Remote has a directory named "data"; a file at the same path is a hard + // conflict (and a 0-size dir must never silently shadow a 0-byte file). + fc.lists[""] = []bunny.Object{{Path: "data", IsDirectory: true}} + captureLogs(t) + + if err := runPush(arc, fc); err == nil { + t.Fatal("expected error on directory/file collision") + } + if got := fc.paths(); len(got) != 0 { + t.Fatalf("expected no uploads, got %v", got) + } +} + +func TestPushAbortsOnListFailure(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "a.js", body: "x"}}) + fc := newFakeAPI() + fc.listErr = errors.New("boom") + captureLogs(t) + + if err := runPush(arc, fc); err == nil { + t.Fatal("expected error") + } + if got := fc.paths(); len(got) != 0 { + t.Fatalf("expected no uploads before abort, got %v", got) + } +} + +func TestPushRunRejectsBadSourceArgs(t *testing.T) { + captureLogs(t) + cmd := newPushCommand(&mainCommand{}) + + var ue usageError + if err := cmd.Run(context.Background(), nil); !errors.As(err, &ue) { + t.Fatalf("missing source: expected usageError, got %v", err) + } + if err := cmd.Run(context.Background(), []string{"a.tgz", "b.tgz"}); !errors.As(err, &ue) { + t.Fatalf("multiple sources: expected usageError, got %v", err) + } +} + +func TestPushRejectsNegativeConcurrencyAtParse(t *testing.T) { + captureLogs(t) + // A negative value is not a valid uint, so pflag rejects it at parse time. + cmd := newPushCommand(&mainCommand{}) + if err := cmd.flags.Parse([]string{"--concurrency", "-1", "a.tgz"}); err == nil { + t.Fatal("--concurrency -1: expected a parse error, got nil") + } +} + +func TestPushRejectsEscapingPrefix(t *testing.T) { + captureLogs(t) + cmd := newPushCommand(&mainCommand{}) + var ue usageError + if err := cmd.flags.Parse([]string{"--prefix", "../x", "a.tgz"}); !errors.As(err, &ue) { + t.Fatalf("--prefix ../x: expected usageError, got %v", err) + } +} + +func TestPushBoundsInFlightUploads(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "a.js", body: "1"}, + {name: "b.js", body: "2"}, + {name: "c.js", body: "3"}, + {name: "d.js", body: "4"}, + {name: "e.js", body: "5"}, + }) + const limit = 2 + gate := &gatingAPI{fakeAPI: newFakeAPI(), release: make(chan struct{})} + captureLogs(t) + + done := make(chan error, 1) + go func() { done <- runPushN(arc, gate, limit) }() + + // The reader blocks on the semaphore once the pool is saturated, so no more + // uploads can begin until one drains; release them so the run finishes. + for i := 0; i < 5; i++ { + gate.release <- struct{}{} + } + if err := <-done; err != nil { + t.Fatalf("push: %v", err) + } + if peak := gate.peak(); peak > limit { + t.Fatalf("peak in-flight = %d, want <= %d", peak, limit) + } +} + +func TestPushAssetFailureReportsOneFileAndSkipsHTML(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "a.js", body: "1"}, + {name: "b.js", body: "2"}, + }) + fc := newFakeAPI() + // Every asset fails: fail-fast cancels the pool after the first genuine + // failure. All genuine failures that landed before cancellation are + // aggregated (§8.3); cancellation collateral (context.Canceled) is filtered + // out (see TestPushFailFastFiltersCancellationCollateral). HTML is never + // written. + fc.failAll = true + captureLogs(t) + + err := runPushN(arc, fc, 8) + var ee exitError + if !errors.As(err, &ee) { + t.Fatalf("expected exitError, got %T", err) + } + for _, p := range fc.order { + if strings.HasSuffix(strings.ToLower(p), ".html") { + t.Fatalf("HTML must not be written when an asset fails; order = %v", fc.order) + } + } +} + +// TestPushFailFastFiltersCancellationCollateral verifies that the fail-fast +// report names ONLY the genuine upload failure, not the in-flight uploads that +// were abandoned by the cancellation. A context.Canceled is a consequence of the +// stop decision, not an independent file failure, and is swallowed uniformly so +// push (fail-fast) and a user SIGINT behave identically (§8.3). The collateral +// errors WRAP context.Canceled. +func TestPushFailFastFiltersCancellationCollateral(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "a.js", body: "1"}, + {name: "b.js", body: "2"}, + {name: "c.js", body: "3"}, + {name: "index.html", body: "<html>"}, + }) + api := &ctxCancelAPI{ + fakeAPI: newFakeAPI(), + triggerPath: "a.js", + triggerErr: fileError{path: "a.js", err: errors.New("boom")}, + } + _, errb := captureLogs(t) + + // Concurrency covers all three assets so b.js/c.js are genuinely in-flight + // (blocked) when a.js triggers the cancellation. + err := runPushN(arc, api, 3) + var ee exitError + if !errors.As(err, &ee) { + t.Fatalf("expected exitError, got %T (%v)", err, err) + } + report := errb.String() + if !strings.Contains(report, "a.js") || !strings.Contains(report, "boom") { + t.Fatalf("report must name the genuine failure a.js: %q", report) + } + if strings.Contains(report, "b.js") || strings.Contains(report, "c.js") { + t.Fatalf("report must NOT name cancellation collateral b.js/c.js: %q", report) + } + for _, p := range api.order { + if strings.HasSuffix(strings.ToLower(p), ".html") { + t.Fatalf("HTML must not be written when an asset fails; order = %v", api.order) + } + } +} + +func TestPushHTMLFailuresAreAllReported(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "about.html", body: "<html>"}, + {name: "a.js", body: "1"}, + }) + fc := newFakeAPI() + // Assets succeed; both HTML commit uploads fail. The drain phase reports + // every failure under the friendly headline. + fc.failErr = errors.New("boom") + fc.failWhen = func(path string) bool { return strings.HasSuffix(path, ".html") } + out, errb := captureLogs(t) + + err := runPushN(arc, fc, 8) + var exit exitError + if !errors.As(err, &exit) { + t.Fatalf("expected exitError, got %v", err) + } + // The drain report lists every HTML failure on stderr... + if !strings.Contains(errb.String(), "index.html") || !strings.Contains(errb.String(), "about.html") { + t.Fatalf("drain report must list every HTML failure: %q", errb.String()) + } + // ...and the summary is still printed (last) on stdout. + if !strings.Contains(out.String(), "failed:") { + t.Fatalf("summary must be printed after a commit-phase failure: %q", out.String()) + } +} + +func TestShouldHintPrune(t *testing.T) { + cases := []struct { + name string + sourceFiles int + remoteFiles int + want bool + }{ + {"below floor", 1, 999, false}, + {"at floor within ratio", 1000, 6000, false}, + {"at exact multiple", 100, 600, false}, + {"one over multiple but below floor", 10, 61, false}, + {"one over multiple and at floor", 200, 1201, true}, + {"empty source", 0, 100000, false}, + } + for _, tc := range cases { + if got := shouldHintPrune(tc.sourceFiles, tc.remoteFiles); got != tc.want { + t.Errorf("%s: shouldHintPrune(%d, %d) = %v, want %v", + tc.name, tc.sourceFiles, tc.remoteFiles, got, tc.want) + } + } +} + +// stalePushSetup builds a one-file source (source_files == 1) and a fake API +// whose remote listing holds remoteFiles file objects and remoteDirs directory +// objects, so tests can drive the hint decision at chosen boundaries. +func stalePushSetup(t *testing.T, remoteFiles, remoteDirs int) (string, *fakeAPI) { + t.Helper() + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + }) + fc := newFakeAPI() + objs := make([]bunny.Object, 0, remoteFiles+remoteDirs) + for i := 0; i < remoteFiles; i++ { + objs = append(objs, bunny.Object{Path: fmt.Sprintf("stale/%d.js", i), Size: 1}) + } + for i := 0; i < remoteDirs; i++ { + objs = append(objs, bunny.Object{Path: fmt.Sprintf("dir%d", i), IsDirectory: true}) + } + fc.lists[""] = objs + return arc, fc +} + +func TestPushHintFiresWhenStale(t *testing.T) { + arc, fc := stalePushSetup(t, 1001, 0) + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + s := out.String() + if !strings.Contains(s, "remote: 1001") { + t.Fatalf("summary missing remote count: %q", s) + } + if !strings.Contains(s, "\n\nHint:") { + t.Fatalf("expected prune hint: %q", s) + } + // The hint rides on stdout with the summary, not the stderr warning path. + if strings.Contains(s, "\n\nHint:") && strings.Index(s, "remote:") > strings.Index(s, "\n\nHint:") { + t.Fatalf("hint must follow the summary: %q", s) + } +} + +func TestPushNoHintWithinRetentionWindow(t *testing.T) { + arc, fc := stalePushSetup(t, 6, 0) + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + if strings.Contains(out.String(), "hint:") { + t.Fatalf("unexpected hint: %q", out.String()) + } +} + +func TestPushNoHintBelowFloor(t *testing.T) { + arc, fc := stalePushSetup(t, 999, 0) + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + if strings.Contains(out.String(), "hint:") { + t.Fatalf("unexpected hint below floor: %q", out.String()) + } +} + +func TestPushHintIgnoresRemoteDirectories(t *testing.T) { + // Many directory nodes, few files: directories must not inflate remote_files. + arc, fc := stalePushSetup(t, 5, 2000) + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + s := out.String() + if !strings.Contains(s, "remote: 5") { + t.Fatalf("directories must not count: %q", s) + } + if strings.Contains(s, "hint:") { + t.Fatalf("unexpected hint from directory nodes: %q", s) + } +} + +func TestPushHintFiresUnderQuiet(t *testing.T) { + arc, fc := stalePushSetup(t, 1001, 0) + out, _ := captureLogs(t) + log.SetQuiet(true) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + if !strings.Contains(out.String(), "\n\nHint:") { + t.Fatalf("hint must appear under --quiet: %q", out.String()) + } +} + +func TestPushHintFiresUnderDryRun(t *testing.T) { + arc, fc := stalePushSetup(t, 1001, 0) + out, _ := captureLogs(t) + dryRun = true + + if err := runPush(arc, dryRunStorageAPI{fc}); err != nil { + t.Fatalf("push: %v", err) + } + if !strings.Contains(out.String(), "\n\nHint:") { + t.Fatalf("hint must appear under dry-run: %q", out.String()) + } +} + +func TestHumanSize(t *testing.T) { + cases := map[int]string{ + 0: "0 B", + 512: "512 B", + 1024: "1.0 KiB", + 1536: "1.5 KiB", + 1024 * 1024: "1.0 MiB", + } + for in, want := range cases { + if got := humanSize(in); got != want { + t.Errorf("humanSize(%d) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/cli/dryrun.go b/internal/cli/dryrun.go new file mode 100644 index 0000000..ed65f48 --- /dev/null +++ b/internal/cli/dryrun.go @@ -0,0 +1,11 @@ +package cli + +import ( + "github.com/spf13/pflag" +) + +var dryRun = false + +func registerDryRunFlag(flags *pflag.FlagSet) { + flags.BoolVar(&dryRun, "dryrun", false, "run the command without executing any mutating operations") +} diff --git a/internal/cli/dryrun_test.go b/internal/cli/dryrun_test.go new file mode 100644 index 0000000..c2538d6 --- /dev/null +++ b/internal/cli/dryrun_test.go @@ -0,0 +1,78 @@ +package cli + +import ( + "context" + "strings" + "testing" + + "tsne.dev/hopper/internal/bunny" +) + +// TestDryRunPushStubsUploads asserts that under dry-run the push flow performs +// NO uploads against the wrapped API, yet prints the normal per-file action +// line and summary (logging is owned by the command, not the decorator). The +// recording fake is the wrapped (real-stand-in) API, so a zero put-count +// provably shows the write side was never invoked. +func TestDryRunPushStubsUploads(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/app.abc.js", body: "console.log(1)"}, + }) + + spy := newFakeAPI() + out, _ := captureLogs(t) + dryRun = true + + if err := runPush(arc, dryRunStorageAPI{spy}); err != nil { + t.Fatalf("push: %v", err) + } + + if got := spy.paths(); len(got) != 0 { + t.Fatalf("expected no real uploads, got %v", got) + } + + s := out.String() + for _, want := range []string{ + "[dryrun] uploading index.html", + "[dryrun] uploading assets/app.abc.js", + "uploaded: 2", + } { + if !strings.Contains(s, want) { + t.Fatalf("output missing %q; got:\n%s", want, s) + } + } +} + +// TestDryRunDeleteStubbed asserts Delete never reaches the wrapped API +// (reused by prune in issue 008, which will own the delete log line). +func TestDryRunDeleteStubbed(t *testing.T) { + spy := newFakeAPI() + api := dryRunStorageAPI{spy} + + if err := api.Delete(context.Background(), "old/file.js"); err != nil { + t.Fatalf("Delete: %v", err) + } + if len(spy.deletes) != 0 { + t.Fatalf("expected no real deletes, got %v", spy.deletes) + } +} + +// TestDryRunListDelegates asserts the read side still reaches the wrapped API +// (real Bunny in production), unaffected by the write stubbing. +func TestDryRunListDelegates(t *testing.T) { + spy := newFakeAPI() + spy.lists[""] = []bunny.Object{{Path: "a.js", Size: 3}} + api := dryRunStorageAPI{spy} + + it := api.List(context.Background(), "") + var got []bunny.Object + for it.Next() { + got = append(got, it.Object()) + } + if err := it.Err(); err != nil { + t.Fatalf("List: %v", err) + } + if len(got) != 1 || got[0].Path != "a.js" { + t.Fatalf("List delegate = %v", got) + } +} diff --git a/internal/cli/duration.go b/internal/cli/duration.go new file mode 100644 index 0000000..ee40562 --- /dev/null +++ b/internal/cli/duration.go @@ -0,0 +1,136 @@ +package cli + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/spf13/pflag" +) + +// Per-component upper bounds. A cache-safety grace window is measured in days +// to a few years; anything past these is a typo or an attempt to overflow the +// cutoff arithmetic. They are chosen so cutoff = now.AddDate(-y,-m,-d) can never +// overflow the int year addition or the int64 range of time.Time (representable +// to ~year 292 billion), so the destructive cutoff can never silently wrap. +const ( + maxDurationYears = 100000 + maxDurationMonths = 100000 * 12 + maxDurationDays = 100000 * 366 +) + +var _ pflag.Value = (*duration)(nil) + +// duration is a calendar-aware grace window . It holds a tuple +// (years, months, days) rather than a time.Duration. +type duration struct { + years int + months int + days int +} + +func registerDurationFlag(flags *pflag.FlagSet, v *duration, name string, def string, usage string) { + if err := v.Set(def); err != nil { + panic(fmt.Sprintf("invalid default for --%s: %v", name, err)) + } + flags.Var(v, name, usage) +} + +// Type implements the `pflags.Value` interface. +func (d *duration) Type() string { + return "duration" +} + +// String implements the `pflags.Value` interface. +func (d *duration) String() string { + sb := &strings.Builder{} + if d.years != 0 { + fmt.Fprintf(sb, "%dy", d.years) + } + if d.months != 0 { + fmt.Fprintf(sb, "%dm", d.months) + } + if d.days != 0 { + fmt.Fprintf(sb, "%dd", d.days) + } + + if sb.Len() == 0 { + return "0d" + } + return sb.String() +} + +// Set parses a combinable value like "1y2m10d", "7d" or "30d". Each component is +// a non-negative integer followed by one of the units y, m (month), d, and each +// unit may appear at most once. An empty value or a value with no components is +// rejected. +// +// Set implements the `pflags.Value` interface. +func (d *duration) Set(raw string) error { + parsed := duration{} + seen := map[byte]bool{} + components := 0 + + for i := 0; i < len(raw); { + c := raw[i] + if c < '0' || c > '9' { + return usageError{fmt.Sprintf("invalid duration %q", raw)} + } + + n := 0 + for i < len(raw) && raw[i] >= '0' && raw[i] <= '9' { + digit := int(raw[i] - '0') + if n > (math.MaxInt-digit)/10 { + return usageError{fmt.Sprintf("invalid duration %q (out of range)", raw)} + } + n = n*10 + digit + i++ + } + if i >= len(raw) { + return usageError{fmt.Sprintf("invalid duration %q (number %d has no unit)", raw, n)} + } + + unit := raw[i] + if seen[unit] { + return usageError{fmt.Sprintf("invalid duration %q (unit %q repeated)", raw, string(unit))} + } + switch unit { + case 'y': + if n > maxDurationYears { + return usageError{fmt.Sprintf("invalid duration %q (years exceeds %d)", raw, maxDurationYears)} + } + parsed.years = n + case 'm': + if n > maxDurationMonths { + return usageError{fmt.Sprintf("invalid duration %q (months exceeds %d)", raw, maxDurationMonths)} + } + parsed.months = n + case 'd': + if n > maxDurationDays { + return usageError{fmt.Sprintf("invalid duration %q (days exceeds %d)", raw, maxDurationDays)} + } + parsed.days = n + default: + return usageError{fmt.Sprintf("invalid duration %q (unknown unit %q)", raw, string(unit))} + } + seen[unit] = true + components++ + i++ + } + + if components == 0 { + return usageError{fmt.Sprintf("invalid duration %q", raw)} + } + + *d = parsed + return nil +} + +func (d duration) cutoff(tm time.Time) time.Time { + return tm.AddDate(-d.years, -d.months, -d.days) +} + +func (d duration) isZero() bool { + return d.years == 0 && d.months == 0 && d.days == 0 +} diff --git a/internal/cli/duration_test.go b/internal/cli/duration_test.go new file mode 100644 index 0000000..60d446a --- /dev/null +++ b/internal/cli/duration_test.go @@ -0,0 +1,51 @@ +package cli + +import ( + "testing" + "time" +) + +func TestDurationValueParses(t *testing.T) { + cases := map[string]duration{ + "30d": {days: 30}, + "7d": {days: 7}, + "1y2m10d": {years: 1, months: 2, days: 10}, + "2m": {months: 2}, + "1y": {years: 1}, + "0d": {days: 0}, + } + for in, want := range cases { + var v duration + if err := v.Set(in); err != nil { + t.Fatalf("Set(%q): %v", in, err) + } + if v != want { + t.Fatalf("Set(%q) = %+v, want %+v", in, v, want) + } + } +} + +func TestDurationValueRejectsBadInput(t *testing.T) { + for _, in := range []string{"", "d", "5", "5x", "1h", "1m2m", "-3d", "y", "5 d", "99999999999999999999d"} { + var v duration + if err := v.Set(in); err == nil { + t.Fatalf("Set(%q): expected error", in) + } + } +} + +func TestDurationValueMonthIsCalendarMonth(t *testing.T) { + var v duration + if err := v.Set("1m"); err != nil { + t.Fatal(err) + } + now := time.Date(2024, time.March, 31, 0, 0, 0, 0, time.UTC) + // A calendar-aware month back from March 31 lands in February, not "31 days". + got := v.cutoff(now) + if got.Month() != time.February && got.Month() != time.March { + t.Fatalf("cutoff month = %v, want calendar-aware", got.Month()) + } + if got.Equal(now.AddDate(0, 0, -30)) { + t.Fatalf("cutoff must not use a fixed 30-day month") + } +} diff --git a/internal/cli/errors.go b/internal/cli/errors.go new file mode 100644 index 0000000..aeeede1 --- /dev/null +++ b/internal/cli/errors.go @@ -0,0 +1,89 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "strings" + + "tsne.dev/hopper/internal/log" +) + +// errorVerboser is implemented by error types that can contribute extra +// technical detail to failure reports under --verbose (e.g. *bunny.apiError +// exposing verb/path/status/cause). reportFailures prints it indented below the +// friendly message when verbose is enabled. +type errorVerboser interface { + ErrorVerbose() string +} + +// usageError marks a parse or validation failure that should print a short +// usage line and exit 2, distinct from a runtime failure (exit 1). +type usageError struct { + msg string +} + +func (e usageError) Error() string { return e.msg } + +// exitError signals that the command has already emitted all its user-facing +// output (e.g. a failure list followed by the summary) and the process should +// simply exit with the given code, printing nothing further. This keeps the +// summary as the last line. +type exitError struct { + code int +} + +func (e exitError) Error() string { return fmt.Sprintf("exit status %d", e.code) } + +// reportFailures prints a batch of per-file failures to stderr under a headline, +// using the same layout everywhere or the context's cancellation cause. +// It returns true if it reported an error, false otherwise. +func reportFailures(ctx context.Context, msg string, errs []error) bool { + if len(errs) > 0 { + sb := strings.Builder{} + sb.WriteString(msg) + for _, err := range errs { + sb.WriteString("\n ") + sb.WriteString(err.Error()) + if log.Verbose() { + var v errorVerboser + if errors.As(err, &v) { + sb.WriteString("\n ") + sb.WriteString(v.ErrorVerbose()) + } + } + } + log.Error(sb.String()) + return true + } else if ctxErr := context.Cause(ctx); ctxErr != nil { + log.Error(ctxErr.Error()) + return true + } else { + return false + } +} + +// fileError attaches the storage path to an operation failure so the report can +// name the offending file. It preserves the wrapped cause (e.g. the bunny API +// error) in the chain so both friendly rendering (Error) and verbose detail +// (ErrorVerbose) still work via errors.As. +type fileError struct { + path string + err error +} + +func (e fileError) Error() string { return e.path + ": " + e.err.Error() } +func (e fileError) Unwrap() error { return e.err } + +type cancellationError struct { + msg string +} + +// CancellationError returns an error that wraps `context.Canceled` but +// with a custom error message. +func CancellationError(msg string) error { + return cancellationError{msg} +} + +func (e cancellationError) Error() string { return e.msg } +func (e cancellationError) Unwrap() error { return context.Canceled } diff --git a/internal/cli/helpers_test.go b/internal/cli/helpers_test.go new file mode 100644 index 0000000..8aa533f --- /dev/null +++ b/internal/cli/helpers_test.go @@ -0,0 +1,231 @@ +package cli + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "testing" + + "tsne.dev/hopper/internal/bunny" + "tsne.dev/hopper/internal/log" +) + +// captureLogs redirects the global logger to fresh buffers for the duration of +// a test and restores the defaults (and the dryRun global) afterward. Returns +// the stdout and stderr buffers. +func captureLogs(t *testing.T) (*bytes.Buffer, *bytes.Buffer) { + t.Helper() + var out, errb bytes.Buffer + log.Init(&out, &errb) + t.Cleanup(func() { + log.Init(os.Stdout, os.Stderr) + log.SetVerbose(false) + log.SetQuiet(false) + dryRun = false + }) + return &out, &errb +} + +// fakeAPI is an in-memory recording storageAPI: the reusable test seam for all +// command tests. It records every attempted Upload/Delete and serves scripted +// List results. It satisfies storageAPI, so it can be substituted for the real +// *bunny.API anywhere (including wrapped by dryRunStorageAPI). +type fakeAPI struct { + mu sync.Mutex + puts map[string][]byte + order []string + deletes []string + lists map[string][]bunny.Object + listErr error + failOn string + failAll bool + failWhen func(path string) bool + failErr error +} + +func newFakeAPI() *fakeAPI { + return &fakeAPI{puts: map[string][]byte{}, lists: map[string][]bunny.Object{}} +} + +func (f *fakeAPI) Upload(ctx context.Context, path string, body []byte) error { + f.mu.Lock() + defer f.mu.Unlock() + f.order = append(f.order, path) + if f.failAll || path == f.failOn || (f.failWhen != nil && f.failWhen(path)) { + if f.failErr != nil { + return f.failErr + } + return errors.New("forced failure") + } + b := make([]byte, len(body)) + copy(b, body) + f.puts[path] = b + return nil +} + +func (f *fakeAPI) Delete(ctx context.Context, path string) error { + return f.recordDelete(path) +} + +// DeleteDir records the logical directory path (no trailing slash) so tests can +// assert on it uniformly with file deletes; the real trailing-slash quirk lives +// in the bunny package and is exercised by bunny's own tests. +func (f *fakeAPI) DeleteDir(ctx context.Context, path string) error { + return f.recordDelete(path) +} + +func (f *fakeAPI) recordDelete(path string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.deletes = append(f.deletes, path) + if f.failAll || path == f.failOn || (f.failWhen != nil && f.failWhen(path)) { + if f.failErr != nil { + return f.failErr + } + return errors.New("forced failure") + } + return nil +} + +func (f *fakeAPI) List(ctx context.Context, dir string) *bunny.Iterator { + return bunny.NewIterator(ctx, dir, f.listDir) +} + +func (f *fakeAPI) listDir(ctx context.Context, dir string) ([]bunny.Object, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.listErr != nil { + return nil, f.listErr + } + return f.lists[dir], nil +} + +func (f *fakeAPI) paths() []string { + f.mu.Lock() + defer f.mu.Unlock() + var ps []string + for p := range f.puts { + ps = append(ps, p) + } + sort.Strings(ps) + return ps +} + +// ctxCancelAPI simulates the real fail-fast scenario: the upload of triggerPath +// fails immediately with a genuine (non-context) error, while every other +// upload blocks until the pool cancels its context and then returns an error +// that WRAPS context.Canceled (mirroring a real *bunny.apiError built from an +// aborted in-flight request). It lets a test assert that cancellation collateral +// is filtered out of the failure report while the genuine trigger survives. +type ctxCancelAPI struct { + *fakeAPI + triggerPath string + triggerErr error +} + +func (c *ctxCancelAPI) Upload(ctx context.Context, path string, body []byte) error { + c.fakeAPI.mu.Lock() + c.fakeAPI.order = append(c.fakeAPI.order, path) + c.fakeAPI.mu.Unlock() + + if path == c.triggerPath { + return c.triggerErr + } + // Collateral: wait for the fail-fast cancellation, then report it as a + // context-wrapped error, exactly like a real aborted upload. + <-ctx.Done() + return fmt.Errorf("upload %s aborted: %w", path, ctx.Err()) +} + +// gatingAPI wraps fakeAPI to make each Upload block on a release channel while +// recording the peak number of concurrent in-flight uploads, so tests can +// assert the pool never exceeds --concurrency. +type gatingAPI struct { + *fakeAPI + release chan struct{} + + gmu sync.Mutex + cur int + maxSeen int +} + +func (g *gatingAPI) Upload(ctx context.Context, path string, body []byte) error { + g.gmu.Lock() + g.cur++ + if g.cur > g.maxSeen { + g.maxSeen = g.cur + } + g.gmu.Unlock() + + <-g.release + + g.gmu.Lock() + g.cur-- + g.gmu.Unlock() + return g.fakeAPI.Upload(ctx, path, body) +} + +func (g *gatingAPI) peak() int { + g.gmu.Lock() + defer g.gmu.Unlock() + return g.maxSeen +} + +// tarEntry describes a single file/header to write into a test archive. +type tarEntry struct { + name string + body string + typeflag byte + linkname string +} + +func makeArchive(t *testing.T, entries []tarEntry) string { + t.Helper() + dir := t.TempDir() + p := filepath.Join(dir, "a.tar.gz") + f, err := os.Create(p) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz := gzip.NewWriter(f) + tw := tar.NewWriter(gz) + for _, e := range entries { + tf := e.typeflag + if tf == 0 { + tf = tar.TypeReg + } + hdr := &tar.Header{ + Name: e.name, + Mode: 0o644, + Size: int64(len(e.body)), + Typeflag: tf, + Linkname: e.linkname, + } + if tf != tar.TypeReg { + hdr.Size = 0 + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if tf == tar.TypeReg { + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return p +} diff --git a/internal/cli/log.go b/internal/cli/log.go new file mode 100644 index 0000000..c837649 --- /dev/null +++ b/internal/cli/log.go @@ -0,0 +1,22 @@ +package cli + +import ( + "tsne.dev/hopper/internal/log" +) + +// logAction prints a per-action line. +// In dry-run mode the line is prefixed with "[dryrun] ". +func logAction(format string, args ...any) { + if dryRun { + format = "[dryrun] " + format + } + log.Print(format, args...) +} + +// logSummary prints the final summary line. +func logSummary(format string, args ...any) { + if dryRun { + format += "\n\nNote: This was a dry run. Nothing on the zone was modified." + } + log.PrintImportant(format, args...) +} diff --git a/internal/cli/path_prefix.go b/internal/cli/path_prefix.go new file mode 100644 index 0000000..355349e --- /dev/null +++ b/internal/cli/path_prefix.go @@ -0,0 +1,51 @@ +package cli + +import ( + "fmt" + "path" + "strings" + + "github.com/spf13/pflag" +) + +var _ pflag.Value = (*pathPrefixValue)(nil) + +type pathPrefixValue struct { + p *string + name string +} + +func registerPathPrefixFlag(flags *pflag.FlagSet, p *string, name string, val string, usage string) { + *p = val + flags.Var(&pathPrefixValue{p, name}, name, usage) +} + +func (v *pathPrefixValue) Type() string { return "string" } +func (v *pathPrefixValue) String() string { return *v.p } + +func (v *pathPrefixValue) Set(raw string) error { + trimmed := strings.Trim(raw, "/") + if trimmed == "" { + *v.p = "" + return nil + } + + cleaned := path.Clean(trimmed) + if cleaned == "." { + *v.p = "" + return nil + } + if cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return usageError{fmt.Sprintf("%s %q escapes", v.name, raw)} + } + + *v.p = cleaned + return nil +} + +func prefixedPath(prefix string, filePath string) string { + if prefix == "" { + return filePath + } + return path.Join(prefix, filePath) +} diff --git a/internal/cli/path_prefix_test.go b/internal/cli/path_prefix_test.go new file mode 100644 index 0000000..bdb4e42 --- /dev/null +++ b/internal/cli/path_prefix_test.go @@ -0,0 +1,52 @@ +package cli + +import ( + "errors" + "testing" +) + +func TestPathPrefixValueSet(t *testing.T) { + cases := map[string]string{ + "": "", + "/": "", + "foo": "foo", + "/foo": "foo", + "foo/": "foo", + "/foo/": "foo", + "foo//bar": "foo/bar", + "foo/./bar": "foo/bar", + "foo/bar/..": "foo", + "a/b/../c": "a/c", + } + for in, want := range cases { + var p string + v := &pathPrefixValue{p: &p} + if err := v.Set(in); err != nil { + t.Errorf("Set(%q): unexpected error %v", in, err) + continue + } + if p != want { + t.Errorf("Set(%q) = %q, want %q", in, p, want) + } + } +} + +func TestPathPrefixValueSetEscapes(t *testing.T) { + for _, in := range []string{"..", "../x", "/../x", "foo/../../x"} { + var p string + v := &pathPrefixValue{p: &p} + var ue usageError + if err := v.Set(in); !errors.As(err, &ue) { + t.Errorf("Set(%q): expected usageError, got %v", in, err) + } + } +} + +func TestPrefixedPath(t *testing.T) { + if got := prefixedPath("", "assets/x.js"); got != "assets/x.js" { + t.Errorf("empty prefix: got %q", got) + } + if got := prefixedPath("foo", "assets/x.js"); got != "foo/assets/x.js" { + t.Errorf("prefixed: got %q", got) + } +} diff --git a/internal/cli/remote.go b/internal/cli/remote.go new file mode 100644 index 0000000..43fe93e --- /dev/null +++ b/internal/cli/remote.go @@ -0,0 +1,94 @@ +package cli + +import ( + "context" + "sort" + "strings" + "time" +) + +type remoteEntry struct { + path string + size int + lastChanged time.Time + isDir bool + deleted bool + + // orderKey is path with '/' replaced by 0x00, precomputed so the sort + // compares native strings. '/' and 0x00 are the only two bytes that can + // never appear in a path, and 0x00 sorts below every other byte, so this + // orders a directory's descendants ("a/...") as a contiguous run immediately + // after the directory ("a") and before any sibling that shares its base name + // ("a.js", "ab"). emptyDirs relies on that contiguity. + orderKey string +} + +// listRemoteEntries fetches all remote entries from the api. The returned list +// is sorted by orderKey (see remoteEntry.orderKey), which is the ordering +// emptyDirs requires. Callers must not re-sort it by path. +func listRemoteEntries(ctx context.Context, api storageAPI, prefix string) ([]remoteEntry, error) { + entries := make([]remoteEntry, 0, 64) + it := api.List(ctx, prefix) + for it.Next() { + o := it.Object() + entries = append(entries, remoteEntry{ + path: o.Path, + size: o.Size, + isDir: o.IsDirectory, + lastChanged: o.LastChanged, + orderKey: pathOrderKey(o.Path), + }) + } + if err := it.Err(); err != nil { + return nil, err + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].orderKey < entries[j].orderKey + }) + return entries, nil +} + +// emptyDirs returns the directory entries whose subtree holds no surviving +// file. entries must be sorted by orderKey (see remoteEntry.orderKey), so a +// directory's subtree is the contiguous run of following entries sharing its +// path prefix. A directory exists only to hold files, so it survives iff some +// file beneath it survives. This makes the bottom-up cascade (a parent emptied +// by removing its last child directory) fall out without extra bookkeeping. +// A file survives unless it was deleted. +func emptyDirs(entries []remoteEntry) []*remoteEntry { + n := len(entries) + empty := make([]*remoteEntry, 0, n/4) + for i := 0; i < n; { + dir := &entries[i] + i++ + if !dir.isDir { + continue + } + + existingFiles := 0 + c := i + for prefix := dir.path + "/"; c < n; c++ { + child := &entries[c] + if !strings.HasPrefix(child.path, prefix) { + break + } + if !child.isDir && !child.deleted { + existingFiles++ + } + } + + if existingFiles == 0 { + // Emit only the top of the empty subtree and skip its descendants. + // Deleting a Bunny directory removes the subtree recursively, so + // deleting `dir` also removes any empty sub-dirs beneath it. + empty = append(empty, dir) + i = c + } + } + + return empty +} + +func pathOrderKey(path string) string { + return strings.ReplaceAll(path, "/", "\x00") +} diff --git a/internal/cli/remote_test.go b/internal/cli/remote_test.go new file mode 100644 index 0000000..b4e20d9 --- /dev/null +++ b/internal/cli/remote_test.go @@ -0,0 +1,141 @@ +package cli + +import ( + "sort" + "strings" + "testing" +) + +func dir(path string) remoteEntry { + return remoteEntry{path: path, isDir: true, orderKey: pathOrderKey(path)} +} +func file(path string) remoteEntry { + return remoteEntry{path: path, orderKey: pathOrderKey(path)} +} + +func deleted(e remoteEntry) remoteEntry { + e.deleted = true + return e +} + +func emptyDirPaths(entries []remoteEntry) []string { + // Reproduce the production ordering (see listRemoteEntries) so the sort rule + // and the emptyDirs scan rule are always exercised together, rather than + // trusting hand-ordered literals. + sort.Slice(entries, func(i, j int) bool { + return entries[i].orderKey < entries[j].orderKey + }) + got := emptyDirs(entries) + paths := make([]string, len(got)) + for i, e := range got { + paths[i] = e.path + } + sort.Strings(paths) + return paths +} + +func TestEmptyDirs(t *testing.T) { + tests := []struct { + name string + entries []remoteEntry + want []string + }{ + { + name: "dir with only a deleted file is empty", + entries: []remoteEntry{ + dir("a"), + deleted(file("a/gone.js")), + }, + want: []string{"a"}, + }, + { + name: "dir with a surviving file is kept", + entries: []remoteEntry{ + dir("a"), + file("a/live.js"), + }, + want: []string{}, + }, + { + name: "bottom-up cascade removes parent and child", + entries: []remoteEntry{ + dir("a"), + dir("a/b"), + deleted(file("a/b/gone.js")), + }, + want: []string{"a"}, + }, + { + name: "deep file deleted but sibling file alive keeps parent", + entries: []remoteEntry{ + dir("a"), + dir("a/b"), + deleted(file("a/b/gone.js")), + file("a/c.js"), + }, + want: []string{"a/b"}, + }, + { + name: "sibling with shared name prefix is not swallowed", + entries: []remoteEntry{ + dir("a"), + deleted(file("a/gone.js")), + file("ab/live.js"), + }, + want: []string{"a"}, + }, + { + name: "sibling colliding on base name does not hide live children", + entries: []remoteEntry{ + dir("a"), + file("a.js"), + file("a/live.js"), + }, + want: []string{}, + }, + { + name: "empty dir with no children is removed", + entries: []remoteEntry{ + dir("a"), + }, + want: []string{"a"}, + }, + { + name: "nested dirs with no files at all are all removed", + entries: []remoteEntry{ + dir("a"), + dir("a/b"), + dir("a/c"), + }, + want: []string{"a"}, + }, + { + name: "empty dir nested under a dir holding a live file keeps outer, removes inner", + entries: []remoteEntry{ + dir("a"), + dir("a/b"), + dir("a/b/c"), + file("a/live.js"), + }, + want: []string{"a/b"}, + }, + { + name: "live file under a nested dir keeps the whole chain", + entries: []remoteEntry{ + dir("a"), + dir("a/b"), + file("a/b/live.js"), + }, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := emptyDirPaths(tt.entries) + if strings.Join(got, ",") != strings.Join(tt.want, ",") { + t.Fatalf("emptyDirs = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/cli/storage.go b/internal/cli/storage.go new file mode 100644 index 0000000..654ed85 --- /dev/null +++ b/internal/cli/storage.go @@ -0,0 +1,46 @@ +package cli + +import ( + "context" + + "tsne.dev/hopper/internal/bunny" +) + +// storageAPI is the full storage surface the dry-run decorator delegates to. +// Its read side (List) must stay real even under --dry-run; its +// write side (Upload/Delete) is what the decorator stubs. The real +// *bunny.Client satisfies this, as does the recording fake used in tests. +type storageAPI interface { + Upload(ctx context.Context, path string, body []byte) error + List(ctx context.Context, dir string) *bunny.Iterator + Delete(ctx context.Context, path string) error + DeleteDir(ctx context.Context, path string) error +} + +func newStorageAPI(endpoint, zone, accessKey string) storageAPI { + var api storageAPI = bunny.NewAPI(endpoint, zone, accessKey) + if dryRun { + api = dryRunStorageAPI{api} + } + return api +} + +type dryRunStorageAPI struct { + upstream storageAPI +} + +func (api dryRunStorageAPI) List(ctx context.Context, dir string) *bunny.Iterator { + return api.upstream.List(ctx, dir) +} + +func (api dryRunStorageAPI) Upload(ctx context.Context, path string, body []byte) error { + return nil +} + +func (api dryRunStorageAPI) Delete(ctx context.Context, path string) error { + return nil +} + +func (api dryRunStorageAPI) DeleteDir(ctx context.Context, path string) error { + return nil +} diff --git a/internal/cli/verbosity_test.go b/internal/cli/verbosity_test.go new file mode 100644 index 0000000..68bda1d --- /dev/null +++ b/internal/cli/verbosity_test.go @@ -0,0 +1,107 @@ +package cli + +import ( + "archive/tar" + "strings" + "testing" + + "tsne.dev/hopper/internal/log" +) + +// verbosityArchive is a small release with one immutable asset and one mutable +// entry point, enough to exercise action lines and the summary. +func verbosityArchive(t *testing.T) string { + return makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/app.abc.js", body: "console.log(1)"}, + // A symlink triggers an always-on warning regardless of mode. + {name: "assets/link.js", typeflag: tar.TypeSymlink, linkname: "app.abc.js"}, + }) +} + +func TestPushDefaultOutput(t *testing.T) { + arc := verbosityArchive(t) + fc := newFakeAPI() + out, errb := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + + stdout := out.String() + if !strings.Contains(stdout, "uploading assets/app.abc.js") { + t.Fatalf("default output missing action line: %q", stdout) + } + if !strings.Contains(stdout, "uploaded: 2") { + t.Fatalf("default output missing summary: %q", stdout) + } + if !strings.Contains(errb.String(), "Warning:") { + t.Fatalf("warning missing from stderr: %q", errb.String()) + } +} + +func TestPushQuietOnlySummaryAndWarnings(t *testing.T) { + arc := verbosityArchive(t) + fc := newFakeAPI() + out, errb := captureLogs(t) + log.SetQuiet(true) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + + stdout := out.String() + if strings.Contains(stdout, "uploading") { + t.Fatalf("quiet must suppress action lines: %q", stdout) + } + if !strings.Contains(stdout, "uploaded: 2") { + t.Fatalf("quiet must keep summary: %q", stdout) + } + if !strings.Contains(errb.String(), "Warning:") { + t.Fatalf("quiet must keep warnings on stderr: %q", errb.String()) + } +} + +func TestPushQuietWinsOverVerbose(t *testing.T) { + arc := verbosityArchive(t) + fc := newFakeAPI() + out, errb := captureLogs(t) + log.SetVerbose(true) + log.SetQuiet(true) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + + stdout := out.String() + if strings.Contains(stdout, "uploading") { + t.Fatalf("quiet+verbose must suppress action lines: %q", stdout) + } + if !strings.Contains(stdout, "uploaded: 2") { + t.Fatalf("quiet+verbose must keep summary: %q", stdout) + } + if !strings.Contains(errb.String(), "Warning:") { + t.Fatalf("quiet+verbose must keep warnings: %q", errb.String()) + } +} + +func TestLogStreamRouting(t *testing.T) { + out, errb := captureLogs(t) + + log.Print("normal") + log.PrintImportant("summary") + log.Warn("careful") + log.Error("boom") + + stdout := out.String() + stderr := errb.String() + if !strings.Contains(stdout, "normal") || !strings.Contains(stdout, "summary") { + t.Fatalf("stdout missing normal output: %q", stdout) + } + if strings.Contains(stdout, "careful") || strings.Contains(stdout, "boom") { + t.Fatalf("warnings/errors leaked to stdout: %q", stdout) + } + if !strings.Contains(stderr, "Warning: careful") || !strings.Contains(stderr, "Error: boom") { + t.Fatalf("stderr missing warning/error: %q", stderr) + } +} diff --git a/internal/cli/worker_pool.go b/internal/cli/worker_pool.go new file mode 100644 index 0000000..fe44644 --- /dev/null +++ b/internal/cli/worker_pool.go @@ -0,0 +1,110 @@ +package cli + +import ( + "context" + "errors" + "runtime" + "sync" + "sync/atomic" +) + +// pool is a bounded worker pool for concurrent, independent tasks (§14). A +// single caller feeds work via submit; up to concurrency tasks run in flight at +// once, bounded by the sem token held from submit until the task returns. +// +// The pool is deliberately upload-agnostic: a task is any func(ctx) error, so +// the same machinery serves push uploads today and prune deletes later. Task +// wording, error identity, and any success side-effect live in the closure. +// +// submit is the only place that logs a per-action line, and it is only ever +// called from one goroutine (the caller feeding work), so output needs no +// locking; only the shared counters are guarded. Tasks run concurrently, so any +// state a task closure touches beyond its own arguments is the caller's +// responsibility to synchronize. +type workerPool struct { + ctx context.Context + cancel context.CancelCauseFunc + sem chan struct{} + wg sync.WaitGroup + + // failFast cancels the pool on the first failure so in-flight tasks are + // abandoned and queued ones never start (the push-asset policy, §8.3). When + // false, every submitted task runs and all failures are collected (the + // commit/prune-delete policy). + failFast bool + + mtx sync.Mutex + errs []error + succeeded int64 +} + +// newWorkerPool creates a new worker pool to process tasks concurrently. If `concurrency` +// is zero, the number of parallel jobs is aut-detected based on the number of CPUs. +func newWorkerPool(ctx context.Context, concurrency uint, failFast bool) *workerPool { + if concurrency == 0 { + concurrency = 2 * uint(runtime.NumCPU()) + } + ctx, cancel := context.WithCancelCause(ctx) + return &workerPool{ + ctx: ctx, + cancel: cancel, + sem: make(chan struct{}, concurrency), + failFast: failFast, + } +} + +// submit blocks until a worker slot is free, then runs task in the background. +// It reports false when the pool has been cancelled (fail-fast or the parent +// context), signalling the caller to stop feeding work. action is the text that +// will be logged before the task is executed. +func (p *workerPool) submit(action string, task func(ctx context.Context) error) bool { + select { + case p.sem <- struct{}{}: + case <-p.ctx.Done(): + return false + } + + logAction(action) + + p.wg.Add(1) + go func() { + defer p.wg.Done() + defer func() { <-p.sem }() + + if err := task(p.ctx); err != nil { + if !errors.Is(err, context.Canceled) { + p.mtx.Lock() + p.errs = append(p.errs, err) + p.mtx.Unlock() + } + if p.failFast { + p.cancel(CancellationError("operation aborted")) + } + return + } + atomic.AddInt64(&p.succeeded, 1) + }() + return true +} + +// cancelled reports whether the pool's context is already done (fail-fast +// triggered or the parent context cancelled). A feeder loop can poll this to +// stop producing/buffering work promptly, instead of only discovering the +// cancellation on its next submit. +func (p *workerPool) cancelled() bool { + select { + case <-p.ctx.Done(): + return true + default: + return false + } +} + +// wait blocks until all in-flight tasks finish, then releases the pool's +// context. It returns the number of succeeded tasks and the errors the +// workers collected. +func (p *workerPool) wait() (int, []error) { + p.wg.Wait() + p.cancel(nil) + return int(p.succeeded), p.errs +} diff --git a/internal/cli/worker_pool_test.go b/internal/cli/worker_pool_test.go new file mode 100644 index 0000000..db3edf0 --- /dev/null +++ b/internal/cli/worker_pool_test.go @@ -0,0 +1,70 @@ +package cli + +import ( + "context" + "errors" + "testing" +) + +// TestWorkerPoolFailFastSwallowsCancellation locks in the §8.3 invariant that a +// fail-fast pool reports only GENUINE failures: the first real error cancels the +// pool, and the in-flight siblings that consequently return context.Canceled are +// swallowed (cancellation collateral is never an independent per-file failure). +func TestWorkerPoolFailFastSwallowsCancellation(t *testing.T) { + pool := newWorkerPool(context.Background(), 8, true) + + // Occupy several slots with tasks that block until the pool is cancelled and + // then return context.Canceled (the collateral that must be swallowed). + const blockers = 5 + for i := 0; i < blockers; i++ { + pool.submit("block", func(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() // context.Canceled + }) + } + + // One genuine failure triggers fail-fast cancellation of the blockers above. + pool.submit("fail", func(ctx context.Context) error { + return fileError{path: "a.js", err: errors.New("boom")} + }) + + succeeded, errs := pool.wait() + if succeeded != 0 { + t.Fatalf("succeeded = %d, want 0", succeeded) + } + if len(errs) != 1 { + t.Fatalf("errs = %d (%v), want exactly 1 genuine failure", len(errs), errs) + } + if errors.Is(errs[0], context.Canceled) { + t.Fatalf("reported error is cancellation collateral, want the genuine failure: %v", errs[0]) + } + var fe fileError + if !errors.As(errs[0], &fe) || fe.path != "a.js" { + t.Fatalf("reported error = %v, want fileError for a.js", errs[0]) + } +} + +// TestWorkerPoolDrainReportsAll confirms the non-fail-fast (drain-and-report) +// mode collects every failure and still runs all tasks. +func TestWorkerPoolDrainReportsAll(t *testing.T) { + pool := newWorkerPool(context.Background(), 4, false) + + const tasks = 6 + for i := 0; i < tasks; i++ { + odd := i%2 == 1 + pool.submit("t", func(ctx context.Context) error { + if odd { + return errors.New("fail") + } + return nil + }) + } + + succeeded, errs := pool.wait() + if succeeded != 3 { + t.Fatalf("succeeded = %d, want 3", succeeded) + } + if len(errs) != 3 { + t.Fatalf("errs = %d, want 3", len(errs)) + } +} diff --git a/internal/log/log.go b/internal/log/log.go new file mode 100644 index 0000000..68635c3 --- /dev/null +++ b/internal/log/log.go @@ -0,0 +1,70 @@ +// Package log centralizes all user-facing output behind package-level +// functions backed by a private global logger. Normal output (actions, summary) +// goes to stdout; warnings and errors go to stderr. For a single-process CLI a +// global sink avoids threading a logger through every call site. +package log + +import ( + "fmt" + "io" + "os" +) + +// global is the process-wide logger. It defaults to the real stdout/stderr so the +// package is usable without explicit setup (e.g. in tests). +var global = &logger{stdout: os.Stdout, stderr: os.Stderr} + +type logger struct { + stdout io.Writer + stderr io.Writer + verbose bool + quiet bool +} + +// Init configures the global logger's destinations. Call once at startup. +func Init(out, err io.Writer) { + global.stdout = out + global.stderr = err +} + +// SetVerbose toggles whether commands include extra detail in their output. +func SetVerbose(v bool) { global.verbose = v } + +// SetQuiet toggles whether normal output is suppressed. Quiet wins over verbose +// (see DESIGN §13): the two are points on one scale, quiet > verbose > default. +func SetQuiet(q bool) { global.quiet = q } + +// Verbose reports whether verbose output is enabled. It folds the quiet override +// in, so callers never re-check quiet: quiet suppresses verbose detail. +func Verbose() bool { return global.verbose && !global.quiet } + +// Print writes a normal output line to stdout. It is suppressed under quiet. +func Print(format string, args ...any) { + if !global.quiet { + fmt.Fprintf(global.stdout, format+"\n", args...) + } +} + +// PrintVerbose writes an extra-detail line to stdout, emitted only when verbose +// output is enabled (and not suppressed by quiet, per Verbose()). +func PrintVerbose(format string, args ...any) { + if Verbose() { + fmt.Fprintf(global.stdout, format+"\n", args...) + } +} + +// PrintImportant writes a line to stdout that survives quiet (e.g. the final +// summary). +func PrintImportant(format string, args ...any) { + fmt.Fprintf(global.stdout, format+"\n", args...) +} + +// Warn prints a warning to stderr. Warnings are never suppressed. +func Warn(format string, args ...any) { + fmt.Fprintf(global.stderr, "Warning: "+format+"\n", args...) +} + +// Error prints an error to stderr. Errors are never suppressed. +func Error(format string, args ...any) { + fmt.Fprintf(global.stderr, "Error: "+format+"\n", args...) +} diff --git a/internal/source/source.go b/internal/source/source.go new file mode 100644 index 0000000..f2f99b4 --- /dev/null +++ b/internal/source/source.go @@ -0,0 +1,213 @@ +// Package source provides a sequential, single-pass iterator over release +// entries (regular files to upload). TarGzSource reads a tar.gz; a directory +// source may be added later behind the same interface. +package source + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "path" + "strings" + + "tsne.dev/hopper/internal/log" +) + +// Entry is a single regular file emitted by a Source. +type Entry struct { + Path string // cleaned, archive-relative key (no prefix applied). + Size int + IsMutable bool // true for mutable entry points (*.html, case-insensitive). + + content io.Reader // unexported; live source stream for THIS entry. +} + +// ReadContent reads the current entry's bytes fully into memory. +// On context cancellation it returns the context's cause promptly instead +// of finishing a large read. It is valid ONLY for the current entry and +// must be called before the next Source.Next(). +func (e Entry) ReadContent(ctx context.Context) ([]byte, error) { + const chunkSize = 512 << 10 // 512 KiB + + initialSize := e.Size + if initialSize == 0 { + initialSize = 4096 + } + + buf := make([]byte, initialSize) + off := 0 + for { + if err := context.Cause(ctx); err != nil { + return nil, err + } + + if len(buf)-off < chunkSize { + newbuf := make([]byte, max(2*len(buf), len(buf)+chunkSize)) + copy(newbuf, buf[:off]) + buf = newbuf + } + + read, err := e.content.Read(buf[off : off+chunkSize]) + if read > 0 { + off += read + } + if err != nil { + if errors.Is(err, io.EOF) { + return buf[:off], nil + } + return nil, err + } + } +} + +// Source is a sequential, single-pass iterator. Returns io.EOF when exhausted. +type Source interface { + Next() (Entry, error) + Close() error +} + +// TarGzSource streams compress/gzip -> archive/tar entries one at a time. +type TarGzSource struct { + f *os.File + gz *gzip.Reader + tr *tar.Reader + root string +} + +// NewTarGzSource opens the given tar.gz archive. The given root selects +// the subtree to emit and is stripped from each selected entry. If root +// is empty all entries are selected. +// +// Note: `root` must arrive pre-normalized (no leading/trailing slashes, +// cleaned of "."/".."). The cli package owns root normalization. +func NewTarGzSource(archivePath string, root string) (*TarGzSource, error) { + f, err := os.Open(archivePath) + if err != nil { + return nil, fmt.Errorf("open archive: %w", err) + } + gz, err := gzip.NewReader(f) + if err != nil { + f.Close() + return nil, fmt.Errorf("gzip: %w", err) + } + + return &TarGzSource{ + f: f, + gz: gz, + tr: tar.NewReader(gz), + root: root, + }, nil +} + +// Next returns the next regular-file entry, filtering by entry type and +// applying path-safety rules. Returns io.EOF when exhausted. +// +// Entry-type handling: regular files are emitted; directories are silently +// ignored; symlinks/hardlinks/devices/FIFOs/etc. are skipped with a warning +// (never fatal). +func (s *TarGzSource) Next() (Entry, error) { + for { + hdr, err := s.tr.Next() + if err != nil { + return Entry{}, err // includes io.EOF + } + + switch hdr.Typeflag { + case tar.TypeReg: + // emit below + case tar.TypeDir: + continue // silently ignored + default: + log.Warn("skipping non-regular source entry %q (type %q)", hdr.Name, string(hdr.Typeflag)) + continue + } + + cleaned, err := cleanArchivePath(hdr.Name) + if err != nil { + return Entry{}, err + } + + path, ok := s.selectPath(cleaned) + if !ok { + continue + } + + return Entry{ + Path: path, + Size: int(hdr.Size), + IsMutable: strings.HasSuffix(strings.ToLower(path), ".html"), + content: s.tr, + }, nil + } +} + +// Close releases the underlying resources. +func (s *TarGzSource) Close() error { + gzErr := s.gz.Close() + fErr := s.f.Close() + if gzErr != nil { + return gzErr + } + return fErr +} + +func (s *TarGzSource) selectPath(path string) (string, bool) { + if s.root == "" { + return path, true + } + prefix := s.root + "/" + if !strings.HasPrefix(path, prefix) { + return "", false + } + return path[len(prefix):], true +} + +// cleanArchivePath validates and cleans a raw archive path: +// - absolute path -> hard error; +// - cleaned path escaping the root (starts with "..") -> hard error; +// - stays within root but contained a ".." segment -> emit, with a warning; +// - pure "."/redundant-slash normalization -> silent. +func cleanArchivePath(raw string) (string, error) { + if path.IsAbs(raw) { + return "", fmt.Errorf("absolute path in archive: %q", raw) + } + + cleaned := path.Clean(raw) + + if cleaned == "." { + return "", fmt.Errorf("empty or invalid path in archive: %q", raw) + } + + if cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return "", fmt.Errorf("path escapes archive root: %q", raw) + } + + // Stayed within root but contained a ".." segment -> warn (legal, unusual). + if containsDotDot(raw) { + log.Warn("path %q contains '..' segment", raw) + } + + return cleaned, nil +} + +// containsDotDot reports whether p has a ".." path segment, scanning by index +// to avoid the allocation of strings.Split. +func containsDotDot(p string) bool { + for len(p) > 0 { + i := strings.IndexByte(p, '/') + var seg string + if i < 0 { + seg, p = p, "" + } else { + seg, p = p[:i], p[i+1:] + } + if seg == ".." { + return true + } + } + return false +} diff --git a/internal/source/source_test.go b/internal/source/source_test.go new file mode 100644 index 0000000..933c67a --- /dev/null +++ b/internal/source/source_test.go @@ -0,0 +1,95 @@ +package source + +import ( + "bytes" + "io" + "testing" + + "tsne.dev/hopper/internal/log" +) + +func TestCleanArchivePath(t *testing.T) { + tests := []struct { + raw string + want string + wantErr bool + warn bool + }{ + {"a/b.js", "a/b.js", false, false}, + {"./a//b.js", "a/b.js", false, false}, + {"/abs/x", "", true, false}, + {"../escape", "", true, false}, + {"a/../../escape", "", true, false}, + {"a/../b.js", "b.js", false, true}, // contained .. but stays in root + } + for _, tt := range tests { + var warn bytes.Buffer + log.Init(io.Discard, &warn) + got, err := cleanArchivePath(tt.raw) + if tt.wantErr { + if err == nil { + t.Errorf("%q: expected error", tt.raw) + } + continue + } + if err != nil { + t.Errorf("%q: unexpected error %v", tt.raw, err) + continue + } + if got != tt.want { + t.Errorf("%q: got %q want %q", tt.raw, got, tt.want) + } + if tt.warn && warn.Len() == 0 { + t.Errorf("%q: expected warning", tt.raw) + } + if !tt.warn && warn.Len() != 0 { + t.Errorf("%q: unexpected warning %q", tt.raw, warn.String()) + } + } +} + +func TestSelectPath(t *testing.T) { + tests := []struct { + cleaned string + root string + want string + wantOK bool + }{ + {"assets/x.js", "", "assets/x.js", true}, + {"README.md", "", "README.md", true}, + {"app/assets/x.js", "app", "assets/x.js", true}, + {"README.md", "app", "", false}, + {"app", "app", "", false}, + {"appendix/x", "app", "", false}, + {"dist/public/index.html", "dist/public", "index.html", true}, + {"dist/private/x", "dist/public", "", false}, + } + for _, tt := range tests { + s := &TarGzSource{root: tt.root} + got, ok := s.selectPath(tt.cleaned) + if ok != tt.wantOK { + t.Errorf("selectPath(%q, %q): ok = %v, want %v", tt.cleaned, tt.root, ok, tt.wantOK) + continue + } + if ok && got != tt.want { + t.Errorf("selectPath(%q, %q) = %q, want %q", tt.cleaned, tt.root, got, tt.want) + } + } +} + +func TestContainsDotDot(t *testing.T) { + cases := map[string]bool{ + "a/b/c": false, + "..": true, + "../x": true, + "a/../b": true, + "a/..b/c": false, + "": false, + "a/b/..": true, + } + for in, want := range cases { + if got := containsDotDot(in); got != want { + t.Errorf("containsDotDot(%q) = %v, want %v", in, got, want) + } + } +} |