diff options
Diffstat (limited to 'internal')
31 files changed, 4187 insertions, 0 deletions
diff --git a/internal/bunny/api.go b/internal/bunny/api.go new file mode 100644 index 0000000..1ffc1b5 --- /dev/null +++ b/internal/bunny/api.go @@ -0,0 +1,170 @@ +package bunny + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "tsne.dev/hopper/internal/log" +) + +// Object is a single node returned by a directory listing. +type Object struct { + Path string + Size int + IsDirectory bool + LastChanged time.Time +} + +type API struct { + endpoint string // hostname only (normalized, no scheme/trailing slash) + zone string + accessKey string + maxRetries int + http *http.Client +} + +func NewAPI(endpoint, zone, accessKey string) *API { + // normalize endpoint + endpoint = strings.TrimPrefix(endpoint, "https://") + endpoint = strings.TrimPrefix(endpoint, "http://") + endpoint = strings.TrimRight(endpoint, "/") + + return &API{ + endpoint: endpoint, + zone: zone, + accessKey: accessKey, + maxRetries: 3, + http: &http.Client{}, + } +} + +// do executes req under the shared retry policy and returns the final +// response (2xx) or a typed *apiError. It owns the universal parts: the +// AccessKey header, the 429+Retry-After retry loop, and status classification. +// Callers own method, body, and op-specific headers, and must read/close the +// returned body. method and path on any apiError are taken from req. The body +// is reset from req.GetBody before each attempt; callers that send a body MUST +// build the request with bytes.NewReader (so GetBody is set) or retries would +// resend an empty body. +func (api *API) do(ctx context.Context, req *http.Request) (*http.Response, error) { + req.Header.Set("AccessKey", api.accessKey) + + attempt := 0 + for { + attempt++ + if req.GetBody != nil { + body, err := req.GetBody() + if err != nil { + return nil, newAPIError(errorKindNetwork, err, req, 0) + } + req.Body = body + } + + resp, err := api.http.Do(req) + if err != nil { + if ctxErr := context.Cause(ctx); ctxErr != nil { + return nil, ctxErr + } + return nil, newAPIError(errorKindNetwork, err, req, 0) + } + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return resp, nil + } + + bunnyMsg := readResponseError(resp.Body) + resp.Body.Close() + + if resp.StatusCode == http.StatusTooManyRequests { + ra := resp.Header.Get("Retry-After") + if ra == "" { + return nil, newAPIError(errorKindRateLimited, errors.New("status 429, no retry-after header received"), req, 0) + } + if attempt > api.maxRetries { + return nil, newAPIError(errorKindRateLimited, errors.New("status 429, retries exhausted"), req, 0) + } + d, perr := parseRetryAfter(ra) + if perr != nil { + return nil, newAPIError(errorKindRateLimited, errors.New("status 429, "+perr.Error()), req, 0) + } + // Surface the cool-down so a long Retry-After wait doesn't look like a + // hang. Always warned (never suppressed): being rate-limited is + // abnormal and the wait is user-visible time. + log.Warn("rate-limited by Bunny (retry in %s; attempt %d/%d)", d.Round(time.Second), attempt, api.maxRetries) + select { + case <-ctx.Done(): + return nil, context.Cause(ctx) + case <-time.After(d): + } + continue + } + + errKind := errorKindFromStatus(resp.StatusCode) + var cause error + if bunnyMsg != "" { + cause = fmt.Errorf("http error %d: %s", resp.StatusCode, bunnyMsg) + } else { + cause = fmt.Errorf("http error %d", resp.StatusCode) + } + return nil, newAPIError(errKind, cause, req, resp.StatusCode) + } +} + +// readResponseError best-effort extracts Bunny's ErrorObject.Message from a +// non-2xx response body. It fully drains (bounded) the body so the connection +// can be reused, and returns "" when the body is absent, unparseable, or +// carries no Message. It never errors: enrichment is optional detail. +func readResponseError(body io.Reader) string { + data, err := io.ReadAll(io.LimitReader(body, 8<<10)) // 8 KiB + if err != nil || len(data) == 0 { + return "" + } + var e struct { + Message string `json:"Message"` + } + if err := json.Unmarshal(data, &e); err != nil { + return "" + } + return strings.TrimSpace(e.Message) +} + +func (api *API) url(path string) string { + return "https://" + api.endpoint + "/" + api.zone + "/" + path +} + +// parseTime parses Bunny's LastChanged timestamp, which is NOT RFC 3339 +// (it carries no timezone, e.g. "2023-06-29T12:00:00.000"), so it cannot be +// unmarshaled into a time.Time directly. +func parseTime(v string) time.Time { + if v == "" { + return time.Time{} + } + if t, err := time.Parse("2006-01-02T15:04:05.999", v); err == nil { + return t.Local() + } + if t, err := time.Parse(time.RFC3339, v); err == nil { + return t.Local() + } + return time.Time{} +} + +// parseRetryAfter handles the delay-seconds form and the HTTP-date form. +func parseRetryAfter(v string) (time.Duration, error) { + if secs, err := strconv.Atoi(v); err == nil { + if secs < 0 { + return 0, fmt.Errorf("negative retry-after header") + } + return time.Duration(secs) * time.Second, nil + } + if t, err := http.ParseTime(v); err == nil { + return max(0, time.Until(t)), nil + } + return 0, fmt.Errorf("unparseable retry-after header %q", v) +} diff --git a/internal/bunny/api_delete.go b/internal/bunny/api_delete.go new file mode 100644 index 0000000..868b0f3 --- /dev/null +++ b/internal/bunny/api_delete.go @@ -0,0 +1,46 @@ +package bunny + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" +) + +// Delete removes a single object with a raw DELETE, applying the shared retry +// policy. +func (api *API) Delete(ctx context.Context, path string) error { + return api.delete(ctx, api.url(path)) +} + +// DeleteDir removes a directory. Bunny's storage endpoints identify a directory +// by a trailing slash on the URL (the same quirk listDir relies on), so we +// append it here rather than leak the convention to callers. The delete is +// recursive. +// +// CAUTION: The delete is recursive AND prune decides emptiness from the +// pre-delete listing snapshot. If another actor uploads a file under this directory +// between the snapshot and this call, the recursive delete will remove it. +func (api *API) DeleteDir(ctx context.Context, path string) error { + u := api.url(path) + if !strings.HasSuffix(u, "/") { + u += "/" + } + return api.delete(ctx, u) +} + +func (api *API) delete(ctx context.Context, u string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, u, nil) + if err != nil { + return fmt.Errorf("build request: %w", err) + } + + resp, err := api.do(ctx, req) + if err != nil { + return err + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + return nil +} diff --git a/internal/bunny/api_list.go b/internal/bunny/api_list.go new file mode 100644 index 0000000..57a7817 --- /dev/null +++ b/internal/bunny/api_list.go @@ -0,0 +1,152 @@ +package bunny + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "tsne.dev/hopper/internal/log" +) + +// Iterator is an old-school pull iterator over the recursive remote listing under +// a starting directory. It yields every entry (files AND directories; a +// directory has Size 0) so consumers can build whatever structure they need: +// push collects files and verifies no file collides with a remote directory; +// prune derives orphans and empty directories. Usage: +// +// it := client.List(ctx, dir) +// for it.Next() { +// rec := it.Object() +// } +// if err := it.Err(); err != nil { ... } +// +// The walk is iterative: a stack holds directories still to list, each listed +// on demand. A 404 listing is an empty subtree (skipped, not fatal); any other +// listing failure stops iteration and surfaces via Err. +type Iterator struct { + ctx context.Context + listDir func(context.Context, string) ([]Object, error) + stack []string + pending []Object + cur Object + err error + done bool +} + +// NewIterator builds an Iterator rooted at dir, listing each directory via listDir. +func NewIterator(ctx context.Context, dir string, listDir func(context.Context, string) ([]Object, error)) *Iterator { + stack := make([]string, 0, 4) + stack = append(stack, dir) + return &Iterator{ctx: ctx, listDir: listDir, stack: stack} +} + +// Next advances to the next entry, returning false when the walk is exhausted +// or a listing failed (check Err). Directory entries are yielded and also queued +// for descent. +func (it *Iterator) Next() bool { + if it.err != nil || it.done { + return false + } + for { + if len(it.pending) > 0 { + o := it.pending[0] + it.pending = it.pending[1:] + if o.IsDirectory { + it.stack = append(it.stack, o.Path) + } + it.cur = o + return true + } + if len(it.stack) == 0 { + it.done = true + return false + } + dir := it.stack[len(it.stack)-1] + it.stack = it.stack[:len(it.stack)-1] + objs, err := it.listDir(it.ctx, dir) + if err != nil { + var apiErr *apiError + if errors.As(err, &apiErr) && apiErr.status == http.StatusNotFound { + continue // 404 = empty subtree + } + it.err = err + return false + } + it.pending = objs + } +} + +// Object returns the entry reached by the most recent Next. +func (it *Iterator) Object() Object { + return it.cur +} + +// Err returns the listing failure that stopped iteration, if any. +func (it *Iterator) Err() error { + return it.err +} + +// List returns an Iterator over the recursive listing rooted at dir. An empty +// dir denotes zone root. +func (api *API) List(ctx context.Context, dir string) *Iterator { + return NewIterator(ctx, dir, api.listDir) +} + +// listDir returns the immediate children of dir. Bunny's list endpoint is +// per-directory and expects a trailing slash on the directory URL. Returned +// Object.Path is the full storage key relative to the zone (matching source +// keys), so the skip-check can match it. +func (api *API) listDir(ctx context.Context, dir string) ([]Object, error) { + if dir == "" { + log.PrintVerbose("listing zone root") + } else { + log.PrintVerbose("listing %s", dir) + } + + u := api.url(dir) + if !strings.HasSuffix(u, "/") { + u += "/" + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + + resp, err := api.do(ctx, req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var raw []listObject + if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + return nil, newAPIError(errorKindInternal, errors.New("failed to decode list response ("+err.Error()+")"), req, resp.StatusCode) + } + + prefix := "/" + api.zone + "/" + objs := make([]Object, len(raw)) + for i, o := range raw { + objs[i] = Object{ + Path: strings.TrimPrefix(o.Path, prefix) + o.ObjectName, + Size: o.Length, + IsDirectory: o.IsDirectory, + LastChanged: parseTime(o.LastChanged), + } + } + return objs, nil +} + +// listObject mirrors a single entry of Bunny's per-directory list response. The +// full storage key is Path+ObjectName (Path includes the zone and trailing +// slash); see List. +type listObject struct { + Path string `json:"Path"` + ObjectName string `json:"ObjectName"` + Length int `json:"Length"` + LastChanged string `json:"LastChanged"` + IsDirectory bool `json:"IsDirectory"` +} diff --git a/internal/bunny/api_test.go b/internal/bunny/api_test.go new file mode 100644 index 0000000..e74a3b9 --- /dev/null +++ b/internal/bunny/api_test.go @@ -0,0 +1,349 @@ +package bunny + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func newTestAPI(srvURL string) *API { + api := NewAPI(srvURL, "zone", "key") + api.http = &http.Client{Transport: rewriteTransport{}} + return api +} + +// rewriteTransport downgrades https://host/... to http://host/... for tests. +type rewriteTransport struct{} + +func (rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.URL.Scheme = "http" + return http.DefaultTransport.RoundTrip(req) +} + +func TestUploadSuccess(t *testing.T) { + var gotKey, gotChecksum, gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotKey = r.Header.Get("AccessKey") + gotChecksum = r.Header.Get("Checksum") + gotPath = r.URL.Path + if r.Header.Get("Content-Type") != "" { + t.Errorf("unexpected Content-Type %q", r.Header.Get("Content-Type")) + } + w.WriteHeader(201) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + if err := api.Upload(context.Background(), "a/b.js", []byte("data")); err != nil { + t.Fatalf("Upload: %v", err) + } + sum := sha256.Sum256([]byte("data")) + wantChecksum := strings.ToUpper(hex.EncodeToString(sum[:])) + if gotKey != "key" || gotChecksum != wantChecksum { + t.Fatalf("headers: key=%q checksum=%q (want %q)", gotKey, gotChecksum, wantChecksum) + } + if gotPath != "/zone/a/b.js" { + t.Fatalf("path = %q", gotPath) + } +} + +func TestUploadRetriesOn429WithRetryAfter(t *testing.T) { + var n int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if atomic.AddInt32(&n, 1) <= 2 { + w.Header().Set("Retry-After", "0") + w.WriteHeader(429) + return + } + w.WriteHeader(200) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + if err := api.Upload(context.Background(), "x", []byte("d")); err != nil { + t.Fatalf("Upload: %v", err) + } + if n != 3 { + t.Fatalf("attempts = %d, want 3", n) + } +} + +func TestUploadFailsOn429NoRetryAfter(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(429) + })) + defer srv.Close() + api := newTestAPI(srv.URL) + if err := api.Upload(context.Background(), "x", []byte("d")); err == nil { + t.Fatal("expected error") + } +} + +func TestUploadFailsOn5xx(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer srv.Close() + api := newTestAPI(srv.URL) + if err := api.Upload(context.Background(), "x", []byte("d")); err == nil { + t.Fatal("expected error") + } +} + +func TestUploadRetriesExhausted(t *testing.T) { + var n int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&n, 1) + w.Header().Set("Retry-After", "0") + w.WriteHeader(429) + })) + defer srv.Close() + api := newTestAPI(srv.URL) + if err := api.Upload(context.Background(), "x", []byte("d")); err == nil { + t.Fatal("expected error") + } + if n != 4 { // initial + 3 retries + t.Fatalf("attempts = %d, want 4", n) + } +} + +func TestUploadClassifiesStatus(t *testing.T) { + cases := []struct { + status int + kind errorKind + }{ + {http.StatusUnauthorized, errorKindAuth}, + {http.StatusForbidden, errorKindAuth}, + {http.StatusBadRequest, errorKindInternal}, + {http.StatusNotFound, errorKindInternal}, + {http.StatusInternalServerError, errorKindServer}, + {http.StatusBadGateway, errorKindServer}, + } + for _, tc := range cases { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.status) + })) + api := newTestAPI(srv.URL) + err := api.Upload(context.Background(), "x", []byte("d")) + srv.Close() + + var apiErr *apiError + if !errors.As(err, &apiErr) { + t.Fatalf("status %d: want *APIError, got %T (%v)", tc.status, err, err) + } + if apiErr.kind != tc.kind { + t.Fatalf("status %d: kind = %d, want %d", tc.status, apiErr.kind, tc.kind) + } + if apiErr.status != tc.status { + t.Fatalf("status %d: APIError.Status = %d", tc.status, apiErr.status) + } + } +} + +func TestUploadSurfacesBunnyErrorMessage(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"HttpCode":400,"Message":"checksum mismatch"}`)) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + err := api.Upload(context.Background(), "x", []byte("d")) + + var apiErr *apiError + if !errors.As(err, &apiErr) { + t.Fatalf("want *apiError, got %T (%v)", err, err) + } + if got := apiErr.ErrorVerbose(); !strings.Contains(got, "checksum mismatch") { + t.Fatalf("verbose detail = %q, want it to contain Bunny's Message", got) + } +} + +func TestListParsesFullPaths(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Write([]byte(`[ + {"Path":"/zone/assets/","ObjectName":"app.abc.js","Length":14,"IsDirectory":false,"LastChanged":"2023-06-29T12:00:00.000"}, + {"Path":"/zone/assets/","ObjectName":"sub","Length":0,"IsDirectory":true,"LastChanged":"2023-06-29T12:00:00"} + ]`)) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + objs, err := api.listDir(context.Background(), "assets") + if err != nil { + t.Fatalf("listDir: %v", err) + } + if gotPath != "/zone/assets/" { + t.Fatalf("list path = %q, want trailing slash", gotPath) + } + if len(objs) != 2 { + t.Fatalf("objs = %v", objs) + } + if objs[0].Path != "assets/app.abc.js" || objs[0].Size != 14 || objs[0].IsDirectory { + t.Fatalf("file object = %+v", objs[0]) + } + if objs[1].Path != "assets/sub" || !objs[1].IsDirectory { + t.Fatalf("dir object = %+v", objs[1]) + } +} + +func TestListRootTrailingSlash(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Write([]byte(`[]`)) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + if _, err := api.listDir(context.Background(), ""); err != nil { + t.Fatalf("listDir: %v", err) + } + if gotPath != "/zone/" { + t.Fatalf("root list path = %q, want /zone/", gotPath) + } +} + +func TestWalkBuildsTreeAndDescends(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/zone/": + w.Write([]byte(`[ + {"Path":"/zone/","ObjectName":"index.html","Length":6,"IsDirectory":false}, + {"Path":"/zone/","ObjectName":"assets","IsDirectory":true} + ]`)) + case "/zone/assets/": + w.Write([]byte(`[{"Path":"/zone/assets/","ObjectName":"app.js","Length":3,"IsDirectory":false}]`)) + default: + t.Errorf("unexpected list path %q", r.URL.Path) + } + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + it := api.List(context.Background(), "") + var got []Object + for it.Next() { + got = append(got, it.Object()) + } + if err := it.Err(); err != nil { + t.Fatalf("Walk: %v", err) + } + if len(got) != 3 { + t.Fatalf("got %d entries: %+v", len(got), got) + } + var sawDir, sawNested bool + for _, o := range got { + if o.Path == "assets" && o.IsDirectory { + sawDir = true + } + if o.Path == "assets/app.js" && !o.IsDirectory && o.Size == 3 { + sawNested = true + } + } + if !sawDir || !sawNested { + t.Fatalf("expected descent into assets; got %+v", got) + } +} + +func TestWalk404IsEmpty(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + it := api.List(context.Background(), "") + for it.Next() { + t.Fatalf("unexpected entry %+v", it.Object()) + } + if err := it.Err(); err != nil { + t.Fatalf("Walk: %v", err) + } +} + +func TestWalkFatalOnNon404(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + it := api.List(context.Background(), "") + for it.Next() { + } + if it.Err() == nil { + t.Fatal("expected error") + } +} + +func TestUploadRateLimitedIsTyped(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(429) + })) + defer srv.Close() + api := newTestAPI(srv.URL) + err := api.Upload(context.Background(), "x", []byte("d")) + var apiErr *apiError + if !errors.As(err, &apiErr) || apiErr.kind != errorKindRateLimited { + t.Fatalf("want KindRateLimited APIError, got %v", err) + } +} + +func TestDeleteSuccess(t *testing.T) { + var gotMethod, gotKey, gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + gotKey = r.Header.Get("AccessKey") + gotPath = r.URL.Path + w.WriteHeader(200) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + if err := api.Delete(context.Background(), "a/b.js"); err != nil { + t.Fatalf("Delete: %v", err) + } + if gotMethod != http.MethodDelete { + t.Fatalf("method = %q", gotMethod) + } + if gotKey != "key" || gotPath != "/zone/a/b.js" { + t.Fatalf("key=%q path=%q", gotKey, gotPath) + } +} + +func TestDeleteFailsOn5xx(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer srv.Close() + + api := newTestAPI(srv.URL) + if err := api.Delete(context.Background(), "a/b.js"); err == nil { + t.Fatal("expected error on 5xx") + } +} + +// TestAPIErrorUnwrapsCause ensures a network apiError preserves its cause in the +// error chain, so errors.Is can detect e.g. context.Canceled from an aborted +// in-flight upload. Without this, the worker pool cannot filter cancellation +// collateral out of a fail-fast failure report. +func TestAPIErrorUnwrapsCause(t *testing.T) { + req, err := http.NewRequest(http.MethodPut, "https://example.test/zone/a.js", nil) + if err != nil { + t.Fatal(err) + } + // Simulate a transport error whose cause is context.Canceled. + apiErr := newAPIError(errorKindNetwork, context.Canceled, req, 0) + if !errors.Is(apiErr, context.Canceled) { + t.Fatalf("apiError must unwrap to its cause; got chain that is not context.Canceled") + } +} diff --git a/internal/bunny/api_upload.go b/internal/bunny/api_upload.go new file mode 100644 index 0000000..f2af534 --- /dev/null +++ b/internal/bunny/api_upload.go @@ -0,0 +1,50 @@ +package bunny + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "io" + "net/http" + "unsafe" +) + +// Upload performs a raw PUT with AccessKey and Checksum headers, applying the +// retry policy: a 429 with Retry-After is retried up to maxRetries times, +// honoring the header; every other non-2xx (429 without Retry-After, 5xx, +// other 4xx) and any network error fails. The Checksum header is the +// hex-encoded SHA256 of body, computed here so callers need not know Bunny's +// integrity scheme. +func (api *API) Upload(ctx context.Context, path string, body []byte) error { + sum := sha256.Sum256(body) + checksum := upperHexEncode(sum[:]) + + // bytes.NewReader lets net/http set req.GetBody automatically, so do can + // replay the body on each retry attempt. + req, err := http.NewRequestWithContext(ctx, http.MethodPut, api.url(path), bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("build request: %w", err) + } + req.Header.Set("Checksum", checksum) + req.ContentLength = int64(len(body)) + + resp, err := api.do(ctx, req) + if err != nil { + return err + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + return nil +} + +func upperHexEncode(src []byte) string { + const alphabet = "0123456789ABCDEF" + + dst := make([]byte, len(src)*2) + for i, b := range src { + dst[i*2] = alphabet[b>>4] + dst[i*2+1] = alphabet[b&0x0f] + } + return unsafe.String(unsafe.SliceData(dst), len(dst)) +} diff --git a/internal/bunny/error.go b/internal/bunny/error.go new file mode 100644 index 0000000..4c17bda --- /dev/null +++ b/internal/bunny/error.go @@ -0,0 +1,95 @@ +package bunny + +import ( + "fmt" + "net/http" +) + +type errorKind int + +const ( + // errorKindNetwork: the request never got a response (DNS, connection, TLS, + // context cancellation surfaced as a transport error). + errorKindNetwork errorKind = iota + // errorKindAuth: the access key was rejected (401/403). + errorKindAuth + // errorKindRateLimited: Bunny is throttling (429, in any retry-exhausted form). + errorKindRateLimited + // errorKindServer: Bunny-side failure (5xx). + errorKindServer + // errorKindInternal: an unexpected client error (non-auth, non-rate-limit 4xx). + // These almost always mean hopper built a bad request: a likely bug. + errorKindInternal + // errorKindTooLarge: Bunny rejected the asset as too large (413). This is the + // operator's payload, not a hopper bug, so it gets a distinct message. + errorKindTooLarge +) + +func errorKindFromStatus(status int) errorKind { + switch { + case status == http.StatusUnauthorized || status == http.StatusForbidden: + return errorKindAuth + case status == http.StatusRequestEntityTooLarge: + return errorKindTooLarge + case status >= 500: + return errorKindServer + default: + return errorKindInternal + } +} + +// apiError is the single error type returned by all API operations. Error() +// renders a friendly, kind-based message for the user; the technical detail +// (method, path, status, wrapped cause) is exposed separately via +// ErrorVerbose() and surfaced only under --verbose by the presentation layer. +// method and path are derived from the request itself (HTTP verb and +// req.URL.Path). +type apiError struct { + method string // HTTP verb, e.g. "PUT" + path string // req.URL.Path, e.g. "/zone/a/b.js" + status int // HTTP status code; 0 for transport-level (KindNetwork) errors + kind errorKind + cause error // wrapped technical cause, exposed via Unwrap (NOT via Error) +} + +func newAPIError(kind errorKind, err error, req *http.Request, statusCode int) *apiError { + return &apiError{ + method: req.Method, + path: req.URL.Path, + status: statusCode, + kind: kind, + cause: err, + } +} + +func (e *apiError) Unwrap() error { return e.cause } + +// ErrorVerbose returns the technical description (verb, path, status, cause) +// surfaced only under --verbose. It is the "re-run with --verbose for details" +// payload; the user-facing message stays in Error(). Any error type may +// implement this interface to contribute extra detail to failure reports. +func (e *apiError) ErrorVerbose() string { + if e.status != 0 { + return fmt.Sprintf("%s %s -> %d: %v", e.method, e.path, e.status, e.cause) + } + return fmt.Sprintf("%s %s -> %v", e.method, e.path, e.cause) +} + +func (e *apiError) Error() string { + switch e.kind { + case errorKindAuth: + return "The access key was rejected. Check your Bunny storage credentials." + case errorKindRateLimited: + return "Bunny is rate-limiting requests. Please try again shortly." + case errorKindServer: + return "Bunny had an internal server error. Please try again later." + case errorKindNetwork: + return "Couldn't reach Bunny. Check your network connection and endpoint." + case errorKindTooLarge: + return "The asset was rejected by Bunny as too large." + case errorKindInternal: + return "Unexpected response from Bunny. This is likely a bug in hopper. Please report it (re-run with --verbose for details)." + default: + return "An unknown Bunny error occurred (re-run with --verbose for details)." + } +} diff --git a/internal/cli/cmd.go b/internal/cli/cmd.go new file mode 100644 index 0000000..698cf7c --- /dev/null +++ b/internal/cli/cmd.go @@ -0,0 +1,73 @@ +package cli + +import ( + "context" + "errors" + "io" + "os" + + "github.com/spf13/pflag" + + "tsne.dev/hopper/internal/log" +) + +// Run executes the CLI and returns the process exit code. +// +// Note: Help paths will call exit directly. +func Run(ctx context.Context, args []string) int { + main := newMainCommand() + err := runCommand(ctx, main, args) + if err != nil { + var exit exitError + if errors.As(err, &exit) { + return exit.code + } + + var ev errorVerboser + if errors.As(err, &ev) { + log.Error("%s\n %s", err.Error(), ev.ErrorVerbose()) + } else { + log.Error(err.Error()) + } + return 1 + } + return 0 +} + +type command interface { + Usage() string + Run(ctx context.Context, args []string) error +} + +func parseFlags(cmd command, flags *pflag.FlagSet, args []string) { + err := flags.Parse(args) + if err != nil { + if errors.Is(err, pflag.ErrHelp) { + log.Print(cmd.Usage()) + os.Exit(0) + } else { + log.Error("%s\n\n%s", err.Error(), cmd.Usage()) + os.Exit(2) + } + } +} + +func runCommand(ctx context.Context, cmd command, args []string) error { + err := cmd.Run(ctx, args) + if err != nil { + var usageErr usageError + if errors.As(err, &usageErr) { + log.Error("%s\n\n%s", usageErr.msg, cmd.Usage()) + os.Exit(2) + } + return err + } + return nil +} + +func newFlagset(name string) *pflag.FlagSet { + fs := pflag.NewFlagSet(name, pflag.ContinueOnError) + fs.SetOutput(io.Discard) + fs.Usage = func() {} + return fs +} diff --git a/internal/cli/cmd_main.go b/internal/cli/cmd_main.go new file mode 100644 index 0000000..3d4c3db --- /dev/null +++ b/internal/cli/cmd_main.go @@ -0,0 +1,130 @@ +package cli + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/spf13/pflag" + + "tsne.dev/hopper/internal/log" +) + +var _ command = (*mainCommand)(nil) + +type mainCommand struct { + flags *pflag.FlagSet + + zone string + endpoint string + accessKeyFile string + accessKey string + verbose bool + quiet bool +} + +func newMainCommand() *mainCommand { + cmd := &mainCommand{} + cmd.flags = newFlagset("hopper") + cmd.flags.SetInterspersed(false) + cmd.flags.StringVar(&cmd.zone, + "zone", + lookupEnv("BUNNY_ZONE", ""), + "Bunny storage zone name (required) (env: BUNNY_ZONE)", + ) + cmd.flags.StringVar(&cmd.endpoint, + "endpoint", + lookupEnv("BUNNY_ENDPOINT", "storage.bunnycdn.com"), + "Bunny storage endpoint hostname (env: BUNNY_ENDPOINT)", + ) + cmd.flags.StringVar(&cmd.accessKeyFile, + "access-key-file", + lookupEnv("BUNNY_ACCESS_KEY_FILE", ""), + "path to a file containing the storage access key (env: BUNNY_ACCESS_KEY_FILE)", + ) + cmd.flags.BoolVar( + &cmd.verbose, + "verbose", + false, + "print more details", + ) + cmd.flags.BoolVar( + &cmd.quiet, + "quiet", + false, + "print only the final summary and errors", + ) + return cmd +} + +func (cmd *mainCommand) Usage() string { + return "Usage:" + + "\n hopper [global flags] ⟨command⟩ [command flags]" + + "\n" + + "\nCommands:" + + "\n push upload a release to Bunny Storage" + + "\n prune remove orphaned files from Bunny Storage" + + "\n" + + "\nGlobal flags:" + + "\n" + cmd.flags.FlagUsages() +} + +func (cmd *mainCommand) Run(ctx context.Context, args []string) error { + parseFlags(cmd, cmd.flags, args) + + log.SetVerbose(cmd.verbose) + log.SetQuiet(cmd.quiet) + + if cmd.accessKeyFile == "" { + cmd.accessKey = os.Getenv("BUNNY_ACCESS_KEY") + } + + switch { + case cmd.zone == "": + return usageError{"missing zone (provide --zone or BUNNY_ZONE)"} + case cmd.endpoint == "": + // --endpoint has a default, so an empty value here is always deliberate. + return usageError{"the endpoint must not be empty"} + case cmd.accessKey == "" && cmd.accessKeyFile == "": + return usageError{"missing access key (provide --access-key-file, BUNNY_ACCESS_KEY_FILE, or BUNNY_ACCESS_KEY)"} + } + + subargs := cmd.flags.Args() + if len(subargs) == 0 || subargs[0] == "" { + return usageError{"missing subcommand"} + } + + var sub command + switch subargs[0] { + case "push": + sub = newPushCommand(cmd) + case "prune": + sub = newPruneCommand(cmd) + default: + return usageError{fmt.Sprintf("unknown subcommand %q", subargs[0])} + } + + return runCommand(ctx, sub, subargs[1:]) +} + +func (cmd *mainCommand) lookupAccessKey() error { + if cmd.accessKey == "" { + accessKey, err := os.ReadFile(cmd.accessKeyFile) + if err != nil { + return fmt.Errorf("read access key file: %w", err) + } + cmd.accessKey = strings.TrimSpace(string(accessKey)) + } + if cmd.accessKey == "" { + return usageError{"empty access key"} + } + return nil +} + +func lookupEnv(env string, defaultValue string) string { + if val := os.Getenv(env); val != "" { + return val + } + return defaultValue +} diff --git a/internal/cli/cmd_main_test.go b/internal/cli/cmd_main_test.go new file mode 100644 index 0000000..d383970 --- /dev/null +++ b/internal/cli/cmd_main_test.go @@ -0,0 +1,126 @@ +package cli + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// clearGlobalEnv neutralizes the BUNNY_ env fallbacks so a test controls +// resolution entirely through flags. +func clearGlobalEnv(t *testing.T) { + t.Helper() + for _, k := range []string{"BUNNY_ZONE", "BUNNY_ENDPOINT", "BUNNY_ACCESS_KEY_FILE", "BUNNY_ACCESS_KEY"} { + t.Setenv(k, "") + } +} + +func writeKeyFile(t *testing.T) string { + t.Helper() + p := filepath.Join(t.TempDir(), "key") + if err := os.WriteFile(p, []byte("secret\n"), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func runMain(args []string) error { + return newMainCommand().Run(context.Background(), args) +} + +func TestMainMissingSubcommand(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + + var ue usageError + if err := runMain(nil); !errors.As(err, &ue) { + t.Fatalf("expected usageError, got %v", err) + } +} + +func TestMainMissingZone(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + + var ue usageError + err := runMain([]string{"push"}) + if !errors.As(err, &ue) || !strings.Contains(err.Error(), "zone") { + t.Fatalf("expected missing-zone usageError, got %v", err) + } +} + +func TestMainMissingAccessKey(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + + var ue usageError + err := runMain([]string{"--zone", "z", "push"}) + if !errors.As(err, &ue) || !strings.Contains(err.Error(), "access key") { + t.Fatalf("expected missing-access-key usageError, got %v", err) + } +} + +func TestMainUnknownSubcommand(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + kf := writeKeyFile(t) + + var ue usageError + err := runMain([]string{"--zone", "z", "--access-key-file", kf, "frob"}) + if !errors.As(err, &ue) || !strings.Contains(err.Error(), "unknown subcommand") { + t.Fatalf("expected unknown-subcommand usageError, got %v", err) + } +} + +func TestMainDispatchesToPrune(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + kf := writeKeyFile(t) + + // A fully-resolved global set reaches dispatch; the missing archive surfaces a + // plain (non-usage) error, proving parse + validation + dispatch worked. + err := runMain([]string{"--zone", "z", "--access-key-file", kf, "prune", "no-such.tgz"}) + if err == nil || !strings.Contains(err.Error(), "open archive") { + t.Fatalf("expected prune dispatch error, got %v", err) + } +} + +func TestMainAccessKeyFromEnv(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + t.Setenv("BUNNY_ACCESS_KEY", "literalkey") + + // With no --access-key-file, the literal BUNNY_ACCESS_KEY resolves the key + // and resolution reaches prune. + err := runMain([]string{"--zone", "z", "prune", "no-such.tgz"}) + if err == nil || !strings.Contains(err.Error(), "open archive") { + t.Fatalf("expected prune dispatch error (env access key resolved), got %v", err) + } +} + +func TestMainZoneFromEnv(t *testing.T) { + clearGlobalEnv(t) + captureLogs(t) + t.Setenv("BUNNY_ZONE", "envzone") + kf := writeKeyFile(t) + + // With zone supplied via env, resolution succeeds and reaches prune. + err := runMain([]string{"--access-key-file", kf, "prune", "no-such.tgz"}) + if err == nil || !strings.Contains(err.Error(), "open archive") { + t.Fatalf("expected prune dispatch error (env zone resolved), got %v", err) + } +} + +func TestLookupEnv(t *testing.T) { + t.Setenv("BUNNY_TEST_LOOKUP", "value") + if got := lookupEnv("BUNNY_TEST_LOOKUP", "fallback"); got != "value" { + t.Fatalf("set env: got %q", got) + } + t.Setenv("BUNNY_TEST_LOOKUP", "") + if got := lookupEnv("BUNNY_TEST_LOOKUP", "fallback"); got != "fallback" { + t.Fatalf("empty env should fall back: got %q", got) + } +} diff --git a/internal/cli/cmd_prune.go b/internal/cli/cmd_prune.go new file mode 100644 index 0000000..14c3f0f --- /dev/null +++ b/internal/cli/cmd_prune.go @@ -0,0 +1,186 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "io" + "sync/atomic" + "time" + + "github.com/spf13/pflag" + + "tsne.dev/hopper/internal/log" + "tsne.dev/hopper/internal/source" +) + +var _ command = (*pruneCommand)(nil) + +type pruneCommand struct { + flags *pflag.FlagSet + main *mainCommand + + concurrency uint + prefix string + root string + olderThan duration +} + +func newPruneCommand(main *mainCommand) *pruneCommand { + cmd := &pruneCommand{ + flags: newFlagset("prune"), + main: main, + } + cmd.flags.UintVar(&cmd.concurrency, "concurrency", 0, "number of concurrent transfers (0 = auto-detect based on CPUs)") + registerPathPrefixFlag(cmd.flags, &cmd.prefix, "prefix", "", "path prefix within the zone to target") + registerPathPrefixFlag(cmd.flags, &cmd.root, "root", "", "subtree of the source to consider live") + registerDurationFlag(cmd.flags, &cmd.olderThan, "older-than", "30d", "grace window, only orphans older than this are deleted") + registerDryRunFlag(cmd.flags) + return cmd +} + +func (cmd *pruneCommand) Usage() string { + return "Usage:" + + "\n hopper [global flags] prune [flags] ⟨source⟩" + + "\n" + + "\nFlags:" + + "\n" + cmd.flags.FlagUsages() +} + +func (cmd *pruneCommand) Run(ctx context.Context, args []string) error { + parseFlags(cmd, cmd.flags, args) + + if cmd.olderThan.isZero() { + return usageError{"grace window must not be zero"} + } + + positionals := cmd.flags.Args() + if len(positionals) == 0 || positionals[0] == "" { + return usageError{"missing source"} + } + if len(positionals) > 1 { + return usageError{"only a single source is supported"} + } + + if err := cmd.main.lookupAccessKey(); err != nil { + return err + } + + api := newStorageAPI(cmd.main.endpoint, cmd.main.zone, cmd.main.accessKey) + return cmd.prune(ctx, api, positionals[0], time.Now()) +} + +func (cmd *pruneCommand) prune(ctx context.Context, api storageAPI, sourceArg string, now time.Time) error { + start := time.Now() + + live, err := cmd.liveSet(sourceArg) + if err != nil { + return err + } + if len(live) == 0 { + return errors.New("empty source (refusing to prune). check --root, --prefix, or the archive contents") + } + + entries, err := listRemoteEntries(ctx, api, cmd.prefix) + if err != nil { + return err + } + + cutoff := cmd.olderThan.cutoff(now) + candidates := make([]*remoteEntry, 0, len(entries)/4) + for i := range entries { + e := &entries[i] + if !e.isDir && !live[e.path] { + if e.lastChanged.IsZero() { + log.Warn("could not determine last modified date for %s (skipping)", e.path) + } else if e.lastChanged.Before(cutoff) { + candidates = append(candidates, e) + } + } + } + + // Each delete is independent, so the pool never fails fast. A successful + // delete tombstones its entry. + pool := newWorkerPool(ctx, cmd.concurrency, false) + freed := int64(0) + for _, e := range candidates { + action := fmt.Sprintf("deleting %s (%s)", e.path, humanSize(e.size)) + pool.submit(action, func(ctx context.Context) error { + if err := api.Delete(ctx, e.path); err != nil { + return fileError{path: e.path, err: err} + } + e.deleted = true + atomic.AddInt64(&freed, int64(e.size)) + return nil + }) + } + + deleted, errs := pool.wait() + + var cmdErr error + if reportFailures(ctx, "failed to delete files", errs) { + cmdErr = exitError{1} + } + + cmd.removeEmptyDirs(ctx, api, entries) + + failed := len(errs) + logSummary("deleted: %d"+ + "\nfailed: %d"+ + "\nfreed: %s"+ + "\nelapsed: %s", + deleted, failed, humanSize64(freed), time.Since(start).Round(time.Millisecond), + ) + + return cmdErr +} + +// removeEmptyDirs deletes directories left with no surviving children after the +// file deletes and returns the number successfully removed. It runs as a +// best-effort second pass: failures are reported but never propagated to the +// exit code. +func (cmd *pruneCommand) removeEmptyDirs(ctx context.Context, api storageAPI, entries []remoteEntry) { + dirs := emptyDirs(entries) + if len(dirs) == 0 { + return + } + + pool := newWorkerPool(ctx, cmd.concurrency, false) + for _, dir := range dirs { + action := fmt.Sprintf("removing empty directory %s", dir.path) + pool.submit(action, func(ctx context.Context) error { + err := api.DeleteDir(ctx, dir.path) + if err != nil { + return fileError{path: dir.path, err: err} + } + return nil + }) + } + + _, errs := pool.wait() + if len(errs) > 0 { + _ = reportFailures(ctx, "failed to remove empty directories", errs) + } +} + +// liveSet walks the source once and returns the set of prefixed storage keys in +// the current release. It iterates entries only and never reads content. +func (cmd *pruneCommand) liveSet(sourceArg string) (map[string]bool, error) { + src, err := source.NewTarGzSource(sourceArg, cmd.root) + if err != nil { + return nil, err + } + defer src.Close() + + live := map[string]bool{} + for { + entry, err := src.Next() + if err != nil { + if errors.Is(err, io.EOF) { + return live, nil + } + return nil, err + } + live[prefixedPath(cmd.prefix, entry.Path)] = true + } +} diff --git a/internal/cli/cmd_prune_test.go b/internal/cli/cmd_prune_test.go new file mode 100644 index 0000000..e3b9398 --- /dev/null +++ b/internal/cli/cmd_prune_test.go @@ -0,0 +1,244 @@ +package cli + +import ( + "context" + "errors" + "sort" + "strings" + "testing" + "time" + + "tsne.dev/hopper/internal/bunny" +) + +func runPrune(source string, api storageAPI, older duration, now time.Time) error { + cmd := &pruneCommand{concurrency: 8, olderThan: older} + return cmd.prune(context.Background(), api, source, now) +} + +func days(n int) duration { return duration{days: n} } + +func sortedDeletes(f *fakeAPI) []string { + f.mu.Lock() + defer f.mu.Unlock() + out := append([]string(nil), f.deletes...) + sort.Strings(out) + return out +} + +// TestPruneSelectsCandidates covers the core §10 correctness properties: an +// orphan older than the cutoff is deleted; a file still in the release is kept +// even with an old LastChanged; an orphan newer than the cutoff is kept. +func TestPruneSelectsCandidates(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/live.js", body: "x"}, + }) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + recent := now.AddDate(0, 0, -5) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "index.html", LastChanged: old}, // live, old -> kept + {Path: "assets", IsDirectory: true}, + {Path: "orphan-old.js", Size: 3, LastChanged: old}, // orphan, old -> deleted + {Path: "orphan-new.js", LastChanged: recent}, // orphan, new -> kept + } + fc.lists["assets"] = []bunny.Object{ + {Path: "assets/live.js", LastChanged: old}, // live asset, old -> kept + {Path: "assets/gone.js", Size: 4, LastChanged: old}, // orphan, old -> deleted + } + captureLogs(t) + + if err := runPrune(arc, fc, days(30), now); err != nil { + t.Fatalf("prune: %v", err) + } + + got := sortedDeletes(fc) + want := []string{"assets/gone.js", "orphan-old.js"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("deletes = %v, want %v", got, want) + } +} + +func TestPruneAppliesPrefix(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "app.js", body: "x"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists["foo"] = []bunny.Object{ + {Path: "foo/app.js", LastChanged: old}, // live (prefixed) -> kept + {Path: "foo/stale.js", LastChanged: old}, // orphan -> deleted + } + captureLogs(t) + + cmd := &pruneCommand{concurrency: 8, prefix: "foo", olderThan: days(30)} + if err := cmd.prune(context.Background(), fc, arc, now); err != nil { + t.Fatalf("prune: %v", err) + } + if got := sortedDeletes(fc); len(got) != 1 || got[0] != "foo/stale.js" { + t.Fatalf("deletes = %v, want [foo/stale.js]", got) + } +} + +func TestPruneEmptyLiveSetIsFatal(t *testing.T) { + arc := makeArchive(t, nil) + fc := newFakeAPI() + // A List failure would be reported if reached; the empty-guard must fire + // first, so no Bunny call happens. + fc.listErr = errors.New("should not be called") + captureLogs(t) + + err := runPrune(arc, fc, days(30), time.Now()) + if err == nil || !strings.Contains(err.Error(), "empty source") { + t.Fatalf("expected empty-live-set error, got %v", err) + } + if len(fc.deletes) != 0 { + t.Fatalf("expected no deletes, got %v", fc.deletes) + } +} + +func TestPruneListFailureIsFatal(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "a.js", body: "x"}}) + fc := newFakeAPI() + fc.listErr = errors.New("boom") + captureLogs(t) + + if err := runPrune(arc, fc, days(30), time.Now()); err == nil { + t.Fatal("expected error") + } + if len(fc.deletes) != 0 { + t.Fatalf("expected no deletes, got %v", fc.deletes) + } +} + +func TestPruneDrainAndReport(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "keep.js", body: "x"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "keep.js", LastChanged: old}, + {Path: "a.js", LastChanged: old}, + {Path: "b.js", LastChanged: old}, + {Path: "c.js", LastChanged: old}, + } + fc.failErr = errors.New("boom") + fc.failWhen = func(path string) bool { return path == "b.js" } + _, errb := captureLogs(t) + + err := runPrune(arc, fc, days(30), now) + var exit exitError + if !errors.As(err, &exit) { + t.Fatalf("expected exitError (non-zero exit) on delete failure, got %v", err) + } + // One failed delete must not stop the others: all three orphans are attempted. + if got := sortedDeletes(fc); strings.Join(got, ",") != "a.js,b.js,c.js" { + t.Fatalf("all candidates must be attempted, got %v", got) + } + if !strings.Contains(errb.String(), "b.js") { + t.Fatalf("failure report must name b.js, got %v", errb.String()) + } +} + +// TestPruneRemovesEmptyDirsBottomUp covers §10 step 5: a directory whose only +// file is deleted is removed, and its now-empty parent is removed too. +func TestPruneRemovesEmptyDirsBottomUp(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "index.html", body: "<html>"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "index.html", LastChanged: old}, + {Path: "a", IsDirectory: true}, + } + fc.lists["a"] = []bunny.Object{{Path: "a/b", IsDirectory: true}} + fc.lists["a/b"] = []bunny.Object{{Path: "a/b/gone.js", Size: 3, LastChanged: old}} + captureLogs(t) + + if err := runPrune(arc, fc, days(30), now); err != nil { + t.Fatalf("prune: %v", err) + } + + got := sortedDeletes(fc) + want := []string{"a", "a/b/gone.js"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("deletes = %v, want %v", got, want) + } +} + +// TestPruneKeepsDirWithFailedChild covers §10 step 5: a directory whose file +// delete FAILED must not be removed, because the node survives. +func TestPruneKeepsDirWithFailedChild(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "index.html", body: "<html>"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "index.html", LastChanged: old}, + {Path: "a", IsDirectory: true}, + } + fc.lists["a"] = []bunny.Object{{Path: "a/gone.js", Size: 3, LastChanged: old}} + fc.failWhen = func(path string) bool { return path == "a/gone.js" } + fc.failErr = errors.New("boom") + captureLogs(t) + + if err := runPrune(arc, fc, days(30), now); err == nil { + t.Fatal("expected non-zero exit on file-delete failure") + } + for _, d := range fc.deletes { + if d == "a" { + t.Fatalf("dir with failed child must not be removed, got %v", fc.deletes) + } + } +} + +// TestPruneEmptyDirFailureNonFatal covers §10 step 6: an empty-dir removal +// failure is warned and counted but never flips the exit code. +func TestPruneEmptyDirFailureNonFatal(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "index.html", body: "<html>"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "index.html", LastChanged: old}, + {Path: "a", IsDirectory: true}, + } + fc.lists["a"] = []bunny.Object{{Path: "a/gone.js", Size: 3, LastChanged: old}} + fc.failWhen = func(path string) bool { return path == "a" } + fc.failErr = errors.New("boom") + _, errb := captureLogs(t) + + if err := runPrune(arc, fc, days(30), now); err != nil { + t.Fatalf("empty-dir failure must be non-fatal, got %v", err) + } + if !strings.Contains(errb.String(), "Error:") { + t.Fatalf("expected error on empty-dir failure, got %q", errb.String()) + } +} + +func TestPruneDryRunDeletesNothing(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "keep.js", body: "x"}}) + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + old := now.AddDate(0, 0, -100) + + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "keep.js", LastChanged: old}, + {Path: "stale.js", LastChanged: old}, + } + dryRun = true + + if err := runPrune(arc, dryRunStorageAPI{fc}, days(30), now); err != nil { + t.Fatalf("prune: %v", err) + } + if len(fc.deletes) != 0 { + t.Fatalf("dry-run must delete nothing, got %v", fc.deletes) + } +} diff --git a/internal/cli/cmd_push.go b/internal/cli/cmd_push.go new file mode 100644 index 0000000..c58f75c --- /dev/null +++ b/internal/cli/cmd_push.go @@ -0,0 +1,266 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/spf13/pflag" + + "tsne.dev/hopper/internal/bunny" + "tsne.dev/hopper/internal/log" + "tsne.dev/hopper/internal/source" +) + +// pruneHintFloor and pruneHintMultiple gate the post-push prune hint: the zone +// must hold at least floor files and more than multiple times the current +// release before stale assets are worth flagging. multiple covers the current +// release plus five retained rollbacks (six releases' worth of hashed assets). +const ( + pruneHintFloor = 1000 + pruneHintMultiple = 6 +) + +var _ command = (*pushCommand)(nil) + +type pushCommand struct { + flags *pflag.FlagSet + main *mainCommand + + concurrency uint + prefix string + root string +} + +func newPushCommand(main *mainCommand) *pushCommand { + cmd := &pushCommand{ + flags: newFlagset("push"), + main: main, + } + cmd.flags.UintVar(&cmd.concurrency, "concurrency", 0, "number of concurrent transfers (0 = auto-detect based on CPUs)") + registerPathPrefixFlag(cmd.flags, &cmd.prefix, "prefix", "", "path prefix within the zone to target") + registerPathPrefixFlag(cmd.flags, &cmd.root, "root", "", "subtree of the source to upload") + registerDryRunFlag(cmd.flags) + return cmd +} + +func (cmd *pushCommand) Usage() string { + return "Usage:" + + "\n hopper [global flags] push [flags] ⟨source⟩" + + "\n" + + "\nFlags:" + + "\n" + cmd.flags.FlagUsages() +} + +func (cmd *pushCommand) Run(ctx context.Context, args []string) error { + parseFlags(cmd, cmd.flags, args) + + positionals := cmd.flags.Args() + if len(positionals) == 0 || positionals[0] == "" { + return usageError{"missing source"} + } + if len(positionals) > 1 { + return usageError{"only a single source is supported"} + } + source := positionals[0] + + if err := cmd.main.lookupAccessKey(); err != nil { + return err + } + + api := newStorageAPI(cmd.main.endpoint, cmd.main.zone, cmd.main.accessKey) + return cmd.push(ctx, api, source) +} + +func (cmd *pushCommand) push(ctx context.Context, api storageAPI, sourceArg string) error { + src, err := source.NewTarGzSource(sourceArg, cmd.root) + if err != nil { + return err + } + defer src.Close() + + // Drain the complete remote listing before any writes. A List failure is + // fatal here, so push never proceeds on a partial picture. + // Index the remote listing by storage key so each source entry's skip / + // conflict decision is an O(1) lookup rather than a linear scan per file + // (the walk itself is O(remote)). A file and a directory can never share an + // exact key in object storage, so keying by path is unambiguous. + start := time.Now() + + remoteByPath := make(map[string]bunny.Object, 128) + remoteFiles := 0 + it := api.List(ctx, cmd.prefix) + for it.Next() { + o := it.Object() + if !o.IsDirectory { + remoteFiles++ + } + remoteByPath[o.Path] = o + } + if err := it.Err(); err != nil { + return err + } + + skipped := 0 + + // stashedMutables holds mutable entries (e.g. *.html files) read during the + // asset phase but not uploaded until all immutable assets are successfully + // uploaded. The stash is in-memory. A release normally has only a handful of + // mutable files only. + stashedMutables := make([]mutableEntry, 0, 8) + + // Upload every immutable asset via the pool and stash mutables. The single + // reader below buffers each entry's bytes and hands them to a fail-fast worker + // pool. The first asset that fails to upload cancels the pool. + assetsPool := newWorkerPool(ctx, cmd.concurrency, true) + var readErr error + for !assetsPool.cancelled() { + entry, err := src.Next() + if err != nil { + if !errors.Is(err, io.EOF) { + readErr = err + } + break + } + + // Decide against the remote listing: a directory at this exact path is a + // hard conflict; a matching-size file is skipped; any other case (absent, + // size mismatch, 0/missing) falls through to re-upload. + key := prefixedPath(cmd.prefix, entry.Path) + remoteObj, remoteExists := remoteByPath[key] + if remoteExists && remoteObj.IsDirectory { + readErr = fmt.Errorf("remote path %q is a directory, release has a file there", key) + break + } + + if entry.IsMutable { + data, err := entry.ReadContent(ctx) + if err != nil { + readErr = fmt.Errorf("read %q: %w", entry.Path, err) + break + } + stashedMutables = append(stashedMutables, mutableEntry{path: key, data: data}) + continue + } + + if remoteExists && remoteObj.Size == entry.Size { + logAction("skipping %s (exists)", key) + skipped++ + continue + } + + data, err := entry.ReadContent(ctx) + if err != nil { + readErr = fmt.Errorf("read %q: %w", entry.Path, err) + break + } + + action := fmt.Sprintf("uploading %s (%s)", key, humanSize(len(data))) + if !assetsPool.submit(action, uploadTask(api, key, data)) { + break + } + } + + // A reader error is fatal and abandons all in-flight uploads. + if readErr != nil { + assetsPool.cancel(errors.New("upload cancelled")) + } + + assetsUploaded, errs := assetsPool.wait() + reported := reportFailures(ctx, "failed to upload files", errs) + if readErr != nil { + return readErr + } else if reported { + return exitError{1} + } + + // Upload all stashed mutable files. This batch is neither atomic nor ordered. + // A partial mutable-phase failure can leave an entry point live while a mutable + // file it references is not yet uploaded. Mitigation: We report an error and + // a re-run is idempotent. + mutablesPool := newWorkerPool(ctx, cmd.concurrency, false) + for _, m := range stashedMutables { + action := fmt.Sprintf("uploading %s (%s)", m.path, humanSize(len(m.data))) + if !mutablesPool.submit(action, uploadTask(api, m.path, m.data)) { + break // the parent context was cancelled + } + } + + // Commit phase (HTML) is drain-and-report: every stashed mutable is attempted + // and all failures are aggregated. Unlike the pre-commit asset phase, uploads + // here genuinely happened, so we print a summary with counts even on partial + // failure — but the failure list goes out first, keeping the summary last. + mutablesUploaded, mutErrs := mutablesPool.wait() + mutablesFailed := len(mutErrs) + + sourceFiles := assetsUploaded + mutablesUploaded + skipped + mutablesFailed + uploaded := assetsUploaded + mutablesUploaded + + if sourceFiles == 0 { + log.Warn("source contains no files to upload (pushed nothing). check --root, --prefix, or the archive contents") + } + + var cmdErr error + if reportFailures(ctx, "failed to upload files", mutErrs) { + cmdErr = exitError{1} + } + + summary := "uploaded: %d" + + "\nskipped: %d" + + "\nremote: %d" + + "\nfailed: %d" + + "\nelapsed: %s" + if shouldHintPrune(sourceFiles, remoteFiles) { + summary += "\n\nHint: There are many stale files on the zone. Consider running hopper prune." + } + logSummary( + summary, + uploaded, skipped, remoteFiles, mutablesFailed, time.Since(start).Round(time.Millisecond), + ) + + return cmdErr +} + +func shouldHintPrune(sourceFiles, remoteFiles int) bool { + return sourceFiles > 0 && + remoteFiles >= pruneHintFloor && + remoteFiles > pruneHintMultiple*sourceFiles +} + +// uploadTask builds a pool task that PUTs data and, on failure, tags the error +// with the path so the report can name the file. +func uploadTask(api storageAPI, path string, data []byte) func(context.Context) error { + return func(ctx context.Context) error { + if err := api.Upload(ctx, path, data); err != nil { + return fileError{path: path, err: err} + } + return nil + } +} + +// mutableEntry is a mutable file (current rule: *.html) read during the asset +// phase and held in memory until the commit point. data is owned by this +// struct. +type mutableEntry struct { + path string + data []byte +} + +func humanSize(n int) string { + return humanSize64(int64(n)) +} + +func humanSize64(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for x := n / unit; x >= unit; x /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} diff --git a/internal/cli/cmd_push_test.go b/internal/cli/cmd_push_test.go new file mode 100644 index 0000000..9a199b2 --- /dev/null +++ b/internal/cli/cmd_push_test.go @@ -0,0 +1,633 @@ +package cli + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "strings" + "testing" + + "tsne.dev/hopper/internal/bunny" + "tsne.dev/hopper/internal/log" +) + +// runPush drives the push flow against a fake storage API for the given archive, +// using the default concurrency. +func runPush(source string, api storageAPI) error { + return runPushN(source, api, 8) +} + +func runPushN(source string, api storageAPI, concurrency uint) error { + cmd := &pushCommand{concurrency: concurrency} + return cmd.push(context.Background(), api, source) +} + +func TestPushAppliesPrefix(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/app.abc.js", body: "console.log(1)"}, + {name: "assets/skip.js", body: "same"}, + }) + fc := newFakeAPI() + // The remote walk starts under the prefix and returns already-prefixed keys. + fc.lists["foo"] = []bunny.Object{{Path: "foo/assets", IsDirectory: true}} + fc.lists["foo/assets"] = []bunny.Object{ + {Path: "foo/assets/skip.js", Size: int(len("same"))}, + } + out, _ := captureLogs(t) + + cmd := &pushCommand{concurrency: 8, prefix: "foo"} + if err := cmd.push(context.Background(), fc, arc); err != nil { + t.Fatalf("push: %v", err) + } + + got := fc.paths() + want := []string{"foo/assets/app.abc.js", "foo/index.html"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("paths = %v, want %v", got, want) + } + if !strings.Contains(out.String(), "skipping foo/assets/skip.js (exists)") { + t.Fatalf("skip-check must compare prefixed key: %q", out.String()) + } +} + +func TestPushSelectsRootAndStrips(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "README.md", body: "ignored"}, + {name: "app/index.html", body: "<html>"}, + {name: "app/assets/x.js", body: "x"}, + {name: "other/y.js", body: "ignored"}, + }) + fc := newFakeAPI() + captureLogs(t) + + cmd := &pushCommand{concurrency: 8, root: "app"} + if err := cmd.push(context.Background(), fc, arc); err != nil { + t.Fatalf("push: %v", err) + } + + got := fc.paths() + want := []string{"assets/x.js", "index.html"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func TestPushSelectsNestedRoot(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "dist/public/index.html", body: "<html>"}, + {name: "dist/public/assets/x.js", body: "x"}, + {name: "dist/private/secret", body: "ignored"}, + }) + fc := newFakeAPI() + captureLogs(t) + + cmd := &pushCommand{concurrency: 8, root: "dist/public"} + if err := cmd.push(context.Background(), fc, arc); err != nil { + t.Fatalf("push: %v", err) + } + + got := fc.paths() + want := []string{"assets/x.js", "index.html"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func TestPushRootWithPrefix(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "app/assets/x.js", body: "x"}, + }) + fc := newFakeAPI() + captureLogs(t) + + cmd := &pushCommand{concurrency: 8, root: "app", prefix: "foo"} + if err := cmd.push(context.Background(), fc, arc); err != nil { + t.Fatalf("push: %v", err) + } + + got := fc.paths() + want := []string{"foo/assets/x.js"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func TestPushRootMatchesNothing(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "app/index.html", body: "<html>"}, + }) + fc := newFakeAPI() + out, _ := captureLogs(t) + + cmd := &pushCommand{concurrency: 8, root: "nope"} + if err := cmd.push(context.Background(), fc, arc); err != nil { + t.Fatalf("push: %v", err) + } + + if got := fc.paths(); len(got) != 0 { + t.Fatalf("paths = %v, want none", got) + } + if !strings.Contains(out.String(), "uploaded: 0") { + t.Fatalf("summary missing: %q", out.String()) + } +} + +func TestPushUploadsRegularFiles(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/app.abc.js", body: "console.log(1)"}, + {name: "dir/", typeflag: tar.TypeDir}, + }) + fc := newFakeAPI() + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + + got := fc.paths() + want := []string{"assets/app.abc.js", "index.html"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("paths = %v, want %v", got, want) + } + if string(fc.puts["index.html"]) != "<html>" { + t.Fatalf("bad body for index.html: %q", fc.puts["index.html"]) + } + if string(fc.puts["assets/app.abc.js"]) != "console.log(1)" { + t.Fatalf("bad body for app.abc.js: %q", fc.puts["assets/app.abc.js"]) + } + if !strings.Contains(out.String(), "uploaded: 2") { + t.Fatalf("summary missing: %q", out.String()) + } +} + +func TestPushUploadsHTMLAfterAllAssets(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "a.js", body: "x"}, + {name: "nested/page.HTML", body: "<p>"}, // case-insensitive .html + {name: "b.css", body: "y"}, + }) + fc := newFakeAPI() + captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + + // Every HTML Put must happen after the last asset Put. + lastAsset := -1 + firstHTML := len(fc.order) + for i, p := range fc.order { + if strings.HasSuffix(strings.ToLower(p), ".html") { + if i < firstHTML { + firstHTML = i + } + } else if i > lastAsset { + lastAsset = i + } + } + if firstHTML <= lastAsset { + t.Fatalf("HTML uploaded before an asset; order = %v", fc.order) + } +} + +func TestPushDoesNotUploadHTMLWhenAssetFails(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "a.js", body: "x"}, + }) + fc := newFakeAPI() + fc.failOn = "a.js" + captureLogs(t) + + if err := runPush(arc, fc); err == nil { + t.Fatal("expected error") + } + for _, p := range fc.order { + if strings.HasSuffix(strings.ToLower(p), ".html") { + t.Fatalf("HTML must not be uploaded when an asset fails; order = %v", fc.order) + } + } +} + +func TestPushSkipsSymlinkWithWarning(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "a.js", body: "x"}, + {name: "link", typeflag: tar.TypeSymlink, linkname: "a.js"}, + }) + fc := newFakeAPI() + _, errb := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + if got := fc.paths(); len(got) != 1 || got[0] != "a.js" { + t.Fatalf("paths = %v", got) + } + if !strings.Contains(errb.String(), "Warning") { + t.Fatalf("expected warning on stderr, got %q", errb.String()) + } +} + +func TestPushFailsOnUploadError(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "a.js", body: "x"}, + {name: "b.js", body: "y"}, + }) + fc := newFakeAPI() + fc.failOn = "a.js" + captureLogs(t) + + if err := runPush(arc, fc); err == nil { + t.Fatal("expected error") + } +} + +func TestPushSkipsUnchanged(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/app.abc.js", body: "console.log(1)"}, + }) + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "index.html", Size: int(len("<html>"))}, + {Path: "assets", IsDirectory: true}, + } + fc.lists["assets"] = []bunny.Object{ + {Path: "assets/app.abc.js", Size: int(len("console.log(1)"))}, + } + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + // The asset is skipped (matching size); the *.html is the commit point and is + // always uploaded even when an identically-sized copy exists remotely. + if got := fc.paths(); len(got) != 1 || got[0] != "index.html" { + t.Fatalf("expected only index.html uploaded, got %v", got) + } + s := out.String() + if !strings.Contains(s, "skipping assets/app.abc.js (exists)") { + t.Fatalf("missing skip line: %q", s) + } + if strings.Contains(s, "skipping index.html") { + t.Fatalf("index.html must never be skipped: %q", s) + } + if !strings.Contains(s, "uploaded: 1") || !strings.Contains(s, "skipped: 1") { + t.Fatalf("summary missing: %q", s) + } +} + +func TestPushReuploadsOnSizeMismatch(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "a.js", body: "hello"}, // 5 bytes + {name: "b.js", body: "world"}, // present, matching size -> skip + }) + fc := newFakeAPI() + fc.lists[""] = []bunny.Object{ + {Path: "a.js", Size: 1}, // size mismatch -> upload + {Path: "b.js", Size: int(len("world"))}, + } + captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + if got := fc.paths(); len(got) != 1 || got[0] != "a.js" { + t.Fatalf("paths = %v, want [a.js]", got) + } +} + +func TestPushErrorsOnDirectoryFileCollision(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "data", body: ""}, // 0-byte asset colliding with a remote dir + }) + fc := newFakeAPI() + // Remote has a directory named "data"; a file at the same path is a hard + // conflict (and a 0-size dir must never silently shadow a 0-byte file). + fc.lists[""] = []bunny.Object{{Path: "data", IsDirectory: true}} + captureLogs(t) + + if err := runPush(arc, fc); err == nil { + t.Fatal("expected error on directory/file collision") + } + if got := fc.paths(); len(got) != 0 { + t.Fatalf("expected no uploads, got %v", got) + } +} + +func TestPushAbortsOnListFailure(t *testing.T) { + arc := makeArchive(t, []tarEntry{{name: "a.js", body: "x"}}) + fc := newFakeAPI() + fc.listErr = errors.New("boom") + captureLogs(t) + + if err := runPush(arc, fc); err == nil { + t.Fatal("expected error") + } + if got := fc.paths(); len(got) != 0 { + t.Fatalf("expected no uploads before abort, got %v", got) + } +} + +func TestPushRunRejectsBadSourceArgs(t *testing.T) { + captureLogs(t) + cmd := newPushCommand(&mainCommand{}) + + var ue usageError + if err := cmd.Run(context.Background(), nil); !errors.As(err, &ue) { + t.Fatalf("missing source: expected usageError, got %v", err) + } + if err := cmd.Run(context.Background(), []string{"a.tgz", "b.tgz"}); !errors.As(err, &ue) { + t.Fatalf("multiple sources: expected usageError, got %v", err) + } +} + +func TestPushRejectsNegativeConcurrencyAtParse(t *testing.T) { + captureLogs(t) + // A negative value is not a valid uint, so pflag rejects it at parse time. + cmd := newPushCommand(&mainCommand{}) + if err := cmd.flags.Parse([]string{"--concurrency", "-1", "a.tgz"}); err == nil { + t.Fatal("--concurrency -1: expected a parse error, got nil") + } +} + +func TestPushRejectsEscapingPrefix(t *testing.T) { + captureLogs(t) + cmd := newPushCommand(&mainCommand{}) + var ue usageError + if err := cmd.flags.Parse([]string{"--prefix", "../x", "a.tgz"}); !errors.As(err, &ue) { + t.Fatalf("--prefix ../x: expected usageError, got %v", err) + } +} + +func TestPushBoundsInFlightUploads(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "a.js", body: "1"}, + {name: "b.js", body: "2"}, + {name: "c.js", body: "3"}, + {name: "d.js", body: "4"}, + {name: "e.js", body: "5"}, + }) + const limit = 2 + gate := &gatingAPI{fakeAPI: newFakeAPI(), release: make(chan struct{})} + captureLogs(t) + + done := make(chan error, 1) + go func() { done <- runPushN(arc, gate, limit) }() + + // The reader blocks on the semaphore once the pool is saturated, so no more + // uploads can begin until one drains; release them so the run finishes. + for i := 0; i < 5; i++ { + gate.release <- struct{}{} + } + if err := <-done; err != nil { + t.Fatalf("push: %v", err) + } + if peak := gate.peak(); peak > limit { + t.Fatalf("peak in-flight = %d, want <= %d", peak, limit) + } +} + +func TestPushAssetFailureReportsOneFileAndSkipsHTML(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "a.js", body: "1"}, + {name: "b.js", body: "2"}, + }) + fc := newFakeAPI() + // Every asset fails: fail-fast cancels the pool after the first genuine + // failure. All genuine failures that landed before cancellation are + // aggregated (§8.3); cancellation collateral (context.Canceled) is filtered + // out (see TestPushFailFastFiltersCancellationCollateral). HTML is never + // written. + fc.failAll = true + captureLogs(t) + + err := runPushN(arc, fc, 8) + var ee exitError + if !errors.As(err, &ee) { + t.Fatalf("expected exitError, got %T", err) + } + for _, p := range fc.order { + if strings.HasSuffix(strings.ToLower(p), ".html") { + t.Fatalf("HTML must not be written when an asset fails; order = %v", fc.order) + } + } +} + +// TestPushFailFastFiltersCancellationCollateral verifies that the fail-fast +// report names ONLY the genuine upload failure, not the in-flight uploads that +// were abandoned by the cancellation. A context.Canceled is a consequence of the +// stop decision, not an independent file failure, and is swallowed uniformly so +// push (fail-fast) and a user SIGINT behave identically (§8.3). The collateral +// errors WRAP context.Canceled. +func TestPushFailFastFiltersCancellationCollateral(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "a.js", body: "1"}, + {name: "b.js", body: "2"}, + {name: "c.js", body: "3"}, + {name: "index.html", body: "<html>"}, + }) + api := &ctxCancelAPI{ + fakeAPI: newFakeAPI(), + triggerPath: "a.js", + triggerErr: fileError{path: "a.js", err: errors.New("boom")}, + } + _, errb := captureLogs(t) + + // Concurrency covers all three assets so b.js/c.js are genuinely in-flight + // (blocked) when a.js triggers the cancellation. + err := runPushN(arc, api, 3) + var ee exitError + if !errors.As(err, &ee) { + t.Fatalf("expected exitError, got %T (%v)", err, err) + } + report := errb.String() + if !strings.Contains(report, "a.js") || !strings.Contains(report, "boom") { + t.Fatalf("report must name the genuine failure a.js: %q", report) + } + if strings.Contains(report, "b.js") || strings.Contains(report, "c.js") { + t.Fatalf("report must NOT name cancellation collateral b.js/c.js: %q", report) + } + for _, p := range api.order { + if strings.HasSuffix(strings.ToLower(p), ".html") { + t.Fatalf("HTML must not be written when an asset fails; order = %v", api.order) + } + } +} + +func TestPushHTMLFailuresAreAllReported(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "about.html", body: "<html>"}, + {name: "a.js", body: "1"}, + }) + fc := newFakeAPI() + // Assets succeed; both HTML commit uploads fail. The drain phase reports + // every failure under the friendly headline. + fc.failErr = errors.New("boom") + fc.failWhen = func(path string) bool { return strings.HasSuffix(path, ".html") } + out, errb := captureLogs(t) + + err := runPushN(arc, fc, 8) + var exit exitError + if !errors.As(err, &exit) { + t.Fatalf("expected exitError, got %v", err) + } + // The drain report lists every HTML failure on stderr... + if !strings.Contains(errb.String(), "index.html") || !strings.Contains(errb.String(), "about.html") { + t.Fatalf("drain report must list every HTML failure: %q", errb.String()) + } + // ...and the summary is still printed (last) on stdout. + if !strings.Contains(out.String(), "failed:") { + t.Fatalf("summary must be printed after a commit-phase failure: %q", out.String()) + } +} + +func TestShouldHintPrune(t *testing.T) { + cases := []struct { + name string + sourceFiles int + remoteFiles int + want bool + }{ + {"below floor", 1, 999, false}, + {"at floor within ratio", 1000, 6000, false}, + {"at exact multiple", 100, 600, false}, + {"one over multiple but below floor", 10, 61, false}, + {"one over multiple and at floor", 200, 1201, true}, + {"empty source", 0, 100000, false}, + } + for _, tc := range cases { + if got := shouldHintPrune(tc.sourceFiles, tc.remoteFiles); got != tc.want { + t.Errorf("%s: shouldHintPrune(%d, %d) = %v, want %v", + tc.name, tc.sourceFiles, tc.remoteFiles, got, tc.want) + } + } +} + +// stalePushSetup builds a one-file source (source_files == 1) and a fake API +// whose remote listing holds remoteFiles file objects and remoteDirs directory +// objects, so tests can drive the hint decision at chosen boundaries. +func stalePushSetup(t *testing.T, remoteFiles, remoteDirs int) (string, *fakeAPI) { + t.Helper() + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + }) + fc := newFakeAPI() + objs := make([]bunny.Object, 0, remoteFiles+remoteDirs) + for i := 0; i < remoteFiles; i++ { + objs = append(objs, bunny.Object{Path: fmt.Sprintf("stale/%d.js", i), Size: 1}) + } + for i := 0; i < remoteDirs; i++ { + objs = append(objs, bunny.Object{Path: fmt.Sprintf("dir%d", i), IsDirectory: true}) + } + fc.lists[""] = objs + return arc, fc +} + +func TestPushHintFiresWhenStale(t *testing.T) { + arc, fc := stalePushSetup(t, 1001, 0) + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + s := out.String() + if !strings.Contains(s, "remote: 1001") { + t.Fatalf("summary missing remote count: %q", s) + } + if !strings.Contains(s, "\n\nHint:") { + t.Fatalf("expected prune hint: %q", s) + } + // The hint rides on stdout with the summary, not the stderr warning path. + if strings.Contains(s, "\n\nHint:") && strings.Index(s, "remote:") > strings.Index(s, "\n\nHint:") { + t.Fatalf("hint must follow the summary: %q", s) + } +} + +func TestPushNoHintWithinRetentionWindow(t *testing.T) { + arc, fc := stalePushSetup(t, 6, 0) + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + if strings.Contains(out.String(), "hint:") { + t.Fatalf("unexpected hint: %q", out.String()) + } +} + +func TestPushNoHintBelowFloor(t *testing.T) { + arc, fc := stalePushSetup(t, 999, 0) + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + if strings.Contains(out.String(), "hint:") { + t.Fatalf("unexpected hint below floor: %q", out.String()) + } +} + +func TestPushHintIgnoresRemoteDirectories(t *testing.T) { + // Many directory nodes, few files: directories must not inflate remote_files. + arc, fc := stalePushSetup(t, 5, 2000) + out, _ := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + s := out.String() + if !strings.Contains(s, "remote: 5") { + t.Fatalf("directories must not count: %q", s) + } + if strings.Contains(s, "hint:") { + t.Fatalf("unexpected hint from directory nodes: %q", s) + } +} + +func TestPushHintFiresUnderQuiet(t *testing.T) { + arc, fc := stalePushSetup(t, 1001, 0) + out, _ := captureLogs(t) + log.SetQuiet(true) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + if !strings.Contains(out.String(), "\n\nHint:") { + t.Fatalf("hint must appear under --quiet: %q", out.String()) + } +} + +func TestPushHintFiresUnderDryRun(t *testing.T) { + arc, fc := stalePushSetup(t, 1001, 0) + out, _ := captureLogs(t) + dryRun = true + + if err := runPush(arc, dryRunStorageAPI{fc}); err != nil { + t.Fatalf("push: %v", err) + } + if !strings.Contains(out.String(), "\n\nHint:") { + t.Fatalf("hint must appear under dry-run: %q", out.String()) + } +} + +func TestHumanSize(t *testing.T) { + cases := map[int]string{ + 0: "0 B", + 512: "512 B", + 1024: "1.0 KiB", + 1536: "1.5 KiB", + 1024 * 1024: "1.0 MiB", + } + for in, want := range cases { + if got := humanSize(in); got != want { + t.Errorf("humanSize(%d) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/cli/dryrun.go b/internal/cli/dryrun.go new file mode 100644 index 0000000..ed65f48 --- /dev/null +++ b/internal/cli/dryrun.go @@ -0,0 +1,11 @@ +package cli + +import ( + "github.com/spf13/pflag" +) + +var dryRun = false + +func registerDryRunFlag(flags *pflag.FlagSet) { + flags.BoolVar(&dryRun, "dryrun", false, "run the command without executing any mutating operations") +} diff --git a/internal/cli/dryrun_test.go b/internal/cli/dryrun_test.go new file mode 100644 index 0000000..c2538d6 --- /dev/null +++ b/internal/cli/dryrun_test.go @@ -0,0 +1,78 @@ +package cli + +import ( + "context" + "strings" + "testing" + + "tsne.dev/hopper/internal/bunny" +) + +// TestDryRunPushStubsUploads asserts that under dry-run the push flow performs +// NO uploads against the wrapped API, yet prints the normal per-file action +// line and summary (logging is owned by the command, not the decorator). The +// recording fake is the wrapped (real-stand-in) API, so a zero put-count +// provably shows the write side was never invoked. +func TestDryRunPushStubsUploads(t *testing.T) { + arc := makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/app.abc.js", body: "console.log(1)"}, + }) + + spy := newFakeAPI() + out, _ := captureLogs(t) + dryRun = true + + if err := runPush(arc, dryRunStorageAPI{spy}); err != nil { + t.Fatalf("push: %v", err) + } + + if got := spy.paths(); len(got) != 0 { + t.Fatalf("expected no real uploads, got %v", got) + } + + s := out.String() + for _, want := range []string{ + "[dryrun] uploading index.html", + "[dryrun] uploading assets/app.abc.js", + "uploaded: 2", + } { + if !strings.Contains(s, want) { + t.Fatalf("output missing %q; got:\n%s", want, s) + } + } +} + +// TestDryRunDeleteStubbed asserts Delete never reaches the wrapped API +// (reused by prune in issue 008, which will own the delete log line). +func TestDryRunDeleteStubbed(t *testing.T) { + spy := newFakeAPI() + api := dryRunStorageAPI{spy} + + if err := api.Delete(context.Background(), "old/file.js"); err != nil { + t.Fatalf("Delete: %v", err) + } + if len(spy.deletes) != 0 { + t.Fatalf("expected no real deletes, got %v", spy.deletes) + } +} + +// TestDryRunListDelegates asserts the read side still reaches the wrapped API +// (real Bunny in production), unaffected by the write stubbing. +func TestDryRunListDelegates(t *testing.T) { + spy := newFakeAPI() + spy.lists[""] = []bunny.Object{{Path: "a.js", Size: 3}} + api := dryRunStorageAPI{spy} + + it := api.List(context.Background(), "") + var got []bunny.Object + for it.Next() { + got = append(got, it.Object()) + } + if err := it.Err(); err != nil { + t.Fatalf("List: %v", err) + } + if len(got) != 1 || got[0].Path != "a.js" { + t.Fatalf("List delegate = %v", got) + } +} diff --git a/internal/cli/duration.go b/internal/cli/duration.go new file mode 100644 index 0000000..ee40562 --- /dev/null +++ b/internal/cli/duration.go @@ -0,0 +1,136 @@ +package cli + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/spf13/pflag" +) + +// Per-component upper bounds. A cache-safety grace window is measured in days +// to a few years; anything past these is a typo or an attempt to overflow the +// cutoff arithmetic. They are chosen so cutoff = now.AddDate(-y,-m,-d) can never +// overflow the int year addition or the int64 range of time.Time (representable +// to ~year 292 billion), so the destructive cutoff can never silently wrap. +const ( + maxDurationYears = 100000 + maxDurationMonths = 100000 * 12 + maxDurationDays = 100000 * 366 +) + +var _ pflag.Value = (*duration)(nil) + +// duration is a calendar-aware grace window . It holds a tuple +// (years, months, days) rather than a time.Duration. +type duration struct { + years int + months int + days int +} + +func registerDurationFlag(flags *pflag.FlagSet, v *duration, name string, def string, usage string) { + if err := v.Set(def); err != nil { + panic(fmt.Sprintf("invalid default for --%s: %v", name, err)) + } + flags.Var(v, name, usage) +} + +// Type implements the `pflags.Value` interface. +func (d *duration) Type() string { + return "duration" +} + +// String implements the `pflags.Value` interface. +func (d *duration) String() string { + sb := &strings.Builder{} + if d.years != 0 { + fmt.Fprintf(sb, "%dy", d.years) + } + if d.months != 0 { + fmt.Fprintf(sb, "%dm", d.months) + } + if d.days != 0 { + fmt.Fprintf(sb, "%dd", d.days) + } + + if sb.Len() == 0 { + return "0d" + } + return sb.String() +} + +// Set parses a combinable value like "1y2m10d", "7d" or "30d". Each component is +// a non-negative integer followed by one of the units y, m (month), d, and each +// unit may appear at most once. An empty value or a value with no components is +// rejected. +// +// Set implements the `pflags.Value` interface. +func (d *duration) Set(raw string) error { + parsed := duration{} + seen := map[byte]bool{} + components := 0 + + for i := 0; i < len(raw); { + c := raw[i] + if c < '0' || c > '9' { + return usageError{fmt.Sprintf("invalid duration %q", raw)} + } + + n := 0 + for i < len(raw) && raw[i] >= '0' && raw[i] <= '9' { + digit := int(raw[i] - '0') + if n > (math.MaxInt-digit)/10 { + return usageError{fmt.Sprintf("invalid duration %q (out of range)", raw)} + } + n = n*10 + digit + i++ + } + if i >= len(raw) { + return usageError{fmt.Sprintf("invalid duration %q (number %d has no unit)", raw, n)} + } + + unit := raw[i] + if seen[unit] { + return usageError{fmt.Sprintf("invalid duration %q (unit %q repeated)", raw, string(unit))} + } + switch unit { + case 'y': + if n > maxDurationYears { + return usageError{fmt.Sprintf("invalid duration %q (years exceeds %d)", raw, maxDurationYears)} + } + parsed.years = n + case 'm': + if n > maxDurationMonths { + return usageError{fmt.Sprintf("invalid duration %q (months exceeds %d)", raw, maxDurationMonths)} + } + parsed.months = n + case 'd': + if n > maxDurationDays { + return usageError{fmt.Sprintf("invalid duration %q (days exceeds %d)", raw, maxDurationDays)} + } + parsed.days = n + default: + return usageError{fmt.Sprintf("invalid duration %q (unknown unit %q)", raw, string(unit))} + } + seen[unit] = true + components++ + i++ + } + + if components == 0 { + return usageError{fmt.Sprintf("invalid duration %q", raw)} + } + + *d = parsed + return nil +} + +func (d duration) cutoff(tm time.Time) time.Time { + return tm.AddDate(-d.years, -d.months, -d.days) +} + +func (d duration) isZero() bool { + return d.years == 0 && d.months == 0 && d.days == 0 +} diff --git a/internal/cli/duration_test.go b/internal/cli/duration_test.go new file mode 100644 index 0000000..60d446a --- /dev/null +++ b/internal/cli/duration_test.go @@ -0,0 +1,51 @@ +package cli + +import ( + "testing" + "time" +) + +func TestDurationValueParses(t *testing.T) { + cases := map[string]duration{ + "30d": {days: 30}, + "7d": {days: 7}, + "1y2m10d": {years: 1, months: 2, days: 10}, + "2m": {months: 2}, + "1y": {years: 1}, + "0d": {days: 0}, + } + for in, want := range cases { + var v duration + if err := v.Set(in); err != nil { + t.Fatalf("Set(%q): %v", in, err) + } + if v != want { + t.Fatalf("Set(%q) = %+v, want %+v", in, v, want) + } + } +} + +func TestDurationValueRejectsBadInput(t *testing.T) { + for _, in := range []string{"", "d", "5", "5x", "1h", "1m2m", "-3d", "y", "5 d", "99999999999999999999d"} { + var v duration + if err := v.Set(in); err == nil { + t.Fatalf("Set(%q): expected error", in) + } + } +} + +func TestDurationValueMonthIsCalendarMonth(t *testing.T) { + var v duration + if err := v.Set("1m"); err != nil { + t.Fatal(err) + } + now := time.Date(2024, time.March, 31, 0, 0, 0, 0, time.UTC) + // A calendar-aware month back from March 31 lands in February, not "31 days". + got := v.cutoff(now) + if got.Month() != time.February && got.Month() != time.March { + t.Fatalf("cutoff month = %v, want calendar-aware", got.Month()) + } + if got.Equal(now.AddDate(0, 0, -30)) { + t.Fatalf("cutoff must not use a fixed 30-day month") + } +} diff --git a/internal/cli/errors.go b/internal/cli/errors.go new file mode 100644 index 0000000..aeeede1 --- /dev/null +++ b/internal/cli/errors.go @@ -0,0 +1,89 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "strings" + + "tsne.dev/hopper/internal/log" +) + +// errorVerboser is implemented by error types that can contribute extra +// technical detail to failure reports under --verbose (e.g. *bunny.apiError +// exposing verb/path/status/cause). reportFailures prints it indented below the +// friendly message when verbose is enabled. +type errorVerboser interface { + ErrorVerbose() string +} + +// usageError marks a parse or validation failure that should print a short +// usage line and exit 2, distinct from a runtime failure (exit 1). +type usageError struct { + msg string +} + +func (e usageError) Error() string { return e.msg } + +// exitError signals that the command has already emitted all its user-facing +// output (e.g. a failure list followed by the summary) and the process should +// simply exit with the given code, printing nothing further. This keeps the +// summary as the last line. +type exitError struct { + code int +} + +func (e exitError) Error() string { return fmt.Sprintf("exit status %d", e.code) } + +// reportFailures prints a batch of per-file failures to stderr under a headline, +// using the same layout everywhere or the context's cancellation cause. +// It returns true if it reported an error, false otherwise. +func reportFailures(ctx context.Context, msg string, errs []error) bool { + if len(errs) > 0 { + sb := strings.Builder{} + sb.WriteString(msg) + for _, err := range errs { + sb.WriteString("\n ") + sb.WriteString(err.Error()) + if log.Verbose() { + var v errorVerboser + if errors.As(err, &v) { + sb.WriteString("\n ") + sb.WriteString(v.ErrorVerbose()) + } + } + } + log.Error(sb.String()) + return true + } else if ctxErr := context.Cause(ctx); ctxErr != nil { + log.Error(ctxErr.Error()) + return true + } else { + return false + } +} + +// fileError attaches the storage path to an operation failure so the report can +// name the offending file. It preserves the wrapped cause (e.g. the bunny API +// error) in the chain so both friendly rendering (Error) and verbose detail +// (ErrorVerbose) still work via errors.As. +type fileError struct { + path string + err error +} + +func (e fileError) Error() string { return e.path + ": " + e.err.Error() } +func (e fileError) Unwrap() error { return e.err } + +type cancellationError struct { + msg string +} + +// CancellationError returns an error that wraps `context.Canceled` but +// with a custom error message. +func CancellationError(msg string) error { + return cancellationError{msg} +} + +func (e cancellationError) Error() string { return e.msg } +func (e cancellationError) Unwrap() error { return context.Canceled } diff --git a/internal/cli/helpers_test.go b/internal/cli/helpers_test.go new file mode 100644 index 0000000..8aa533f --- /dev/null +++ b/internal/cli/helpers_test.go @@ -0,0 +1,231 @@ +package cli + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "testing" + + "tsne.dev/hopper/internal/bunny" + "tsne.dev/hopper/internal/log" +) + +// captureLogs redirects the global logger to fresh buffers for the duration of +// a test and restores the defaults (and the dryRun global) afterward. Returns +// the stdout and stderr buffers. +func captureLogs(t *testing.T) (*bytes.Buffer, *bytes.Buffer) { + t.Helper() + var out, errb bytes.Buffer + log.Init(&out, &errb) + t.Cleanup(func() { + log.Init(os.Stdout, os.Stderr) + log.SetVerbose(false) + log.SetQuiet(false) + dryRun = false + }) + return &out, &errb +} + +// fakeAPI is an in-memory recording storageAPI: the reusable test seam for all +// command tests. It records every attempted Upload/Delete and serves scripted +// List results. It satisfies storageAPI, so it can be substituted for the real +// *bunny.API anywhere (including wrapped by dryRunStorageAPI). +type fakeAPI struct { + mu sync.Mutex + puts map[string][]byte + order []string + deletes []string + lists map[string][]bunny.Object + listErr error + failOn string + failAll bool + failWhen func(path string) bool + failErr error +} + +func newFakeAPI() *fakeAPI { + return &fakeAPI{puts: map[string][]byte{}, lists: map[string][]bunny.Object{}} +} + +func (f *fakeAPI) Upload(ctx context.Context, path string, body []byte) error { + f.mu.Lock() + defer f.mu.Unlock() + f.order = append(f.order, path) + if f.failAll || path == f.failOn || (f.failWhen != nil && f.failWhen(path)) { + if f.failErr != nil { + return f.failErr + } + return errors.New("forced failure") + } + b := make([]byte, len(body)) + copy(b, body) + f.puts[path] = b + return nil +} + +func (f *fakeAPI) Delete(ctx context.Context, path string) error { + return f.recordDelete(path) +} + +// DeleteDir records the logical directory path (no trailing slash) so tests can +// assert on it uniformly with file deletes; the real trailing-slash quirk lives +// in the bunny package and is exercised by bunny's own tests. +func (f *fakeAPI) DeleteDir(ctx context.Context, path string) error { + return f.recordDelete(path) +} + +func (f *fakeAPI) recordDelete(path string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.deletes = append(f.deletes, path) + if f.failAll || path == f.failOn || (f.failWhen != nil && f.failWhen(path)) { + if f.failErr != nil { + return f.failErr + } + return errors.New("forced failure") + } + return nil +} + +func (f *fakeAPI) List(ctx context.Context, dir string) *bunny.Iterator { + return bunny.NewIterator(ctx, dir, f.listDir) +} + +func (f *fakeAPI) listDir(ctx context.Context, dir string) ([]bunny.Object, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.listErr != nil { + return nil, f.listErr + } + return f.lists[dir], nil +} + +func (f *fakeAPI) paths() []string { + f.mu.Lock() + defer f.mu.Unlock() + var ps []string + for p := range f.puts { + ps = append(ps, p) + } + sort.Strings(ps) + return ps +} + +// ctxCancelAPI simulates the real fail-fast scenario: the upload of triggerPath +// fails immediately with a genuine (non-context) error, while every other +// upload blocks until the pool cancels its context and then returns an error +// that WRAPS context.Canceled (mirroring a real *bunny.apiError built from an +// aborted in-flight request). It lets a test assert that cancellation collateral +// is filtered out of the failure report while the genuine trigger survives. +type ctxCancelAPI struct { + *fakeAPI + triggerPath string + triggerErr error +} + +func (c *ctxCancelAPI) Upload(ctx context.Context, path string, body []byte) error { + c.fakeAPI.mu.Lock() + c.fakeAPI.order = append(c.fakeAPI.order, path) + c.fakeAPI.mu.Unlock() + + if path == c.triggerPath { + return c.triggerErr + } + // Collateral: wait for the fail-fast cancellation, then report it as a + // context-wrapped error, exactly like a real aborted upload. + <-ctx.Done() + return fmt.Errorf("upload %s aborted: %w", path, ctx.Err()) +} + +// gatingAPI wraps fakeAPI to make each Upload block on a release channel while +// recording the peak number of concurrent in-flight uploads, so tests can +// assert the pool never exceeds --concurrency. +type gatingAPI struct { + *fakeAPI + release chan struct{} + + gmu sync.Mutex + cur int + maxSeen int +} + +func (g *gatingAPI) Upload(ctx context.Context, path string, body []byte) error { + g.gmu.Lock() + g.cur++ + if g.cur > g.maxSeen { + g.maxSeen = g.cur + } + g.gmu.Unlock() + + <-g.release + + g.gmu.Lock() + g.cur-- + g.gmu.Unlock() + return g.fakeAPI.Upload(ctx, path, body) +} + +func (g *gatingAPI) peak() int { + g.gmu.Lock() + defer g.gmu.Unlock() + return g.maxSeen +} + +// tarEntry describes a single file/header to write into a test archive. +type tarEntry struct { + name string + body string + typeflag byte + linkname string +} + +func makeArchive(t *testing.T, entries []tarEntry) string { + t.Helper() + dir := t.TempDir() + p := filepath.Join(dir, "a.tar.gz") + f, err := os.Create(p) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz := gzip.NewWriter(f) + tw := tar.NewWriter(gz) + for _, e := range entries { + tf := e.typeflag + if tf == 0 { + tf = tar.TypeReg + } + hdr := &tar.Header{ + Name: e.name, + Mode: 0o644, + Size: int64(len(e.body)), + Typeflag: tf, + Linkname: e.linkname, + } + if tf != tar.TypeReg { + hdr.Size = 0 + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if tf == tar.TypeReg { + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return p +} diff --git a/internal/cli/log.go b/internal/cli/log.go new file mode 100644 index 0000000..c837649 --- /dev/null +++ b/internal/cli/log.go @@ -0,0 +1,22 @@ +package cli + +import ( + "tsne.dev/hopper/internal/log" +) + +// logAction prints a per-action line. +// In dry-run mode the line is prefixed with "[dryrun] ". +func logAction(format string, args ...any) { + if dryRun { + format = "[dryrun] " + format + } + log.Print(format, args...) +} + +// logSummary prints the final summary line. +func logSummary(format string, args ...any) { + if dryRun { + format += "\n\nNote: This was a dry run. Nothing on the zone was modified." + } + log.PrintImportant(format, args...) +} diff --git a/internal/cli/path_prefix.go b/internal/cli/path_prefix.go new file mode 100644 index 0000000..355349e --- /dev/null +++ b/internal/cli/path_prefix.go @@ -0,0 +1,51 @@ +package cli + +import ( + "fmt" + "path" + "strings" + + "github.com/spf13/pflag" +) + +var _ pflag.Value = (*pathPrefixValue)(nil) + +type pathPrefixValue struct { + p *string + name string +} + +func registerPathPrefixFlag(flags *pflag.FlagSet, p *string, name string, val string, usage string) { + *p = val + flags.Var(&pathPrefixValue{p, name}, name, usage) +} + +func (v *pathPrefixValue) Type() string { return "string" } +func (v *pathPrefixValue) String() string { return *v.p } + +func (v *pathPrefixValue) Set(raw string) error { + trimmed := strings.Trim(raw, "/") + if trimmed == "" { + *v.p = "" + return nil + } + + cleaned := path.Clean(trimmed) + if cleaned == "." { + *v.p = "" + return nil + } + if cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return usageError{fmt.Sprintf("%s %q escapes", v.name, raw)} + } + + *v.p = cleaned + return nil +} + +func prefixedPath(prefix string, filePath string) string { + if prefix == "" { + return filePath + } + return path.Join(prefix, filePath) +} diff --git a/internal/cli/path_prefix_test.go b/internal/cli/path_prefix_test.go new file mode 100644 index 0000000..bdb4e42 --- /dev/null +++ b/internal/cli/path_prefix_test.go @@ -0,0 +1,52 @@ +package cli + +import ( + "errors" + "testing" +) + +func TestPathPrefixValueSet(t *testing.T) { + cases := map[string]string{ + "": "", + "/": "", + "foo": "foo", + "/foo": "foo", + "foo/": "foo", + "/foo/": "foo", + "foo//bar": "foo/bar", + "foo/./bar": "foo/bar", + "foo/bar/..": "foo", + "a/b/../c": "a/c", + } + for in, want := range cases { + var p string + v := &pathPrefixValue{p: &p} + if err := v.Set(in); err != nil { + t.Errorf("Set(%q): unexpected error %v", in, err) + continue + } + if p != want { + t.Errorf("Set(%q) = %q, want %q", in, p, want) + } + } +} + +func TestPathPrefixValueSetEscapes(t *testing.T) { + for _, in := range []string{"..", "../x", "/../x", "foo/../../x"} { + var p string + v := &pathPrefixValue{p: &p} + var ue usageError + if err := v.Set(in); !errors.As(err, &ue) { + t.Errorf("Set(%q): expected usageError, got %v", in, err) + } + } +} + +func TestPrefixedPath(t *testing.T) { + if got := prefixedPath("", "assets/x.js"); got != "assets/x.js" { + t.Errorf("empty prefix: got %q", got) + } + if got := prefixedPath("foo", "assets/x.js"); got != "foo/assets/x.js" { + t.Errorf("prefixed: got %q", got) + } +} diff --git a/internal/cli/remote.go b/internal/cli/remote.go new file mode 100644 index 0000000..43fe93e --- /dev/null +++ b/internal/cli/remote.go @@ -0,0 +1,94 @@ +package cli + +import ( + "context" + "sort" + "strings" + "time" +) + +type remoteEntry struct { + path string + size int + lastChanged time.Time + isDir bool + deleted bool + + // orderKey is path with '/' replaced by 0x00, precomputed so the sort + // compares native strings. '/' and 0x00 are the only two bytes that can + // never appear in a path, and 0x00 sorts below every other byte, so this + // orders a directory's descendants ("a/...") as a contiguous run immediately + // after the directory ("a") and before any sibling that shares its base name + // ("a.js", "ab"). emptyDirs relies on that contiguity. + orderKey string +} + +// listRemoteEntries fetches all remote entries from the api. The returned list +// is sorted by orderKey (see remoteEntry.orderKey), which is the ordering +// emptyDirs requires. Callers must not re-sort it by path. +func listRemoteEntries(ctx context.Context, api storageAPI, prefix string) ([]remoteEntry, error) { + entries := make([]remoteEntry, 0, 64) + it := api.List(ctx, prefix) + for it.Next() { + o := it.Object() + entries = append(entries, remoteEntry{ + path: o.Path, + size: o.Size, + isDir: o.IsDirectory, + lastChanged: o.LastChanged, + orderKey: pathOrderKey(o.Path), + }) + } + if err := it.Err(); err != nil { + return nil, err + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].orderKey < entries[j].orderKey + }) + return entries, nil +} + +// emptyDirs returns the directory entries whose subtree holds no surviving +// file. entries must be sorted by orderKey (see remoteEntry.orderKey), so a +// directory's subtree is the contiguous run of following entries sharing its +// path prefix. A directory exists only to hold files, so it survives iff some +// file beneath it survives. This makes the bottom-up cascade (a parent emptied +// by removing its last child directory) fall out without extra bookkeeping. +// A file survives unless it was deleted. +func emptyDirs(entries []remoteEntry) []*remoteEntry { + n := len(entries) + empty := make([]*remoteEntry, 0, n/4) + for i := 0; i < n; { + dir := &entries[i] + i++ + if !dir.isDir { + continue + } + + existingFiles := 0 + c := i + for prefix := dir.path + "/"; c < n; c++ { + child := &entries[c] + if !strings.HasPrefix(child.path, prefix) { + break + } + if !child.isDir && !child.deleted { + existingFiles++ + } + } + + if existingFiles == 0 { + // Emit only the top of the empty subtree and skip its descendants. + // Deleting a Bunny directory removes the subtree recursively, so + // deleting `dir` also removes any empty sub-dirs beneath it. + empty = append(empty, dir) + i = c + } + } + + return empty +} + +func pathOrderKey(path string) string { + return strings.ReplaceAll(path, "/", "\x00") +} diff --git a/internal/cli/remote_test.go b/internal/cli/remote_test.go new file mode 100644 index 0000000..b4e20d9 --- /dev/null +++ b/internal/cli/remote_test.go @@ -0,0 +1,141 @@ +package cli + +import ( + "sort" + "strings" + "testing" +) + +func dir(path string) remoteEntry { + return remoteEntry{path: path, isDir: true, orderKey: pathOrderKey(path)} +} +func file(path string) remoteEntry { + return remoteEntry{path: path, orderKey: pathOrderKey(path)} +} + +func deleted(e remoteEntry) remoteEntry { + e.deleted = true + return e +} + +func emptyDirPaths(entries []remoteEntry) []string { + // Reproduce the production ordering (see listRemoteEntries) so the sort rule + // and the emptyDirs scan rule are always exercised together, rather than + // trusting hand-ordered literals. + sort.Slice(entries, func(i, j int) bool { + return entries[i].orderKey < entries[j].orderKey + }) + got := emptyDirs(entries) + paths := make([]string, len(got)) + for i, e := range got { + paths[i] = e.path + } + sort.Strings(paths) + return paths +} + +func TestEmptyDirs(t *testing.T) { + tests := []struct { + name string + entries []remoteEntry + want []string + }{ + { + name: "dir with only a deleted file is empty", + entries: []remoteEntry{ + dir("a"), + deleted(file("a/gone.js")), + }, + want: []string{"a"}, + }, + { + name: "dir with a surviving file is kept", + entries: []remoteEntry{ + dir("a"), + file("a/live.js"), + }, + want: []string{}, + }, + { + name: "bottom-up cascade removes parent and child", + entries: []remoteEntry{ + dir("a"), + dir("a/b"), + deleted(file("a/b/gone.js")), + }, + want: []string{"a"}, + }, + { + name: "deep file deleted but sibling file alive keeps parent", + entries: []remoteEntry{ + dir("a"), + dir("a/b"), + deleted(file("a/b/gone.js")), + file("a/c.js"), + }, + want: []string{"a/b"}, + }, + { + name: "sibling with shared name prefix is not swallowed", + entries: []remoteEntry{ + dir("a"), + deleted(file("a/gone.js")), + file("ab/live.js"), + }, + want: []string{"a"}, + }, + { + name: "sibling colliding on base name does not hide live children", + entries: []remoteEntry{ + dir("a"), + file("a.js"), + file("a/live.js"), + }, + want: []string{}, + }, + { + name: "empty dir with no children is removed", + entries: []remoteEntry{ + dir("a"), + }, + want: []string{"a"}, + }, + { + name: "nested dirs with no files at all are all removed", + entries: []remoteEntry{ + dir("a"), + dir("a/b"), + dir("a/c"), + }, + want: []string{"a"}, + }, + { + name: "empty dir nested under a dir holding a live file keeps outer, removes inner", + entries: []remoteEntry{ + dir("a"), + dir("a/b"), + dir("a/b/c"), + file("a/live.js"), + }, + want: []string{"a/b"}, + }, + { + name: "live file under a nested dir keeps the whole chain", + entries: []remoteEntry{ + dir("a"), + dir("a/b"), + file("a/b/live.js"), + }, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := emptyDirPaths(tt.entries) + if strings.Join(got, ",") != strings.Join(tt.want, ",") { + t.Fatalf("emptyDirs = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/cli/storage.go b/internal/cli/storage.go new file mode 100644 index 0000000..654ed85 --- /dev/null +++ b/internal/cli/storage.go @@ -0,0 +1,46 @@ +package cli + +import ( + "context" + + "tsne.dev/hopper/internal/bunny" +) + +// storageAPI is the full storage surface the dry-run decorator delegates to. +// Its read side (List) must stay real even under --dry-run; its +// write side (Upload/Delete) is what the decorator stubs. The real +// *bunny.Client satisfies this, as does the recording fake used in tests. +type storageAPI interface { + Upload(ctx context.Context, path string, body []byte) error + List(ctx context.Context, dir string) *bunny.Iterator + Delete(ctx context.Context, path string) error + DeleteDir(ctx context.Context, path string) error +} + +func newStorageAPI(endpoint, zone, accessKey string) storageAPI { + var api storageAPI = bunny.NewAPI(endpoint, zone, accessKey) + if dryRun { + api = dryRunStorageAPI{api} + } + return api +} + +type dryRunStorageAPI struct { + upstream storageAPI +} + +func (api dryRunStorageAPI) List(ctx context.Context, dir string) *bunny.Iterator { + return api.upstream.List(ctx, dir) +} + +func (api dryRunStorageAPI) Upload(ctx context.Context, path string, body []byte) error { + return nil +} + +func (api dryRunStorageAPI) Delete(ctx context.Context, path string) error { + return nil +} + +func (api dryRunStorageAPI) DeleteDir(ctx context.Context, path string) error { + return nil +} diff --git a/internal/cli/verbosity_test.go b/internal/cli/verbosity_test.go new file mode 100644 index 0000000..68bda1d --- /dev/null +++ b/internal/cli/verbosity_test.go @@ -0,0 +1,107 @@ +package cli + +import ( + "archive/tar" + "strings" + "testing" + + "tsne.dev/hopper/internal/log" +) + +// verbosityArchive is a small release with one immutable asset and one mutable +// entry point, enough to exercise action lines and the summary. +func verbosityArchive(t *testing.T) string { + return makeArchive(t, []tarEntry{ + {name: "index.html", body: "<html>"}, + {name: "assets/app.abc.js", body: "console.log(1)"}, + // A symlink triggers an always-on warning regardless of mode. + {name: "assets/link.js", typeflag: tar.TypeSymlink, linkname: "app.abc.js"}, + }) +} + +func TestPushDefaultOutput(t *testing.T) { + arc := verbosityArchive(t) + fc := newFakeAPI() + out, errb := captureLogs(t) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + + stdout := out.String() + if !strings.Contains(stdout, "uploading assets/app.abc.js") { + t.Fatalf("default output missing action line: %q", stdout) + } + if !strings.Contains(stdout, "uploaded: 2") { + t.Fatalf("default output missing summary: %q", stdout) + } + if !strings.Contains(errb.String(), "Warning:") { + t.Fatalf("warning missing from stderr: %q", errb.String()) + } +} + +func TestPushQuietOnlySummaryAndWarnings(t *testing.T) { + arc := verbosityArchive(t) + fc := newFakeAPI() + out, errb := captureLogs(t) + log.SetQuiet(true) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + + stdout := out.String() + if strings.Contains(stdout, "uploading") { + t.Fatalf("quiet must suppress action lines: %q", stdout) + } + if !strings.Contains(stdout, "uploaded: 2") { + t.Fatalf("quiet must keep summary: %q", stdout) + } + if !strings.Contains(errb.String(), "Warning:") { + t.Fatalf("quiet must keep warnings on stderr: %q", errb.String()) + } +} + +func TestPushQuietWinsOverVerbose(t *testing.T) { + arc := verbosityArchive(t) + fc := newFakeAPI() + out, errb := captureLogs(t) + log.SetVerbose(true) + log.SetQuiet(true) + + if err := runPush(arc, fc); err != nil { + t.Fatalf("push: %v", err) + } + + stdout := out.String() + if strings.Contains(stdout, "uploading") { + t.Fatalf("quiet+verbose must suppress action lines: %q", stdout) + } + if !strings.Contains(stdout, "uploaded: 2") { + t.Fatalf("quiet+verbose must keep summary: %q", stdout) + } + if !strings.Contains(errb.String(), "Warning:") { + t.Fatalf("quiet+verbose must keep warnings: %q", errb.String()) + } +} + +func TestLogStreamRouting(t *testing.T) { + out, errb := captureLogs(t) + + log.Print("normal") + log.PrintImportant("summary") + log.Warn("careful") + log.Error("boom") + + stdout := out.String() + stderr := errb.String() + if !strings.Contains(stdout, "normal") || !strings.Contains(stdout, "summary") { + t.Fatalf("stdout missing normal output: %q", stdout) + } + if strings.Contains(stdout, "careful") || strings.Contains(stdout, "boom") { + t.Fatalf("warnings/errors leaked to stdout: %q", stdout) + } + if !strings.Contains(stderr, "Warning: careful") || !strings.Contains(stderr, "Error: boom") { + t.Fatalf("stderr missing warning/error: %q", stderr) + } +} diff --git a/internal/cli/worker_pool.go b/internal/cli/worker_pool.go new file mode 100644 index 0000000..fe44644 --- /dev/null +++ b/internal/cli/worker_pool.go @@ -0,0 +1,110 @@ +package cli + +import ( + "context" + "errors" + "runtime" + "sync" + "sync/atomic" +) + +// pool is a bounded worker pool for concurrent, independent tasks (§14). A +// single caller feeds work via submit; up to concurrency tasks run in flight at +// once, bounded by the sem token held from submit until the task returns. +// +// The pool is deliberately upload-agnostic: a task is any func(ctx) error, so +// the same machinery serves push uploads today and prune deletes later. Task +// wording, error identity, and any success side-effect live in the closure. +// +// submit is the only place that logs a per-action line, and it is only ever +// called from one goroutine (the caller feeding work), so output needs no +// locking; only the shared counters are guarded. Tasks run concurrently, so any +// state a task closure touches beyond its own arguments is the caller's +// responsibility to synchronize. +type workerPool struct { + ctx context.Context + cancel context.CancelCauseFunc + sem chan struct{} + wg sync.WaitGroup + + // failFast cancels the pool on the first failure so in-flight tasks are + // abandoned and queued ones never start (the push-asset policy, §8.3). When + // false, every submitted task runs and all failures are collected (the + // commit/prune-delete policy). + failFast bool + + mtx sync.Mutex + errs []error + succeeded int64 +} + +// newWorkerPool creates a new worker pool to process tasks concurrently. If `concurrency` +// is zero, the number of parallel jobs is aut-detected based on the number of CPUs. +func newWorkerPool(ctx context.Context, concurrency uint, failFast bool) *workerPool { + if concurrency == 0 { + concurrency = 2 * uint(runtime.NumCPU()) + } + ctx, cancel := context.WithCancelCause(ctx) + return &workerPool{ + ctx: ctx, + cancel: cancel, + sem: make(chan struct{}, concurrency), + failFast: failFast, + } +} + +// submit blocks until a worker slot is free, then runs task in the background. +// It reports false when the pool has been cancelled (fail-fast or the parent +// context), signalling the caller to stop feeding work. action is the text that +// will be logged before the task is executed. +func (p *workerPool) submit(action string, task func(ctx context.Context) error) bool { + select { + case p.sem <- struct{}{}: + case <-p.ctx.Done(): + return false + } + + logAction(action) + + p.wg.Add(1) + go func() { + defer p.wg.Done() + defer func() { <-p.sem }() + + if err := task(p.ctx); err != nil { + if !errors.Is(err, context.Canceled) { + p.mtx.Lock() + p.errs = append(p.errs, err) + p.mtx.Unlock() + } + if p.failFast { + p.cancel(CancellationError("operation aborted")) + } + return + } + atomic.AddInt64(&p.succeeded, 1) + }() + return true +} + +// cancelled reports whether the pool's context is already done (fail-fast +// triggered or the parent context cancelled). A feeder loop can poll this to +// stop producing/buffering work promptly, instead of only discovering the +// cancellation on its next submit. +func (p *workerPool) cancelled() bool { + select { + case <-p.ctx.Done(): + return true + default: + return false + } +} + +// wait blocks until all in-flight tasks finish, then releases the pool's +// context. It returns the number of succeeded tasks and the errors the +// workers collected. +func (p *workerPool) wait() (int, []error) { + p.wg.Wait() + p.cancel(nil) + return int(p.succeeded), p.errs +} diff --git a/internal/cli/worker_pool_test.go b/internal/cli/worker_pool_test.go new file mode 100644 index 0000000..db3edf0 --- /dev/null +++ b/internal/cli/worker_pool_test.go @@ -0,0 +1,70 @@ +package cli + +import ( + "context" + "errors" + "testing" +) + +// TestWorkerPoolFailFastSwallowsCancellation locks in the §8.3 invariant that a +// fail-fast pool reports only GENUINE failures: the first real error cancels the +// pool, and the in-flight siblings that consequently return context.Canceled are +// swallowed (cancellation collateral is never an independent per-file failure). +func TestWorkerPoolFailFastSwallowsCancellation(t *testing.T) { + pool := newWorkerPool(context.Background(), 8, true) + + // Occupy several slots with tasks that block until the pool is cancelled and + // then return context.Canceled (the collateral that must be swallowed). + const blockers = 5 + for i := 0; i < blockers; i++ { + pool.submit("block", func(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() // context.Canceled + }) + } + + // One genuine failure triggers fail-fast cancellation of the blockers above. + pool.submit("fail", func(ctx context.Context) error { + return fileError{path: "a.js", err: errors.New("boom")} + }) + + succeeded, errs := pool.wait() + if succeeded != 0 { + t.Fatalf("succeeded = %d, want 0", succeeded) + } + if len(errs) != 1 { + t.Fatalf("errs = %d (%v), want exactly 1 genuine failure", len(errs), errs) + } + if errors.Is(errs[0], context.Canceled) { + t.Fatalf("reported error is cancellation collateral, want the genuine failure: %v", errs[0]) + } + var fe fileError + if !errors.As(errs[0], &fe) || fe.path != "a.js" { + t.Fatalf("reported error = %v, want fileError for a.js", errs[0]) + } +} + +// TestWorkerPoolDrainReportsAll confirms the non-fail-fast (drain-and-report) +// mode collects every failure and still runs all tasks. +func TestWorkerPoolDrainReportsAll(t *testing.T) { + pool := newWorkerPool(context.Background(), 4, false) + + const tasks = 6 + for i := 0; i < tasks; i++ { + odd := i%2 == 1 + pool.submit("t", func(ctx context.Context) error { + if odd { + return errors.New("fail") + } + return nil + }) + } + + succeeded, errs := pool.wait() + if succeeded != 3 { + t.Fatalf("succeeded = %d, want 3", succeeded) + } + if len(errs) != 3 { + t.Fatalf("errs = %d, want 3", len(errs)) + } +} diff --git a/internal/log/log.go b/internal/log/log.go new file mode 100644 index 0000000..68635c3 --- /dev/null +++ b/internal/log/log.go @@ -0,0 +1,70 @@ +// Package log centralizes all user-facing output behind package-level +// functions backed by a private global logger. Normal output (actions, summary) +// goes to stdout; warnings and errors go to stderr. For a single-process CLI a +// global sink avoids threading a logger through every call site. +package log + +import ( + "fmt" + "io" + "os" +) + +// global is the process-wide logger. It defaults to the real stdout/stderr so the +// package is usable without explicit setup (e.g. in tests). +var global = &logger{stdout: os.Stdout, stderr: os.Stderr} + +type logger struct { + stdout io.Writer + stderr io.Writer + verbose bool + quiet bool +} + +// Init configures the global logger's destinations. Call once at startup. +func Init(out, err io.Writer) { + global.stdout = out + global.stderr = err +} + +// SetVerbose toggles whether commands include extra detail in their output. +func SetVerbose(v bool) { global.verbose = v } + +// SetQuiet toggles whether normal output is suppressed. Quiet wins over verbose +// (see DESIGN §13): the two are points on one scale, quiet > verbose > default. +func SetQuiet(q bool) { global.quiet = q } + +// Verbose reports whether verbose output is enabled. It folds the quiet override +// in, so callers never re-check quiet: quiet suppresses verbose detail. +func Verbose() bool { return global.verbose && !global.quiet } + +// Print writes a normal output line to stdout. It is suppressed under quiet. +func Print(format string, args ...any) { + if !global.quiet { + fmt.Fprintf(global.stdout, format+"\n", args...) + } +} + +// PrintVerbose writes an extra-detail line to stdout, emitted only when verbose +// output is enabled (and not suppressed by quiet, per Verbose()). +func PrintVerbose(format string, args ...any) { + if Verbose() { + fmt.Fprintf(global.stdout, format+"\n", args...) + } +} + +// PrintImportant writes a line to stdout that survives quiet (e.g. the final +// summary). +func PrintImportant(format string, args ...any) { + fmt.Fprintf(global.stdout, format+"\n", args...) +} + +// Warn prints a warning to stderr. Warnings are never suppressed. +func Warn(format string, args ...any) { + fmt.Fprintf(global.stderr, "Warning: "+format+"\n", args...) +} + +// Error prints an error to stderr. Errors are never suppressed. +func Error(format string, args ...any) { + fmt.Fprintf(global.stderr, "Error: "+format+"\n", args...) +} diff --git a/internal/source/source.go b/internal/source/source.go new file mode 100644 index 0000000..f2f99b4 --- /dev/null +++ b/internal/source/source.go @@ -0,0 +1,213 @@ +// Package source provides a sequential, single-pass iterator over release +// entries (regular files to upload). TarGzSource reads a tar.gz; a directory +// source may be added later behind the same interface. +package source + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "path" + "strings" + + "tsne.dev/hopper/internal/log" +) + +// Entry is a single regular file emitted by a Source. +type Entry struct { + Path string // cleaned, archive-relative key (no prefix applied). + Size int + IsMutable bool // true for mutable entry points (*.html, case-insensitive). + + content io.Reader // unexported; live source stream for THIS entry. +} + +// ReadContent reads the current entry's bytes fully into memory. +// On context cancellation it returns the context's cause promptly instead +// of finishing a large read. It is valid ONLY for the current entry and +// must be called before the next Source.Next(). +func (e Entry) ReadContent(ctx context.Context) ([]byte, error) { + const chunkSize = 512 << 10 // 512 KiB + + initialSize := e.Size + if initialSize == 0 { + initialSize = 4096 + } + + buf := make([]byte, initialSize) + off := 0 + for { + if err := context.Cause(ctx); err != nil { + return nil, err + } + + if len(buf)-off < chunkSize { + newbuf := make([]byte, max(2*len(buf), len(buf)+chunkSize)) + copy(newbuf, buf[:off]) + buf = newbuf + } + + read, err := e.content.Read(buf[off : off+chunkSize]) + if read > 0 { + off += read + } + if err != nil { + if errors.Is(err, io.EOF) { + return buf[:off], nil + } + return nil, err + } + } +} + +// Source is a sequential, single-pass iterator. Returns io.EOF when exhausted. +type Source interface { + Next() (Entry, error) + Close() error +} + +// TarGzSource streams compress/gzip -> archive/tar entries one at a time. +type TarGzSource struct { + f *os.File + gz *gzip.Reader + tr *tar.Reader + root string +} + +// NewTarGzSource opens the given tar.gz archive. The given root selects +// the subtree to emit and is stripped from each selected entry. If root +// is empty all entries are selected. +// +// Note: `root` must arrive pre-normalized (no leading/trailing slashes, +// cleaned of "."/".."). The cli package owns root normalization. +func NewTarGzSource(archivePath string, root string) (*TarGzSource, error) { + f, err := os.Open(archivePath) + if err != nil { + return nil, fmt.Errorf("open archive: %w", err) + } + gz, err := gzip.NewReader(f) + if err != nil { + f.Close() + return nil, fmt.Errorf("gzip: %w", err) + } + + return &TarGzSource{ + f: f, + gz: gz, + tr: tar.NewReader(gz), + root: root, + }, nil +} + +// Next returns the next regular-file entry, filtering by entry type and +// applying path-safety rules. Returns io.EOF when exhausted. +// +// Entry-type handling: regular files are emitted; directories are silently +// ignored; symlinks/hardlinks/devices/FIFOs/etc. are skipped with a warning +// (never fatal). +func (s *TarGzSource) Next() (Entry, error) { + for { + hdr, err := s.tr.Next() + if err != nil { + return Entry{}, err // includes io.EOF + } + + switch hdr.Typeflag { + case tar.TypeReg: + // emit below + case tar.TypeDir: + continue // silently ignored + default: + log.Warn("skipping non-regular source entry %q (type %q)", hdr.Name, string(hdr.Typeflag)) + continue + } + + cleaned, err := cleanArchivePath(hdr.Name) + if err != nil { + return Entry{}, err + } + + path, ok := s.selectPath(cleaned) + if !ok { + continue + } + + return Entry{ + Path: path, + Size: int(hdr.Size), + IsMutable: strings.HasSuffix(strings.ToLower(path), ".html"), + content: s.tr, + }, nil + } +} + +// Close releases the underlying resources. +func (s *TarGzSource) Close() error { + gzErr := s.gz.Close() + fErr := s.f.Close() + if gzErr != nil { + return gzErr + } + return fErr +} + +func (s *TarGzSource) selectPath(path string) (string, bool) { + if s.root == "" { + return path, true + } + prefix := s.root + "/" + if !strings.HasPrefix(path, prefix) { + return "", false + } + return path[len(prefix):], true +} + +// cleanArchivePath validates and cleans a raw archive path: +// - absolute path -> hard error; +// - cleaned path escaping the root (starts with "..") -> hard error; +// - stays within root but contained a ".." segment -> emit, with a warning; +// - pure "."/redundant-slash normalization -> silent. +func cleanArchivePath(raw string) (string, error) { + if path.IsAbs(raw) { + return "", fmt.Errorf("absolute path in archive: %q", raw) + } + + cleaned := path.Clean(raw) + + if cleaned == "." { + return "", fmt.Errorf("empty or invalid path in archive: %q", raw) + } + + if cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return "", fmt.Errorf("path escapes archive root: %q", raw) + } + + // Stayed within root but contained a ".." segment -> warn (legal, unusual). + if containsDotDot(raw) { + log.Warn("path %q contains '..' segment", raw) + } + + return cleaned, nil +} + +// containsDotDot reports whether p has a ".." path segment, scanning by index +// to avoid the allocation of strings.Split. +func containsDotDot(p string) bool { + for len(p) > 0 { + i := strings.IndexByte(p, '/') + var seg string + if i < 0 { + seg, p = p, "" + } else { + seg, p = p[:i], p[i+1:] + } + if seg == ".." { + return true + } + } + return false +} diff --git a/internal/source/source_test.go b/internal/source/source_test.go new file mode 100644 index 0000000..933c67a --- /dev/null +++ b/internal/source/source_test.go @@ -0,0 +1,95 @@ +package source + +import ( + "bytes" + "io" + "testing" + + "tsne.dev/hopper/internal/log" +) + +func TestCleanArchivePath(t *testing.T) { + tests := []struct { + raw string + want string + wantErr bool + warn bool + }{ + {"a/b.js", "a/b.js", false, false}, + {"./a//b.js", "a/b.js", false, false}, + {"/abs/x", "", true, false}, + {"../escape", "", true, false}, + {"a/../../escape", "", true, false}, + {"a/../b.js", "b.js", false, true}, // contained .. but stays in root + } + for _, tt := range tests { + var warn bytes.Buffer + log.Init(io.Discard, &warn) + got, err := cleanArchivePath(tt.raw) + if tt.wantErr { + if err == nil { + t.Errorf("%q: expected error", tt.raw) + } + continue + } + if err != nil { + t.Errorf("%q: unexpected error %v", tt.raw, err) + continue + } + if got != tt.want { + t.Errorf("%q: got %q want %q", tt.raw, got, tt.want) + } + if tt.warn && warn.Len() == 0 { + t.Errorf("%q: expected warning", tt.raw) + } + if !tt.warn && warn.Len() != 0 { + t.Errorf("%q: unexpected warning %q", tt.raw, warn.String()) + } + } +} + +func TestSelectPath(t *testing.T) { + tests := []struct { + cleaned string + root string + want string + wantOK bool + }{ + {"assets/x.js", "", "assets/x.js", true}, + {"README.md", "", "README.md", true}, + {"app/assets/x.js", "app", "assets/x.js", true}, + {"README.md", "app", "", false}, + {"app", "app", "", false}, + {"appendix/x", "app", "", false}, + {"dist/public/index.html", "dist/public", "index.html", true}, + {"dist/private/x", "dist/public", "", false}, + } + for _, tt := range tests { + s := &TarGzSource{root: tt.root} + got, ok := s.selectPath(tt.cleaned) + if ok != tt.wantOK { + t.Errorf("selectPath(%q, %q): ok = %v, want %v", tt.cleaned, tt.root, ok, tt.wantOK) + continue + } + if ok && got != tt.want { + t.Errorf("selectPath(%q, %q) = %q, want %q", tt.cleaned, tt.root, got, tt.want) + } + } +} + +func TestContainsDotDot(t *testing.T) { + cases := map[string]bool{ + "a/b/c": false, + "..": true, + "../x": true, + "a/../b": true, + "a/..b/c": false, + "": false, + "a/b/..": true, + } + for in, want := range cases { + if got := containsDotDot(in); got != want { + t.Errorf("containsDotDot(%q) = %v, want %v", in, got, want) + } + } +} |