[http handler] add GET fallback if HEAD is not allowed or forbidden to get key size

http.go

@@ -75,35 +75,55 @@ func HTTPHandle(ctx context.Context, opts ...HTTPOption) (*HTTPHandler, error) {
 	return handler, nil
 }
 
-func handleResponse(r *http.Response) (io.ReadCloser, int64, error) {
+func handleResponse(r *http.Response) (int64, error) {
 	if r.StatusCode == 404 {
-		return nil, -1, syscall.ENOENT
+		return -1, syscall.ENOENT
 	}
 	if r.StatusCode == 416 {
-		return nil, 0, io.EOF
+		return 0, io.EOF
 	}
-	return nil, 0, fmt.Errorf("new reader for %s: status code %d", r.Request.URL.String(), r.StatusCode)
+	return 0, fmt.Errorf("reader for %s: status code %d", r.Request.URL.String(), r.StatusCode)
+}
+
+func (h *HTTPHandler) keySize(key string) (int64, error) {
+	req, _ := http.NewRequestWithContext(h.ctx, "HEAD", key, nil)
+	for _, mw := range h.requestMiddlewares {
+		mw(req)
+	}
+	r, err := h.client.Do(req)
+	if err != nil {
+		return 0, fmt.Errorf("reader for %s: %w", key, err)
+	}
+	defer r.Body.Close()
+	if r.StatusCode == 200 {
+		return r.ContentLength, nil
+	}
+	if r.StatusCode != 403 && r.StatusCode != 405 {
+		return handleResponse(r)
+	}
+	// retry with get
+	req.Method = "GET"
+	r, err = h.client.Do(req)
+	if err != nil {
+		return 0, fmt.Errorf("reader for %s: %w", key, err)
+	}
+	defer r.Body.Close()
+	if r.StatusCode == 200 {
+		return r.ContentLength, nil
+	}
+	return handleResponse(r)
 }
 
 func (h *HTTPHandler) StreamAt(key string, off int64, n int64) (io.ReadCloser, int64, error) {
 	// HEAD request to get object size as it is not returned in range requests
 	var size int64
+	var err error
 	if off == 0 {
-		req, _ := http.NewRequestWithContext(h.ctx, "HEAD", key, nil)
-		for _, mw := range h.requestMiddlewares {
-			mw(req)
-		}
-		r, err := h.client.Do(req)
+		size, err = h.keySize(key)
 		if err != nil {
-			return nil, 0, fmt.Errorf("new reader for %s: %w", key, err)
-		}
-		defer r.Body.Close()
-		if r.StatusCode != 200 {
-			return handleResponse(r)
+			return nil, size, err
 		}
-		size = r.ContentLength
 	}
-
 	// GET request to fetch range
 	req, _ := http.NewRequestWithContext(h.ctx, "GET", key, nil)
 	for _, mw := range h.requestMiddlewares {
@@ -112,10 +132,11 @@ func (h *HTTPHandler) StreamAt(key string, off int64, n int64) (io.ReadCloser, i
 	req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+n-1))
 	r, err := h.client.Do(req)
 	if err != nil {
-		return nil, 0, fmt.Errorf("new reader for %s: %w", key, err)
+		return nil, 0, fmt.Errorf("reader for %s: %w", key, err)
 	}
 	if r.StatusCode != 200 && r.StatusCode != 206 {
-		return handleResponse(r)
+		size, err = handleResponse(r)
+		return nil, size, err
 	}
 	return r.Body, size, err
 }

http_test.go

@@ -15,20 +15,41 @@
 package osio
 
 import (
+	"bytes"
 	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
 	"syscall"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
+type httpmock struct {
+	*http.Client
+	ok *bool
+}
+
+func (m httpmock) Do(req *http.Request) (*http.Response, error) {
+	fmt.Println(req)
+	if req.Method == "HEAD" {
+		*m.ok = true
+		return &http.Response{
+			StatusCode: http.StatusMethodNotAllowed,
+			Body:       ioutil.NopCloser(&bytes.Buffer{}),
+		}, nil
+	}
+	return m.Client.Do(req)
+}
+
 func TestHTTP(t *testing.T) {
 	ctx := context.Background()
 	hh, _ := HTTPHandle(ctx)
 	httpa, _ := NewAdapter(hh)
 
 	// bucket not found
-	_, err := httpa.Reader("https://storage.googleapis.com/godal-ci-data-public/doesnotexist.tif")
+	_, err := httpa.Reader("https://storage.googleapis.com/godal-ci-data-public-notexists/test.tif")
 	assert.Equal(t, err, syscall.ENOENT)
 
 	// object not found
@@ -39,7 +60,13 @@ func TestHTTP(t *testing.T) {
 	r, err := httpa.Reader("https://storage.googleapis.com/godal-ci-data-public/test.tif")
 	assert.NoError(t, err)
 	assert.Equal(t, int64(212), r.Size())
+
+	// check fallback
+	var ok bool
+	hh, _ = HTTPHandle(ctx, HTTPClient(httpmock{Client: &http.Client{}, ok: &ok}))
+	httpa, _ = NewAdapter(hh)
 	r, err = httpa.Reader("https://storage.googleapis.com/godal-ci-data-public/test.tif")
 	assert.NoError(t, err)
 	assert.Equal(t, int64(212), r.Size())
+	assert.True(t, ok)
 }