Fix Bug: retry scrape will lost POST requestData

colly.go

@@ -233,6 +233,8 @@ var (
  ErrQueueFull = errors.New("Queue MaxSize reached")
  // ErrMaxRequests is the error returned when exceeding max requests
  ErrMaxRequests = errors.New("Max Requests limit reached")
+ // ErrRetryBodyUnseekable is the error when retry with not seekable body
+ ErrRetryBodyUnseekable = errors.New("Retry Body Unseekable")
 )
 
 var envMap = map[string]func(*Collector, string){
@@ -629,6 +631,13 @@ func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, c
  if _, ok := hdr["User-Agent"]; !ok {
  hdr.Set("User-Agent", c.UserAgent)
  }
+ if seeker, ok := requestData.(io.ReadSeeker); ok {
+ _, err := seeker.Seek(0, io.SeekStart)
+ if err != nil {
+ return err
+ }
+ }
+
  req, err := http.NewRequest(method, parsedURL.String(), requestData)
  if err != nil {
  return err
@@ -1440,7 +1449,8 @@ func createMultipartReader(boundary string, data map[string][]byte) io.Reader {
  buffer.WriteString("\n")
  }
  buffer.WriteString(dashBoundary + "--\n\n")
- return buffer
+ return bytes.NewReader(buffer.Bytes())
+
 }
 
 // randomBoundary was borrowed from

colly_test.go

@@ -1703,3 +1703,82 @@ func requireSessionCookieAuthPage(handler http.Handler) http.Handler {
  handler.ServeHTTP(w, r)
  })
 }
+
+func TestCollectorPostRetry(t *testing.T) {
+ ts := newTestServer()
+ defer ts.Close()
+
+ postValue := "hello"
+ c := NewCollector()
+ try := false
+ c.OnResponse(func(r *Response) {
+ if r.Ctx.Get("notFirst") == "" {
+ r.Ctx.Put("notFirst", "first")
+ _ = r.Request.Retry()
+ return
+ }
+ if postValue != string(r.Body) {
+ t.Error("Failed to send data with POST")
+ }
+ try = true
+ })
+
+ c.Post(ts.URL+"/login", map[string]string{
+ "name": postValue,
+ })
+ if !try {
+ t.Error("OnResponse Retry was not called")
+ }
+}
+func TestCollectorGetRetry(t *testing.T) {
+ ts := newTestServer()
+ defer ts.Close()
+ try := false
+
+ c := NewCollector()
+
+ c.OnResponse(func(r *Response) {
+ if r.Ctx.Get("notFirst") == "" {
+ r.Ctx.Put("notFirst", "first")
+ _ = r.Request.Retry()
+ return
+ }
+ if !bytes.Equal(r.Body, serverIndexResponse) {
+ t.Error("Response body does not match with the original content")
+ }
+ try = true
+ })
+
+ c.Visit(ts.URL)
+ if !try {
+ t.Error("OnResponse Retry was not called")
+ }
+}
+
+func TestCollectorPostRetryUnseekable(t *testing.T) {
+ ts := newTestServer()
+ defer ts.Close()
+ try := false
+ postValue := "hello"
+ c := NewCollector()
+
+ c.OnResponse(func(r *Response) {
+ if postValue != string(r.Body) {
+ t.Error("Failed to send data with POST")
+ }
+
+ if r.Ctx.Get("notFirst") == "" {
+ r.Ctx.Put("notFirst", "first")
+ err := r.Request.Retry()
+ if !errors.Is(err, ErrRetryBodyUnseekable) {
+ t.Errorf("Unexpected error Type ErrRetryBodyUnseekable : %v", err)
+ }
+ return
+ }
+ try = true
+ })
+ c.Request("POST", ts.URL+"/login", bytes.NewBuffer([]byte("name="+postValue)), nil, nil)
+ if try {
+ t.Error("OnResponse Retry was called but BodyUnseekable")
+ }
+}

request.go

@@ -152,6 +152,9 @@ func (r *Request) PostMultipart(URL string, requestData map[string][]byte) error
 // Retry submits HTTP request again with the same parameters
 func (r *Request) Retry() error {
  r.Headers.Del("Cookie")
+ if _, ok := r.Body.(io.ReadSeeker); r.Body != nil && !ok {
+ return ErrRetryBodyUnseekable
+ }
  return r.collector.scrape(r.URL.String(), r.Method, r.Depth, r.Body, r.Ctx, *r.Headers, false)
 }