FEAT: Support for forwarding response headers from http task (#36)

* FEAT: Support for forwarding response headers from http task

Author: prasadlohakpure

internal/pkg/config/config.go

@@ -14,7 +14,7 @@ import (
 
 const (
 	templateName     = `render`
-	placeholderRegex = "([a-zA-Z0-9_]+?)"
+	placeholderRegex = "([a-zA-Z0-9_-]+?)"
 )
 
 // Generic String type that can evaluate both macro and context templates

internal/pkg/jq/jq.go

@@ -50,7 +50,8 @@ func (q *Query) Execute(document []byte, inputs ...any) (any, error) {
 	var data any
 
 	if err := json.Unmarshal(document, &data); err != nil {
-		return nil, err
+		// If the document is not valid JSON, treat it as a raw string
+		data = string(document)
 	}
 
 	values := make([]any, 0, len(inputs)+1)

internal/pkg/pipeline/task/http/README.md

@@ -16,6 +16,12 @@ The HTTP task operates in two modes depending on whether an input channel is pro
 
 In both modes, the task sends HTTP response data to its output channel and supports automatic retries, OAuth authentication, and proxy configuration.
 
+### Response Format and Headers
+
+The HTTP task outputs the response body as-is (maintains backward compatibility). Response headers are automatically stored in the record's context with the prefix `http-header-`, making them accessible to downstream tasks via context variables.
+
+For example, if the HTTP response includes a `Content-Type` header, it will be available as `{{ context "http-header-Content-Type" }}` in subsequent tasks. Note that HTTP header names are case-sensitive when used as context keys. Go's HTTP library canonicalizes header names (for example, `content-type` becomes `Content-Type`), so you must use the canonical form when accessing headers via context (for example, `http-header-Content-Type`, not `http-header-content-type`).
+
 ## Configuration Fields
 
 | Field | Type | Default | Description |
@@ -33,6 +39,7 @@ In both modes, the task sends HTTP response data to its output channel and suppo
 | `oauth` | object | - | OAuth configuration (see OAuth section) |
 | `proxy` | object | - | Proxy configuration |
 | `next_page` | string/object | - | JQ expression to extract next page URL (string) or pagination config (object with`endpoint`, `method`, `body`, `headers`) |
+| `context` | map[string]string | - | JQ expressions to extract values from the response and store in record context |
 | `fail_on_error` | bool | `false` | Whether to stop the pipeline if this task encounters an error |
 
 ## OAuth Configuration
@@ -96,6 +103,27 @@ tasks:
       Authorization: Bearer {{ context "auth_token" }}
 ```
 
+### Setting context values from response and accessing headers:
+```yaml
+tasks:
+  - name: fetch_user
+    type: http
+    method: GET
+    endpoint: https://api.example.com/user/123
+    context:
+      user_name: ".data | fromjson | .name"
+      user_email: ".data | fromjson | .email"
+  
+  - name: use_context
+    type: jq
+    path: |
+      {
+        "greeting": "Hello {{ context "user_name" }}",
+        "email": "{{ context "user_email" }}",
+        "content_type": "{{ context "http-header-Content-Type" }}"
+      }
+```
+
 ## Sample Pipelines
 
 - `test/pipelines/convert_industries.yaml` - HTTP GET request to fetch CSV data

internal/pkg/pipeline/task/http/http.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/patterninc/caterpillar/internal/pkg/config"
@@ -22,6 +23,7 @@ const (
 	defaultExpectedStatuses = `200`
 	defaultMethod           = http.MethodGet
 	defaultTimeout          = duration.Duration(90 * time.Second)
+	headerContextPrefix     = "http-header-%s"
 )
 
 var (
@@ -170,6 +172,13 @@ func (h *httpCore) processItem(rc *record.Record, output chan<- *record.Record)
 		}
 
 		if output != nil {
+			// Store headers in the context for downstream tasks to access
+			// Header names are preserved as-is and stored with the http-header- prefix
+			for headerName, headerValues := range result.Headers {
+				contextKey := fmt.Sprintf(headerContextPrefix, headerName)
+				rc.SetContextValue(contextKey, strings.Join(headerValues, "; "))
+			}
+
 			h.SendData(rc.Context, []byte(result.Data), output)
 		}
 

test/pipelines/context_test.yaml

@@ -30,6 +30,8 @@ tasks:
         "user_name": "{{ context "user_name" }}",
         "user_email": "{{ context "user_email" }}",
         "user_company": "{{ context "user_company" }}",
+        "headers_content_type": "{{ context "http-header-Content-Type" }}",
+        "headers_age": "{{ context "http-header-Age" }}",
         "original_data": .
       }