diff --git a/modules/caddyhttp/headers/headers.go b/modules/caddyhttp/headers/headers.go
index 3571dd92..4cef0a9f 100644
--- a/modules/caddyhttp/headers/headers.go
+++ b/modules/caddyhttp/headers/headers.go
@@ -213,7 +213,7 @@ func (ops HeaderOps) ApplyTo(hdr http.Header, repl *caddy.Replacer) {
 
 	// replace
 	for fieldName, replacements := range ops.Replace {
-		fieldName = repl.ReplaceAll(fieldName, "")
+		fieldName = http.CanonicalHeaderKey(repl.ReplaceAll(fieldName, ""))
 
 		// all fields...
 		if fieldName == "*" {
@@ -237,11 +237,17 @@ func (ops HeaderOps) ApplyTo(hdr http.Header, repl *caddy.Replacer) {
 		for _, r := range replacements {
 			search := repl.ReplaceAll(r.Search, "")
 			replace := repl.ReplaceAll(r.Replace, "")
-			for i := range hdr[fieldName] {
-				if r.re != nil {
-					hdr[fieldName][i] = r.re.ReplaceAllString(hdr[fieldName][i], replace)
-				} else {
-					hdr[fieldName][i] = strings.ReplaceAll(hdr[fieldName][i], search, replace)
+			for hdrFieldName, vals := range hdr {
+				// see issue #4330 for why we don't simply use hdr[fieldName]
+				if http.CanonicalHeaderKey(hdrFieldName) != fieldName {
+					continue
+				}
+				for i := range vals {
+					if r.re != nil {
+						hdr[hdrFieldName][i] = r.re.ReplaceAllString(hdr[hdrFieldName][i], replace)
+					} else {
+						hdr[hdrFieldName][i] = strings.ReplaceAll(hdr[hdrFieldName][i], search, replace)
+					}
 				}
 			}
 		}
diff --git a/modules/caddyhttp/headers/headers_test.go b/modules/caddyhttp/headers/headers_test.go
index 11bdb0df..fb68225e 100644
--- a/modules/caddyhttp/headers/headers_test.go
+++ b/modules/caddyhttp/headers/headers_test.go
@@ -160,6 +160,28 @@ func TestHandler(t *testing.T) {
 				"Fail-5xx": []string{"true"},
 			},
 		},
+		{
+			handler: Handler{
+				Request: &HeaderOps{
+					Replace: map[string][]Replacement{
+						"Case-Insensitive": {
+							Replacement{
+								Search:  "issue4330",
+								Replace: "issue #4330",
+							},
+						},
+					},
+				},
+			},
+			reqHeader: http.Header{
+				"case-insensitive": []string{"issue4330"},
+				"Other-Header":     []string{"issue4330"},
+			},
+			expectedReqHeader: http.Header{
+				"case-insensitive": []string{"issue #4330"},
+				"Other-Header":     []string{"issue4330"},
+			},
+		},
 	} {
 		rr := httptest.NewRecorder()
 
@@ -191,7 +213,7 @@ func TestHandler(t *testing.T) {
 		})
 
 		if err := tc.handler.ServeHTTP(rr, req, next); err != nil {
-			t.Errorf("Test %d: %w", i, err)
+			t.Errorf("Test %d: %v", i, err)
 			continue
 		}