diff --git a/app/controllers/activitypub/base_controller.rb b/app/controllers/activitypub/base_controller.rb
new file mode 100644
index 000000000..a3b5c4dfa
--- /dev/null
+++ b/app/controllers/activitypub/base_controller.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class ActivityPub::BaseController < Api::BaseController
+  private
+
+  def set_cache_headers
+    response.headers['Vary'] = 'Signature' if authorized_fetch_mode?
+  end
+end
diff --git a/app/controllers/activitypub/collections_controller.rb b/app/controllers/activitypub/collections_controller.rb
index 035467f41..fa925b204 100644
--- a/app/controllers/activitypub/collections_controller.rb
+++ b/app/controllers/activitypub/collections_controller.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ActivityPub::CollectionsController < Api::BaseController
+class ActivityPub::CollectionsController < ActivityPub::BaseController
   include SignatureVerification
   include AccountOwnedConcern
 
diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb
index cdfd28ba8..891756b7e 100644
--- a/app/controllers/activitypub/outboxes_controller.rb
+++ b/app/controllers/activitypub/outboxes_controller.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ActivityPub::OutboxesController < Api::BaseController
+class ActivityPub::OutboxesController < ActivityPub::BaseController
   LIMIT = 20
 
   include SignatureVerification
diff --git a/app/controllers/activitypub/replies_controller.rb b/app/controllers/activitypub/replies_controller.rb
index 020c077ab..ab755ed4e 100644
--- a/app/controllers/activitypub/replies_controller.rb
+++ b/app/controllers/activitypub/replies_controller.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ActivityPub::RepliesController < Api::BaseController
+class ActivityPub::RepliesController < ActivityPub::BaseController
   include SignatureAuthentication
   include Authorization
   include AccountOwnedConcern
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 16e7d70a3..26f3b1def 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -160,6 +160,6 @@ class ApplicationController < ActionController::Base
   end
 
   def set_cache_headers
-    response.headers['Vary'] = 'Accept, Signature'
+    response.headers['Vary'] = public_fetch_mode? ? 'Accept' : 'Accept, Signature'
   end
 end