diff --git a/mypy/build.py b/mypy/build.py
index 96a67105c816..716a97cdcaca 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -3806,6 +3806,16 @@ def find_module_and_diagnose(
             raise ModuleNotFound
 
 
+def excluded_by_follow_imports(path: str, options: Options) -> bool:
+    """Check if the module at path is excluded from build by follow-imports=skip/error.
+
+    Stubs are only excluded if follow_imports_for_stubs is set.
+    """
+    return options.follow_imports in ("skip", "error") and (
+        not path.endswith(".pyi") or options.follow_imports_for_stubs
+    )
+
+
 def exist_added_packages(suppressed: list[str], manager: BuildManager) -> bool:
     """Find if there are any newly added packages that were previously suppressed.
 
@@ -3829,9 +3839,7 @@ def exist_added_packages(suppressed: list[str], manager: BuildManager) -> bool:
         #     follow-imports = normal
         # But such cases are extremely rare, and this allows us to avoid
         # massive performance impact in much more common situations.
-        if options.follow_imports in ("skip", "error") and (
-            not path.endswith(".pyi") or options.follow_imports_for_stubs
-        ):
+        if excluded_by_follow_imports(path, options):
             continue
         if os.path.basename(path) in ("__init__.py", "__init__.pyi"):
             return True
diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py
index 066fbf9bed2d..6707ebbdce30 100644
--- a/mypy/dmypy_server.py
+++ b/mypy/dmypy_server.py
@@ -825,8 +825,9 @@ def find_added_suppressed(
                 continue
             result = finder.find_module(module, fast_path=True)
             if isinstance(result, str) and module not in seen:
-                # When not following imports, we only follow imports to .pyi files.
-                if not self.following_imports() and not result.endswith(".pyi"):
+                if mypy.build.excluded_by_follow_imports(
+                    result, self.options.clone_for_module(module)
+                ):
                     continue
                 found.append((module, result))
                 seen.add(module)
diff --git a/test-data/unit/fine-grained-follow-imports.test b/test-data/unit/fine-grained-follow-imports.test
index d716a57123dc..aad3ef2ab427 100644
--- a/test-data/unit/fine-grained-follow-imports.test
+++ b/test-data/unit/fine-grained-follow-imports.test
@@ -846,3 +846,30 @@ class A: ...
 [typing fixtures/typing-typeddict.pyi]
 [out]
 ==
+
+[case testFollowImportsPerModuleSkipNotAddedBack]
+# flags: --follow-imports=normal
+# cmd: mypy main.py
+
+[file mypy.ini]
+\[mypy]
+follow_imports = normal
+\[mypy-pkg.sub]
+follow_imports = skip
+
+[file main.py]
+import pkg.sub
+reveal_type(pkg.sub.x)
+
+[file pkg/__init__.py]
+
+[file pkg/sub.py]
+x = 1
+
+[file unrelated.py.2]
+# Trigger a second increment with no relevant changes.
+
+[out]
+main.py:2: note: Revealed type is "Any"
+==
+main.py:2: note: Revealed type is "Any"