diff --git a/.changes/core-always-rebuilds-due-to-permissions-dir.md b/.changes/core-always-rebuilds-due-to-permissions-dir.md
new file mode 100644
index 000000000..d59930c69
--- /dev/null
+++ b/.changes/core-always-rebuilds-due-to-permissions-dir.md
@@ -0,0 +1,6 @@
+---
+"tauri-build": "patch:bug"
+---
+
+Fix tauri always rebuilding even if source code didn't change.
+
diff --git a/core/tauri-build/src/acl.rs b/core/tauri-build/src/acl.rs
index ea0961a73..593b41783 100644
--- a/core/tauri-build/src/acl.rs
+++ b/core/tauri-build/src/acl.rs
@@ -360,10 +360,12 @@ pub fn inline_plugins(
       )?);
     } else {
       let default_permissions_path = Path::new("permissions").join(name);
-      println!(
-        "cargo:rerun-if-changed={}",
-        default_permissions_path.display()
-      );
+      if default_permissions_path.exists() {
+        println!(
+          "cargo:rerun-if-changed={}",
+          default_permissions_path.display()
+        );
+      }
       permission_files.extend(tauri_utils::acl::build::define_permissions(
         &default_permissions_path
           .join("**")
@@ -418,10 +420,12 @@ pub fn app_manifest_permissions(
     )?);
   } else {
     let default_permissions_path = Path::new("permissions");
-    println!(
-      "cargo:rerun-if-changed={}",
-      default_permissions_path.display()
-    );
+    if default_permissions_path.exists() {
+      println!(
+        "cargo:rerun-if-changed={}",
+        default_permissions_path.display()
+      );
+    }
 
     let permissions_root = current_dir()?.join("permissions");
     let inlined_plugins_permissions: Vec<_> = inlined_plugins