diff --git a/files/entrypoint b/files/entrypoint
index 16d2270..9c3d6e3 100755
--- a/files/entrypoint
+++ b/files/entrypoint
@@ -67,14 +67,14 @@ if [ ! -f "$userConfFinalPath" ]; then
     # Generate unique ssh keys for this container, if needed
     if [ ! -f /etc/ssh/ssh_host_ed25519_key ]; then
         ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N ''
-    else
-        chmod 700 /etc/ssh/ssh_host_ed25519_key
     fi
     if [ ! -f /etc/ssh/ssh_host_rsa_key ]; then
         ssh-keygen -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -N ''
-    else
-        chmod 700 /etc/ssh/ssh_host_rsa_key
     fi
+
+    # Restrict access from other users
+    chmod 600 /etc/ssh/ssh_host_ed25519_key || true
+    chmod 600 /etc/ssh/ssh_host_rsa_key || true
 fi
 
 # Source custom scripts, if any