diff --git a/lib/src/main/java/org/asamk/signal/manager/syncStorage/ContactRecordProcessor.java b/lib/src/main/java/org/asamk/signal/manager/syncStorage/ContactRecordProcessor.java
index aece5a14..ed0b4546 100644
--- a/lib/src/main/java/org/asamk/signal/manager/syncStorage/ContactRecordProcessor.java
+++ b/lib/src/main/java/org/asamk/signal/manager/syncStorage/ContactRecordProcessor.java
@@ -117,24 +117,35 @@ public class ContactRecordProcessor extends DefaultStorageRecordProcessor<Signal
 
         IdentityState identityState;
         ByteString identityKey;
-        if (remote.identityKey.size() > 0 && (
-                !account.isPrimaryDevice()
-                        || remote.identityState != local.identityState
-                        || local.identityKey.size() == 0
-
-        )) {
-            identityState = remote.identityState;
-            identityKey = remote.identityKey;
-        } else {
-            identityState = local.identityState;
-            identityKey = local.identityKey.size() > 0 ? local.identityKey : ByteString.EMPTY;
-        }
 
+        // Parse ACI/PNI first so we can determine if contact has identity
         final var localAci = ACI.parseOrNull(local.aci, local.aciBinary);
         final var localPni = PNI.parseOrNull(local.pni, local.pniBinary);
         final var remoteAci = ACI.parseOrNull(remote.aci, remote.aciBinary);
         final var remotePni = PNI.parseOrNull(remote.pni, remote.pniBinary);
 
+        final var hasLocalIdentity = localAci != null || localPni != null;
+        final var hasRemoteIdentity = remoteAci != null || remotePni != null;
+
+        final var remoteIdentityKeySize = remote.identityKey.size();
+        final var localIdentityKeySize = local.identityKey.size();
+        final var statesDiffer = remote.identityState != local.identityState;
+
+        if (remoteIdentityKeySize > 0 && (!account.isPrimaryDevice() || statesDiffer || localIdentityKeySize == 0)) {
+            identityState = remote.identityState;
+            identityKey = remote.identityKey;
+        } else {
+            identityState = local.identityState;
+            // Only use local's identity key if:
+            // 1. Contact has ACI or PNI
+            // 2. Remote also has an identity key (if remote size=0, respect that decision)
+            if (hasLocalIdentity && localIdentityKeySize > 0 && remoteIdentityKeySize > 0) {
+                identityKey = local.identityKey;
+            } else {
+                identityKey = ByteString.EMPTY;
+            }
+        }
+
         if (localAci != null
                 && local.identityKey.size() > 0
                 && remote.identityKey.size() > 0
diff --git a/lib/src/main/java/org/asamk/signal/manager/syncStorage/StorageSyncModels.java b/lib/src/main/java/org/asamk/signal/manager/syncStorage/StorageSyncModels.java
index eb3b8ab7..3811b548 100644
--- a/lib/src/main/java/org/asamk/signal/manager/syncStorage/StorageSyncModels.java
+++ b/lib/src/main/java/org/asamk/signal/manager/syncStorage/StorageSyncModels.java
@@ -92,6 +92,7 @@ public final class StorageSyncModels {
 
     public static ContactRecord localToRemoteRecord(Recipient recipient, IdentityInfo identity) {
         final var address = recipient.getAddress();
+        final var aciPresent = address.aci().isPresent();
         final var pniPresent = address.pni().isPresent();
 
         final var builder = SignalContactRecord.Companion.newBuilder(recipient.getStorageRecord())
@@ -128,7 +129,7 @@ public final class StorageSyncModels {
                     .archived(recipient.getContact().isArchived())
                     .hidden(recipient.getContact().isHidden());
         }
-        if (identity != null) {
+        if (identity != null && aciPresent) {
             builder.identityKey(ByteString.of(identity.getIdentityKey().serialize()))
                     .identityState(localToRemote(identity.getTrustLevel()));
         }