diff --git a/lib/src/main/java/org/asamk/signal/manager/helper/GroupV2Helper.java b/lib/src/main/java/org/asamk/signal/manager/helper/GroupV2Helper.java
index 21db29df..b5b76c16 100644
--- a/lib/src/main/java/org/asamk/signal/manager/helper/GroupV2Helper.java
+++ b/lib/src/main/java/org/asamk/signal/manager/helper/GroupV2Helper.java
@@ -476,10 +476,25 @@ class GroupV2Helper {
             throw new IOException("Cannot join a V2 group as self does not have a versioned profile");
         }
 
-        final var change = groupOperations.createAcceptInviteChange(profileKeyCredential);
+        // We need to accept the invite with the ACI or PNI that was used for the invitation
+        final var selfAddress = context.getRecipientHelper().resolveSignalServiceAddress(selfRecipientId);
+        final var selfServiceId = groupInfoV2.getPendingMemberAddresses()
+                .stream()
+                .filter(s -> s.matches(selfAddress))
+                .findFirst();
+        if (selfServiceId.isEmpty()) {
+            throw new IOException("Cannot find service ID for self to accept invite");
+        }
+        final var serviceId = selfServiceId.get().getServiceId();
 
-        final var aci = context.getRecipientHelper().resolveSignalServiceAddress(selfRecipientId).getServiceId();
-        change.sourceUserId(aci.toByteString());
+        final GroupChange.Actions.Builder change;
+        if (serviceId instanceof ACI) {
+            change = groupOperations.createAcceptInviteChange(profileKeyCredential);
+        } else {
+            change = groupOperations.createAcceptPniInviteChange(profileKeyCredential);
+        }
+
+        change.sourceUserId(serviceId.toByteString());
 
         return commitChange(groupInfoV2, change);
     }
diff --git a/lib/src/main/java/org/asamk/signal/manager/storage/groups/GroupInfoV2.java b/lib/src/main/java/org/asamk/signal/manager/storage/groups/GroupInfoV2.java
index 8c370286..dbb123c5 100644
--- a/lib/src/main/java/org/asamk/signal/manager/storage/groups/GroupInfoV2.java
+++ b/lib/src/main/java/org/asamk/signal/manager/storage/groups/GroupInfoV2.java
@@ -13,6 +13,7 @@ import org.signal.storageservice.storage.protos.groups.Member;
 import org.signal.storageservice.storage.protos.groups.local.DecryptedGroup;
 import org.signal.storageservice.storage.protos.groups.local.EnabledState;
 import org.whispersystems.signalservice.api.push.DistributionId;
+import org.whispersystems.signalservice.api.push.SignalServiceAddress;
 
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -153,6 +154,15 @@ public final class GroupInfoV2 extends GroupInfo {
                 .collect(Collectors.toSet());
     }
 
+    public Set<SignalServiceAddress> getPendingMemberAddresses() {
+        if (this.group == null) {
+            return Set.of();
+        }
+        return group.pendingMembers.stream()
+                .map(m -> new SignalServiceAddress(ServiceId.parseOrThrow(m.serviceIdBytes)))
+                .collect(Collectors.toSet());
+    }
+
     @Override
     public Set<RecipientId> getRequestingMembers() {
         if (this.group == null) {